import os import sys import json import tempfile import subprocess import shutil from pathlib import Path # Root of the repo — wherever app.py lives REPO_ROOT = Path(__file__).parent.resolve() import gradio as gr import spaces import numpy as np import torch # ────────────────────────────────────────────── # Lazy model loading (done inside GPU decorator) # ────────────────────────────────────────────── _model_loaded = False def _ensure_models(): global _model_loaded if _model_loaded: return # Models are expected to be pre-downloaded to ./checkpoints/ # on the Space via the HF repo or a setup script. _model_loaded = True # ────────────────────────────────────────────── # Core inference helpers # ────────────────────────────────────────────── def _write_caption_txt(image_path: str, caption: str) -> str: """Write a .txt caption file beside the image and return the directory.""" img_path = Path(image_path) txt_path = img_path.with_suffix(".txt") txt_path.write_text(caption) return str(img_path.parent) def _run(cmd: list[str], desc: str = "") -> tuple[bool, str]: """Run a Python -m command using the current interpreter with PYTHONPATH set.""" # Replace bare "python" / "python3" with the real interpreter path if cmd[0] in ("python", "python3"): cmd = [sys.executable] + cmd[1:] run_env = { **os.environ, "PYTHONPATH": str(REPO_ROOT), # makes lyra_2._src.* importable "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", } print(f"[Lyra] {desc}: {' '.join(cmd)}") result = subprocess.run( cmd, capture_output=True, text=True, env=run_env, cwd=str(REPO_ROOT), # run from repo root ) log = result.stdout + "\n" + result.stderr return result.returncode == 0, log # ────────────────────────────────────────────── # Zoom-in / Zoom-out trajectory (Option 1) # ────────────────────────────────────────────── @spaces.GPU(duration=190) def run_zoomgs( image, caption: str, sample_id: int, zoom_in_strength: float, zoom_out_strength: float, num_frames_in: int, num_frames_out: int, use_dmd: bool, run_reconstruction: bool, ): _ensure_models() with tempfile.TemporaryDirectory() as tmp: # Save uploaded image + caption img_path = Path(tmp) / "input.png" caption_path = Path(tmp) / "input.txt" from PIL import Image Image.fromarray(image).save(img_path) caption_path.write_text(caption.strip() or "A scenic outdoor environment.") output_dir = Path(tmp) / "outputs" / "zoomgs" output_dir.mkdir(parents=True, exist_ok=True) cmd = [ "python", "-m", "lyra_2._src.inference.lyra2_zoomgs_inference", "--input_image_path", str(tmp), "--sample_id", "0", # we always name it input.png → id 0 equivalent "--experiment", "lyra2", "--checkpoint_dir", "checkpoints/model", "--prompt_dir", str(tmp), "--output_path", str(output_dir), "--num_frames_zoom_in", str(num_frames_in), "--num_frames_zoom_out", str(num_frames_out), "--zoom_in_strength", str(zoom_in_strength), "--zoom_out_strength", str(zoom_out_strength), ] if use_dmd: cmd.append("--use_dmd") ok, log = _run(cmd, "ZoomGS video generation") # Locate output video video_path = output_dir / "0" / "videos" / "0.mp4" if not video_path.exists(): # Fallback: search recursively candidates = list(output_dir.rglob("*.mp4")) video_path = candidates[0] if candidates else None gs_video = None if run_reconstruction and video_path and video_path.exists(): ok2, log2 = _run( ["python", "-m", "lyra_2._src.inference.vipe_da3_gs_recon", "--input_video_path", str(video_path)], "GS reconstruction", ) log += "\n" + log2 ply_candidates = list(output_dir.rglob("gs_trajectory.mp4")) if ply_candidates: gs_video = str(ply_candidates[0]) return ( str(video_path) if video_path and video_path.exists() else None, gs_video, log[-4000:], ) # ────────────────────────────────────────────── # Custom trajectory (Option 2) # ────────────────────────────────────────────── @spaces.GPU(duration=900) def run_custom_traj( image, trajectory_file, captions_json: str, num_frames: int, pose_scale: float, use_dmd: bool, run_reconstruction: bool, ): _ensure_models() with tempfile.TemporaryDirectory() as tmp: from PIL import Image img_path = Path(tmp) / "first_frame.png" Image.fromarray(image).save(img_path) traj_path = Path(tmp) / "trajectory.npz" shutil.copy(trajectory_file.name, traj_path) captions_path = Path(tmp) / "captions.json" try: json.loads(captions_json) # validate captions_path.write_text(captions_json) except json.JSONDecodeError: captions_path.write_text(json.dumps({"0": captions_json})) output_dir = Path(tmp) / "outputs" / "custom" output_dir.mkdir(parents=True, exist_ok=True) cmd = [ "python", "-m", "lyra_2._src.inference.lyra2_custom_traj_inference", "--input_image_path", str(img_path), "--trajectory_path", str(traj_path), "--experiment", "lyra2", "--checkpoint_dir", "checkpoints/model", "--captions_path", str(captions_path), "--num_frames", str(num_frames), "--output_path", str(output_dir), "--pose_scale", str(pose_scale), ] if use_dmd: cmd.append("--use_dmd") ok, log = _run(cmd, "Custom trajectory video generation") video_candidates = list(output_dir.rglob("*.mp4")) video_path = video_candidates[0] if video_candidates else None gs_video = None if run_reconstruction and video_path: ok2, log2 = _run( ["python", "-m", "lyra_2._src.inference.vipe_da3_gs_recon", "--input_video_path", str(video_path)], "GS reconstruction", ) log += "\n" + log2 ply_candidates = list(output_dir.rglob("gs_trajectory.mp4")) if ply_candidates: gs_video = str(ply_candidates[0]) return ( str(video_path) if video_path else None, gs_video, log[-4000:], ) # ────────────────────────────────────────────── # UI # ────────────────────────────────────────────── CSS = """ /* ── Global reset & fonts ── */ @import url('https://fonts.googleapis.com/css2?family=Syne:wght@400;600;700;800&family=DM+Mono:wght@300;400;500&display=swap'); :root { --bg: #0a0c10; --surface: #111318; --border: #1e2230; --accent: #5affb0; --accent2: #a78bfa; --text: #e8eaf0; --muted: #5a5f72; --radius: 12px; --font-head: 'Syne', sans-serif; --font-mono: 'DM Mono', monospace; } body, .gradio-container { background: var(--bg) !important; color: var(--text) !important; font-family: var(--font-head) !important; } /* Header banner */ #header { background: linear-gradient(135deg, #0d1117 0%, #161b27 60%, #0f1520 100%); border: 1px solid var(--border); border-radius: var(--radius); padding: 32px 40px 28px; margin-bottom: 24px; position: relative; overflow: hidden; } #header::before { content: ''; position: absolute; inset: 0; background: radial-gradient(ellipse 70% 60% at 80% 50%, rgba(94,255,176,0.06) 0%, transparent 70%), radial-gradient(ellipse 50% 80% at 20% 80%, rgba(167,139,250,0.06) 0%, transparent 70%); pointer-events: none; } #header h1 { font-size: 2.4rem; font-weight: 800; letter-spacing: -0.02em; margin: 0 0 8px; background: linear-gradient(90deg, var(--accent) 0%, var(--accent2) 100%); -webkit-background-clip: text; -webkit-text-fill-color: transparent; } #header p { color: var(--muted); font-family: var(--font-mono); font-size: 0.85rem; margin: 0; letter-spacing: 0.02em; } #header .badge { display: inline-block; margin-right: 8px; padding: 3px 10px; background: rgba(94,255,176,0.1); border: 1px solid rgba(94,255,176,0.25); border-radius: 20px; color: var(--accent); font-size: 0.75rem; font-family: var(--font-mono); } /* Tabs */ .tab-nav button { background: transparent !important; border: none !important; border-bottom: 2px solid transparent !important; color: var(--muted) !important; font-family: var(--font-head) !important; font-weight: 600 !important; font-size: 0.95rem !important; padding: 10px 20px !important; transition: all .2s !important; } .tab-nav button.selected, .tab-nav button:hover { color: var(--accent) !important; border-bottom-color: var(--accent) !important; background: transparent !important; } /* Panels / blocks */ .gr-panel, .gr-box, .gradio-group { background: var(--surface) !important; border: 1px solid var(--border) !important; border-radius: var(--radius) !important; } /* Inputs */ input, textarea, .gr-input, .gr-textbox textarea { background: #0d0f14 !important; border: 1px solid var(--border) !important; color: var(--text) !important; font-family: var(--font-mono) !important; border-radius: 8px !important; } input:focus, textarea:focus { border-color: var(--accent) !important; box-shadow: 0 0 0 2px rgba(94,255,176,0.12) !important; } /* Sliders */ input[type=range] { accent-color: var(--accent) !important; } /* Buttons */ button.primary, .gr-button-primary { background: linear-gradient(135deg, var(--accent) 0%, #38d9a9 100%) !important; color: #0a0c10 !important; font-family: var(--font-head) !important; font-weight: 700 !important; border: none !important; border-radius: 8px !important; padding: 12px 28px !important; font-size: 0.95rem !important; letter-spacing: 0.01em !important; transition: opacity .2s !important; } button.primary:hover { opacity: 0.85 !important; } button.secondary, .gr-button-secondary { background: transparent !important; border: 1px solid var(--border) !important; color: var(--muted) !important; font-family: var(--font-head) !important; border-radius: 8px !important; } /* Labels */ label, .gr-form > label, .block > label span { color: var(--muted) !important; font-family: var(--font-mono) !important; font-size: 0.8rem !important; letter-spacing: 0.04em !important; text-transform: uppercase !important; } /* Log box */ #log-box textarea { font-size: 0.78rem !important; color: #7af0b0 !important; background: #060709 !important; } /* Accordion */ .gr-accordion { border-color: var(--border) !important; } /* Info note */ .info-note { background: rgba(167,139,250,0.07); border: 1px solid rgba(167,139,250,0.2); border-radius: 8px; padding: 12px 16px; font-family: var(--font-mono); font-size: 0.8rem; color: #c4b5fd; line-height: 1.6; } """ def build_app(): with gr.Blocks() as demo: # ── Header ────────────────────────────────── gr.HTML(""" """) # ── Tabs ───────────────────────────────────── with gr.Tabs(): # ════════════════════════════════════════ # TAB 1 — Zoom-in / Zoom-out # ════════════════════════════════════════ with gr.Tab("🔭 Zoom Trajectory"): gr.HTML('
Generate a zoom-in → zoom-out exploration video from a single image, then optionally lift it to a 3D Gaussian Splatting scene.
') with gr.Row(): with gr.Column(scale=1): z_image = gr.Image(label="Input Image", type="numpy", height=280) z_caption = gr.Textbox( label="Scene Caption", placeholder="A sunlit forest clearing with tall pine trees…", lines=2, ) with gr.Accordion("Advanced Options", open=False): with gr.Row(): z_in_str = gr.Slider(0.1, 3.0, value=0.5, step=0.1, label="Zoom-in Strength") z_out_str = gr.Slider(0.1, 3.0, value=1.5, step=0.1, label="Zoom-out Strength") with gr.Row(): z_frames_in = gr.Slider(81, 401, value=81, step=80, label="Frames Zoom-in (1+80k)") z_frames_out = gr.Slider(81, 401, value=241, step=80, label="Frames Zoom-out (1+80k)") with gr.Row(): z_dmd = gr.Checkbox(label="⚡ Fast Mode (DMD ×15 speedup, lower quality)", value=False) z_recon = gr.Checkbox(label="🧊 Run 3DGS Reconstruction after video", value=True) z_btn = gr.Button("Generate World", variant="primary") with gr.Column(scale=1): z_video = gr.Video(label="Generated Exploration Video", height=280) z_gs_vid = gr.Video(label="3DGS Flythrough (if reconstruction enabled)", height=280) z_log = gr.Textbox(label="Log", lines=6, interactive=False, elem_id="log-box") z_btn.click( fn=run_zoomgs, inputs=[z_image, z_caption, gr.State(0), z_in_str, z_out_str, z_frames_in, z_frames_out, z_dmd, z_recon], outputs=[z_video, z_gs_vid, z_log], ) # ════════════════════════════════════════ # TAB 2 — Custom Trajectory # ════════════════════════════════════════ with gr.Tab("🎮 Custom Trajectory"): gr.HTML('
Provide your own camera trajectory (.npz with w2c, intrinsics, image_height, image_width) and per-chunk captions (JSON keyed by frame index, e.g. {"0": "…", "81": "…"}).
') with gr.Row(): with gr.Column(scale=1): c_image = gr.Image(label="First Frame", type="numpy", height=240) c_traj = gr.File(label="Trajectory (.npz)", file_types=[".npz"]) c_captions = gr.Textbox( label='Per-chunk Captions (JSON or single string)', placeholder='{"0": "A grand hall interior", "81": "Corridor leading outside"}', lines=3, ) with gr.Accordion("Advanced Options", open=False): with gr.Row(): c_frames = gr.Slider(81, 961, value=481, step=80, label="Num Frames (1+80k)") c_pose_scale = gr.Slider(0.1, 5.0, value=1.0, step=0.1, label="Pose Scale") with gr.Row(): c_dmd = gr.Checkbox(label="⚡ Fast Mode (DMD)", value=False) c_recon = gr.Checkbox(label="🧊 Run 3DGS Reconstruction", value=True) c_btn = gr.Button("Generate World", variant="primary") with gr.Column(scale=1): c_video = gr.Video(label="Generated Video", height=260) c_gs_vid = gr.Video(label="3DGS Flythrough", height=260) c_log = gr.Textbox(label="Log", lines=6, interactive=False, elem_id="log-box") c_btn.click( fn=run_custom_traj, inputs=[c_image, c_traj, c_captions, c_frames, c_pose_scale, c_dmd, c_recon], outputs=[c_video, c_gs_vid, c_log], ) # ════════════════════════════════════════ # TAB 3 — Model Info # ════════════════════════════════════════ with gr.Tab("ℹ️ About"): gr.Markdown(""" ## Lyra 2.0 — Explorable Generative 3D Worlds **NVIDIA Research** · [Paper](https://arxiv.org/abs/2604.13036) · [Project Page](https://research.nvidia.com/labs/sil/projects/lyra2/) · [HuggingFace](https://huggingface.co/nvidia/Lyra-2.0) ### How it works Lyra 2.0 solves two fundamental failure modes of long-horizon 3D world generation: | Problem | Solution | |---|---| | **Spatial Forgetting** — previously seen regions fall out of context and are hallucinated on revisit | Per-frame 3D geometry used for information routing — retrieve past frames and establish dense correspondences | | **Temporal Drifting** — autoregressive errors accumulate and distort appearance/geometry | Self-augmented training histories expose the model to its own degraded outputs, teaching correction not propagation | The generated video is then lifted to a **3D Gaussian Splatting** scene via VIPE pose estimation + Depth Anything 3 depth. ### GPU Requirements - Recommended: **H100 80 GB** (or A100 80 GB) - ~9 min per 80 frames at full quality · ~35 s with `--use_dmd` (DMD fast mode) - GS reconstruction adds ~1 min on top ### Checkpoint Setup Checkpoints are expected at `./checkpoints/model/`. Download from HuggingFace: ```bash huggingface-cli download nvidia/Lyra-2.0 \\ --include "checkpoints/*" \\ --local-dir . ``` ### Citation ```bibtex @article{shen2026lyra2, title={Lyra 2.0: Explorable Generative 3D Worlds}, author={Shen, Tianchang and Bahmani, Sherwin and He, Kai and ...}, journal={arXiv preprint arXiv:2604.13036}, year={2026} } ``` *Model weights released under [NVIDIA Internal Scientific Research and Development Model License](https://www.nvidia.com/en-us/agreements/enterprise-software/nvidia-internal-scientific-research-and-development-model-license/).* """) return demo if __name__ == "__main__": demo = build_app() demo.launch(css=CSS)