import os import sys import json import tempfile import subprocess import shutil from pathlib import Path # Root of the repo — wherever app.py lives REPO_ROOT = Path(__file__).parent.resolve() import gradio as gr import spaces import numpy as np import torch # ────────────────────────────────────────────── # Lazy model loading (done inside GPU decorator) # ────────────────────────────────────────────── _model_loaded = False def _ensure_models(): global _model_loaded if _model_loaded: return # Models are expected to be pre-downloaded to ./checkpoints/ # on the Space via the HF repo or a setup script. _model_loaded = True # ────────────────────────────────────────────── # Core inference helpers # ────────────────────────────────────────────── def _write_caption_txt(image_path: str, caption: str) -> str: """Write a .txt caption file beside the image and return the directory.""" img_path = Path(image_path) txt_path = img_path.with_suffix(".txt") txt_path.write_text(caption) return str(img_path.parent) def _run(cmd: list[str], desc: str = "") -> tuple[bool, str]: """Run a Python -m command using the current interpreter with PYTHONPATH set.""" # Replace bare "python" / "python3" with the real interpreter path if cmd[0] in ("python", "python3"): cmd = [sys.executable] + cmd[1:] run_env = { **os.environ, "PYTHONPATH": str(REPO_ROOT), # makes lyra_2._src.* importable "PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True", } print(f"[Lyra] {desc}: {' '.join(cmd)}") result = subprocess.run( cmd, capture_output=True, text=True, env=run_env, cwd=str(REPO_ROOT), # run from repo root ) log = result.stdout + "\n" + result.stderr return result.returncode == 0, log # ────────────────────────────────────────────── # Zoom-in / Zoom-out trajectory (Option 1) # ────────────────────────────────────────────── @spaces.GPU(duration=190) def run_zoomgs( image, caption: str, sample_id: int, zoom_in_strength: float, zoom_out_strength: float, num_frames_in: int, num_frames_out: int, use_dmd: bool, run_reconstruction: bool, ): _ensure_models() with tempfile.TemporaryDirectory() as tmp: # Save uploaded image + caption img_path = Path(tmp) / "input.png" caption_path = Path(tmp) / "input.txt" from PIL import Image Image.fromarray(image).save(img_path) caption_path.write_text(caption.strip() or "A scenic outdoor environment.") output_dir = Path(tmp) / "outputs" / "zoomgs" output_dir.mkdir(parents=True, exist_ok=True) cmd = [ "python", "-m", "lyra_2._src.inference.lyra2_zoomgs_inference", "--input_image_path", str(tmp), "--sample_id", "0", # we always name it input.png → id 0 equivalent "--experiment", "lyra2", "--checkpoint_dir", "checkpoints/model", "--prompt_dir", str(tmp), "--output_path", str(output_dir), "--num_frames_zoom_in", str(num_frames_in), "--num_frames_zoom_out", str(num_frames_out), "--zoom_in_strength", str(zoom_in_strength), "--zoom_out_strength", str(zoom_out_strength), ] if use_dmd: cmd.append("--use_dmd") ok, log = _run(cmd, "ZoomGS video generation") # Locate output video video_path = output_dir / "0" / "videos" / "0.mp4" if not video_path.exists(): # Fallback: search recursively candidates = list(output_dir.rglob("*.mp4")) video_path = candidates[0] if candidates else None gs_video = None if run_reconstruction and video_path and video_path.exists(): ok2, log2 = _run( ["python", "-m", "lyra_2._src.inference.vipe_da3_gs_recon", "--input_video_path", str(video_path)], "GS reconstruction", ) log += "\n" + log2 ply_candidates = list(output_dir.rglob("gs_trajectory.mp4")) if ply_candidates: gs_video = str(ply_candidates[0]) return ( str(video_path) if video_path and video_path.exists() else None, gs_video, log[-4000:], ) # ────────────────────────────────────────────── # Custom trajectory (Option 2) # ────────────────────────────────────────────── @spaces.GPU(duration=900) def run_custom_traj( image, trajectory_file, captions_json: str, num_frames: int, pose_scale: float, use_dmd: bool, run_reconstruction: bool, ): _ensure_models() with tempfile.TemporaryDirectory() as tmp: from PIL import Image img_path = Path(tmp) / "first_frame.png" Image.fromarray(image).save(img_path) traj_path = Path(tmp) / "trajectory.npz" shutil.copy(trajectory_file.name, traj_path) captions_path = Path(tmp) / "captions.json" try: json.loads(captions_json) # validate captions_path.write_text(captions_json) except json.JSONDecodeError: captions_path.write_text(json.dumps({"0": captions_json})) output_dir = Path(tmp) / "outputs" / "custom" output_dir.mkdir(parents=True, exist_ok=True) cmd = [ "python", "-m", "lyra_2._src.inference.lyra2_custom_traj_inference", "--input_image_path", str(img_path), "--trajectory_path", str(traj_path), "--experiment", "lyra2", "--checkpoint_dir", "checkpoints/model", "--captions_path", str(captions_path), "--num_frames", str(num_frames), "--output_path", str(output_dir), "--pose_scale", str(pose_scale), ] if use_dmd: cmd.append("--use_dmd") ok, log = _run(cmd, "Custom trajectory video generation") video_candidates = list(output_dir.rglob("*.mp4")) video_path = video_candidates[0] if video_candidates else None gs_video = None if run_reconstruction and video_path: ok2, log2 = _run( ["python", "-m", "lyra_2._src.inference.vipe_da3_gs_recon", "--input_video_path", str(video_path)], "GS reconstruction", ) log += "\n" + log2 ply_candidates = list(output_dir.rglob("gs_trajectory.mp4")) if ply_candidates: gs_video = str(ply_candidates[0]) return ( str(video_path) if video_path else None, gs_video, log[-4000:], ) # ────────────────────────────────────────────── # UI # ────────────────────────────────────────────── CSS = """ /* ── Global reset & fonts ── */ @import url('https://fonts.googleapis.com/css2?family=Syne:wght@400;600;700;800&family=DM+Mono:wght@300;400;500&display=swap'); :root { --bg: #0a0c10; --surface: #111318; --border: #1e2230; --accent: #5affb0; --accent2: #a78bfa; --text: #e8eaf0; --muted: #5a5f72; --radius: 12px; --font-head: 'Syne', sans-serif; --font-mono: 'DM Mono', monospace; } body, .gradio-container { background: var(--bg) !important; color: var(--text) !important; font-family: var(--font-head) !important; } /* Header banner */ #header { background: linear-gradient(135deg, #0d1117 0%, #161b27 60%, #0f1520 100%); border: 1px solid var(--border); border-radius: var(--radius); padding: 32px 40px 28px; margin-bottom: 24px; position: relative; overflow: hidden; } #header::before { content: ''; position: absolute; inset: 0; background: radial-gradient(ellipse 70% 60% at 80% 50%, rgba(94,255,176,0.06) 0%, transparent 70%), radial-gradient(ellipse 50% 80% at 20% 80%, rgba(167,139,250,0.06) 0%, transparent 70%); pointer-events: none; } #header h1 { font-size: 2.4rem; font-weight: 800; letter-spacing: -0.02em; margin: 0 0 8px; background: linear-gradient(90deg, var(--accent) 0%, var(--accent2) 100%); -webkit-background-clip: text; -webkit-text-fill-color: transparent; } #header p { color: var(--muted); font-family: var(--font-mono); font-size: 0.85rem; margin: 0; letter-spacing: 0.02em; } #header .badge { display: inline-block; margin-right: 8px; padding: 3px 10px; background: rgba(94,255,176,0.1); border: 1px solid rgba(94,255,176,0.25); border-radius: 20px; color: var(--accent); font-size: 0.75rem; font-family: var(--font-mono); } /* Tabs */ .tab-nav button { background: transparent !important; border: none !important; border-bottom: 2px solid transparent !important; color: var(--muted) !important; font-family: var(--font-head) !important; font-weight: 600 !important; font-size: 0.95rem !important; padding: 10px 20px !important; transition: all .2s !important; } .tab-nav button.selected, .tab-nav button:hover { color: var(--accent) !important; border-bottom-color: var(--accent) !important; background: transparent !important; } /* Panels / blocks */ .gr-panel, .gr-box, .gradio-group { background: var(--surface) !important; border: 1px solid var(--border) !important; border-radius: var(--radius) !important; } /* Inputs */ input, textarea, .gr-input, .gr-textbox textarea { background: #0d0f14 !important; border: 1px solid var(--border) !important; color: var(--text) !important; font-family: var(--font-mono) !important; border-radius: 8px !important; } input:focus, textarea:focus { border-color: var(--accent) !important; box-shadow: 0 0 0 2px rgba(94,255,176,0.12) !important; } /* Sliders */ input[type=range] { accent-color: var(--accent) !important; } /* Buttons */ button.primary, .gr-button-primary { background: linear-gradient(135deg, var(--accent) 0%, #38d9a9 100%) !important; color: #0a0c10 !important; font-family: var(--font-head) !important; font-weight: 700 !important; border: none !important; border-radius: 8px !important; padding: 12px 28px !important; font-size: 0.95rem !important; letter-spacing: 0.01em !important; transition: opacity .2s !important; } button.primary:hover { opacity: 0.85 !important; } button.secondary, .gr-button-secondary { background: transparent !important; border: 1px solid var(--border) !important; color: var(--muted) !important; font-family: var(--font-head) !important; border-radius: 8px !important; } /* Labels */ label, .gr-form > label, .block > label span { color: var(--muted) !important; font-family: var(--font-mono) !important; font-size: 0.8rem !important; letter-spacing: 0.04em !important; text-transform: uppercase !important; } /* Log box */ #log-box textarea { font-size: 0.78rem !important; color: #7af0b0 !important; background: #060709 !important; } /* Accordion */ .gr-accordion { border-color: var(--border) !important; } /* Info note */ .info-note { background: rgba(167,139,250,0.07); border: 1px solid rgba(167,139,250,0.2); border-radius: 8px; padding: 12px 16px; font-family: var(--font-mono); font-size: 0.8rem; color: #c4b5fd; line-height: 1.6; } """ def build_app(): with gr.Blocks() as demo: # ── Header ────────────────────────────────── gr.HTML("""
NVIDIA Research 3D Gaussian Splatting arXiv 2604.13036
Generate persistent, explorable 3D worlds from a single image. Walk through scenes, revisit areas — no spatial forgetting, no temporal drift.
w2c, intrinsics, image_height, image_width) and per-chunk captions (JSON keyed by frame index, e.g. {"0": "…", "81": "…"}).