From d79d5607713378a64935d57a4c44ce82db9bced3 Mon Sep 17 00:00:00 2001 From: Jeff Emmett Date: Wed, 11 Mar 2026 14:29:15 -0700 Subject: [PATCH] feat(rsplat): interactive demo scenes + AI image-to-3D generation Replace fake demo data with real HuggingFace CDN Gaussian splats (train, truck, garden) that open in an inline 3D viewer without server round-trips. Add "Generate from Image" tab that sends images to fal.ai Trellis for AI-powered 3D model generation. Add gallery header and progress UI. Co-Authored-By: Claude Opus 4.6 --- .../rsplat/components/folk-splat-viewer.ts | 180 ++++++++++++++++-- modules/rsplat/components/splat.css | 68 +++++++ server/index.ts | 45 +++++ 3 files changed, 281 insertions(+), 12 deletions(-) diff --git a/modules/rsplat/components/folk-splat-viewer.ts b/modules/rsplat/components/folk-splat-viewer.ts index 81929c7..e64221c 100644 --- a/modules/rsplat/components/folk-splat-viewer.ts +++ b/modules/rsplat/components/folk-splat-viewer.ts @@ -22,6 +22,7 @@ interface SplatItem { processing_status?: string; source_file_count?: number; created_at: string; + demoUrl?: string; } export class FolkSplatViewer extends HTMLElement { @@ -32,7 +33,8 @@ export class FolkSplatViewer extends HTMLElement { private _splatTitle = ""; private _splatDesc = ""; private _viewer: any = null; - private _uploadMode: "splat" | "media" = "splat"; + private _uploadMode: "splat" | "media" | "generate" = "splat"; + private _inlineViewer = false; private _offlineUnsub: (() => void) | null = null; static get observedAttributes() { @@ -100,12 +102,9 @@ export class FolkSplatViewer extends HTMLElement { private loadDemoData() { this._splats = [ - { id: "s1", slug: "matterhorn-scan", title: "Matterhorn Summit", description: "Photogrammetry capture of the Matterhorn peak from drone footage, 42 source images.", file_format: "splat", file_size_bytes: 18_874_368, view_count: 284, contributor_name: "Alpine Explorer Team", processing_status: "ready", created_at: "2026-02-10" }, - { id: "s2", slug: "community-garden", title: "Community Garden Plot", description: "3D scan of the shared garden space — beds, paths, tool shed.", file_format: "ply", file_size_bytes: 24_117_248, view_count: 156, contributor_name: "Garden Collective", processing_status: "ready", created_at: "2026-02-15" }, - { id: "s3", slug: "print-shop-interior", title: "Print Shop Interior", description: "Interior scan of Druckwerkstatt Berlin — letterpress, risograph, binding station.", file_format: "spz", file_size_bytes: 31_457_280, view_count: 93, contributor_name: "Druckwerkstatt", processing_status: "ready", created_at: "2026-02-18" }, - { id: "s4", slug: "chamonix-trailhead", title: "Chamonix Trailhead", description: "360° capture of the Lac Blanc trailhead parking area and signage.", file_format: "splat", file_size_bytes: 12_582_912, view_count: 67, processing_status: "ready", created_at: "2026-02-20" }, - { id: "s5", slug: "zermatt-bridge", title: "Zermatt Suspension Bridge", description: "Charles Kuonen bridge scan from 18 photos. Processing complete.", file_format: "ply", file_size_bytes: 0, view_count: 0, source_file_count: 18, processing_status: "processing", created_at: "2026-02-25" }, - { id: "s6", slug: "mycorrhiza-sculpture", title: "Mycorrhiza Sculpture", description: "Uploaded for 3D reconstruction from video. Queued.", file_format: "splat", file_size_bytes: 0, view_count: 0, source_file_count: 1, processing_status: "pending", created_at: "2026-02-27" }, + { id: "s1", slug: "train", title: "Train", description: "Classic Gaussian splatting demo scene — a model train on a track.", file_format: "splat", file_size_bytes: 6_291_456, view_count: 1842, contributor_name: "MipNeRF 360", processing_status: "ready", created_at: "2025-06-15", demoUrl: "https://huggingface.co/cakewalk/splat-data/resolve/main/train.splat" }, + { id: "s2", slug: "truck", title: "Truck", description: "Photogrammetry capture of a pickup truck — 360° Gaussian splat reconstruction.", file_format: "splat", file_size_bytes: 6_291_456, view_count: 1536, contributor_name: "MipNeRF 360", processing_status: "ready", created_at: "2025-06-15", demoUrl: "https://huggingface.co/cakewalk/splat-data/resolve/main/truck.splat" }, + { id: "s3", slug: "garden", title: "Garden", description: "Outdoor garden scene — dense foliage and complex lighting captured as Gaussian splats.", file_format: "splat", file_size_bytes: 6_291_456, view_count: 1203, contributor_name: "MipNeRF 360", processing_status: "ready", created_at: "2025-06-15", demoUrl: "https://huggingface.co/cakewalk/splat-data/resolve/main/garden.splat" }, ]; } @@ -132,9 +131,13 @@ export class FolkSplatViewer extends HTMLElement { const cards = this._splats.map((s) => { const status = s.processing_status || "ready"; const isReady = status === "ready"; - const tag = isReady ? "a" : "div"; - const href = isReady ? ` href="/${this._spaceSlug}/rsplat/view/${s.slug}"` : ""; + const isDemo = !!s.demoUrl; + // Demo cards use a button instead of a link to avoid server round-trip + const tag = isReady ? (isDemo ? "div" : "a") : "div"; + const href = isReady && !isDemo ? ` href="/${this._spaceSlug}/rsplat/view/${s.slug}"` : ""; + const demoAttr = isDemo ? ` data-demo-url="${esc(s.demoUrl!)}" data-demo-title="${esc(s.title)}" data-demo-desc="${esc(s.description || "")}" role="button" tabindex="0"` : ""; const statusClass = !isReady ? ` splat-card--${status}` : ""; + const demoClass = isDemo ? " splat-card--demo" : ""; let overlay = ""; if (status === "pending") { @@ -150,7 +153,7 @@ export class FolkSplatViewer extends HTMLElement { : `${s.view_count} views`; return ` - <${tag} class="splat-card${statusClass}" data-collab-id="splat:${s.id}"${href}> + <${tag} class="splat-card${statusClass}${demoClass}" data-collab-id="splat:${s.id}"${href}${demoAttr}>
${overlay} ${isReady ? "🔮" : "📸"} @@ -177,12 +180,17 @@ export class FolkSplatViewer extends HTMLElement { this.innerHTML = `
+
+

rSplat

+

Explore and create 3D Gaussian splat scenes

+
${empty}
${cards}
+
@@ -220,27 +228,50 @@ export class FolkSplatViewer extends HTMLElement {
+ + +
`; this.setupUploadHandlers(); this.setupMediaHandlers(); + this.setupGenerateHandlers(); this.setupToggle(); + this.setupDemoCardHandlers(); } private setupToggle() { const buttons = this.querySelectorAll(".splat-upload__toggle-btn"); const splatMode = this.querySelector("#splat-mode-splat") as HTMLElement; const mediaMode = this.querySelector("#splat-mode-media") as HTMLElement; + const generateMode = this.querySelector("#splat-mode-generate") as HTMLElement; buttons.forEach((btn) => { btn.addEventListener("click", () => { - const mode = btn.dataset.mode as "splat" | "media"; + const mode = btn.dataset.mode as "splat" | "media" | "generate"; this._uploadMode = mode; buttons.forEach((b) => b.classList.toggle("active", b.dataset.mode === mode)); splatMode.style.display = mode === "splat" ? "" : "none"; mediaMode.style.display = mode === "media" ? "" : "none"; + if (generateMode) generateMode.style.display = mode === "generate" ? "" : "none"; }); }); } @@ -436,9 +467,118 @@ export class FolkSplatViewer extends HTMLElement { }); } + private setupDemoCardHandlers() { + this.querySelectorAll(".splat-card--demo").forEach((card) => { + card.style.cursor = "pointer"; + card.addEventListener("click", (e) => { + e.preventDefault(); + const url = card.dataset.demoUrl; + const title = card.dataset.demoTitle || ""; + const desc = card.dataset.demoDesc || ""; + if (!url) return; + this._mode = "viewer"; + this._splatUrl = url; + this._splatTitle = title; + this._splatDesc = desc; + this._inlineViewer = true; + this.renderViewer(); + }); + }); + } + + private setupGenerateHandlers() { + const browse = this.querySelector("#generate-browse") as HTMLElement; + const fileInput = this.querySelector("#generate-file") as HTMLInputElement; + const preview = this.querySelector("#generate-preview") as HTMLElement; + const actions = this.querySelector("#generate-actions") as HTMLElement; + const submitBtn = this.querySelector("#generate-submit") as HTMLButtonElement; + const progress = this.querySelector("#generate-progress") as HTMLElement; + const status = this.querySelector("#generate-status") as HTMLElement; + + if (!fileInput) return; + + let selectedFile: File | null = null; + + browse?.addEventListener("click", () => fileInput.click()); + + fileInput.addEventListener("change", () => { + if (fileInput.files?.[0]) { + selectedFile = fileInput.files[0]; + const reader = new FileReader(); + reader.onload = () => { + preview.innerHTML = `Preview`; + preview.style.display = "block"; + actions.style.display = "flex"; + status.textContent = ""; + }; + reader.readAsDataURL(selectedFile); + } + }); + + submitBtn?.addEventListener("click", async () => { + if (!selectedFile) return; + + submitBtn.disabled = true; + actions.style.display = "none"; + progress.style.display = "block"; + status.textContent = ""; + + try { + const reader = new FileReader(); + const dataUrl = await new Promise((resolve) => { + reader.onload = () => resolve(reader.result as string); + reader.readAsDataURL(selectedFile!); + }); + + const res = await fetch("/api/3d-gen", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ image_url: dataUrl }), + }); + + if (res.status === 503) { + status.textContent = "AI generation not available — FAL_KEY not configured"; + progress.style.display = "none"; + actions.style.display = "flex"; + submitBtn.disabled = false; + return; + } + + if (!res.ok) { + const err = await res.json().catch(() => ({ error: "Generation failed" })); + status.textContent = (err as any).error || "Generation failed"; + progress.style.display = "none"; + actions.style.display = "flex"; + submitBtn.disabled = false; + return; + } + + const data = await res.json() as { url: string; format: string }; + progress.style.display = "none"; + + // Open inline viewer with generated splat + this._mode = "viewer"; + this._splatUrl = data.url; + this._splatTitle = selectedFile.name.replace(/\.[^.]+$/, ""); + this._splatDesc = "AI-generated 3D model"; + this._inlineViewer = true; + this.renderViewer(); + } catch (e) { + status.textContent = "Network error — could not reach server"; + progress.style.display = "none"; + actions.style.display = "flex"; + submitBtn.disabled = false; + } + }); + } + // ── Viewer ── private renderViewer() { + const backEl = this._inlineViewer + ? `` + : `← Gallery`; + this.innerHTML = `
@@ -446,7 +586,7 @@ export class FolkSplatViewer extends HTMLElement {
Loading splat...
- ← Gallery + ${backEl}
${this._splatTitle ? `
@@ -458,6 +598,22 @@ export class FolkSplatViewer extends HTMLElement {
`; + if (this._inlineViewer) { + this.querySelector("#splat-back-btn")?.addEventListener("click", () => { + if (this._viewer) { + try { this._viewer.dispose(); } catch {} + this._viewer = null; + } + this._mode = "gallery"; + this._inlineViewer = false; + this._splatUrl = ""; + this._splatTitle = ""; + this._splatDesc = ""; + if (this._spaceSlug === "demo") this.loadDemoData(); + this.renderGallery(); + }); + } + this.initThreeViewer(); } diff --git a/modules/rsplat/components/splat.css b/modules/rsplat/components/splat.css index 129d2ab..6c56c73 100644 --- a/modules/rsplat/components/splat.css +++ b/modules/rsplat/components/splat.css @@ -313,6 +313,74 @@ margin-bottom: 0.5rem; } +/* ── Generate mode ── */ + +.splat-generate__preview { + display: none; + margin: 1rem auto; + max-width: 240px; +} + +.splat-generate__preview img { + width: 100%; + border-radius: 8px; + border: 1px solid var(--splat-border); +} + +.splat-generate__actions { + display: flex; + justify-content: center; + margin-bottom: 0.5rem; +} + +.splat-generate__progress { + max-width: 300px; + margin: 1rem auto; + text-align: center; +} + +.splat-generate__progress-bar { + height: 4px; + border-radius: 2px; + background: var(--splat-border); + overflow: hidden; + position: relative; +} + +.splat-generate__progress-bar::after { + content: ""; + position: absolute; + inset: 0; + background: var(--splat-accent); + width: 40%; + border-radius: 2px; + animation: splat-progress-slide 1.2s ease-in-out infinite; +} + +@keyframes splat-progress-slide { + 0% { transform: translateX(-100%); } + 100% { transform: translateX(350%); } +} + +.splat-generate__progress-text { + margin-top: 0.75rem; + color: var(--splat-text-muted); + font-size: 0.8rem; +} + +/* ── Demo card ── */ + +.splat-card--demo { + cursor: pointer; +} + +/* ── Back button (inline viewer) ── */ + +button.splat-viewer__back { + cursor: pointer; + font-family: inherit; +} + /* ── Viewer ── */ .splat-viewer { diff --git a/server/index.ts b/server/index.ts index 5d9bbe8..8c92a7a 100644 --- a/server/index.ts +++ b/server/index.ts @@ -1069,6 +1069,51 @@ app.post("/api/video-gen/i2v", async (c) => { return c.json({ url: videoUrl, video_url: videoUrl }); }); +// Image-to-3D via fal.ai Trellis +app.post("/api/3d-gen", async (c) => { + if (!FAL_KEY) return c.json({ error: "FAL_KEY not configured" }, 503); + + const { image_url } = await c.req.json(); + if (!image_url) return c.json({ error: "image_url required" }, 400); + + try { + const res = await fetch("https://fal.run/fal-ai/trellis", { + method: "POST", + headers: { + Authorization: `Key ${FAL_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ image_url }), + }); + + if (!res.ok) { + const err = await res.text(); + console.error("[3d-gen] fal.ai error:", err); + return c.json({ error: "3D generation failed" }, 502); + } + + const data = await res.json(); + // Trellis returns glb_url and/or model_mesh — download the GLB + const modelUrl = data.glb_url || data.model_mesh?.url || data.output?.url; + if (!modelUrl) return c.json({ error: "No 3D model returned" }, 502); + + // Download the model file + const modelRes = await fetch(modelUrl); + if (!modelRes.ok) return c.json({ error: "Failed to download model" }, 502); + + const modelBuf = await modelRes.arrayBuffer(); + const ext = modelUrl.includes(".ply") ? "ply" : "glb"; + const filename = `splat-${Date.now()}-${Math.random().toString(36).slice(2, 6)}.${ext}`; + const dir = resolve(process.env.FILES_DIR || "./data/files", "generated"); + await Bun.write(resolve(dir, filename), modelBuf); + + return c.json({ url: `/data/files/generated/${filename}`, format: ext }); + } catch (e: any) { + console.error("[3d-gen] error:", e.message); + return c.json({ error: "3D generation failed" }, 502); + } +}); + // Blender 3D generation via LLM + RunPod const RUNPOD_API_KEY = process.env.RUNPOD_API_KEY || "";