feat(rsplat): interactive demo scenes + AI image-to-3D generation
Replace fake demo data with real HuggingFace CDN Gaussian splats (train, truck, garden) that open in an inline 3D viewer without server round-trips. Add "Generate from Image" tab that sends images to fal.ai Trellis for AI-powered 3D model generation. Add gallery header and progress UI. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
e47cd35a34
commit
d79d560771
|
|
@ -22,6 +22,7 @@ interface SplatItem {
|
|||
processing_status?: string;
|
||||
source_file_count?: number;
|
||||
created_at: string;
|
||||
demoUrl?: string;
|
||||
}
|
||||
|
||||
export class FolkSplatViewer extends HTMLElement {
|
||||
|
|
@ -32,7 +33,8 @@ export class FolkSplatViewer extends HTMLElement {
|
|||
private _splatTitle = "";
|
||||
private _splatDesc = "";
|
||||
private _viewer: any = null;
|
||||
private _uploadMode: "splat" | "media" = "splat";
|
||||
private _uploadMode: "splat" | "media" | "generate" = "splat";
|
||||
private _inlineViewer = false;
|
||||
private _offlineUnsub: (() => void) | null = null;
|
||||
|
||||
static get observedAttributes() {
|
||||
|
|
@ -100,12 +102,9 @@ export class FolkSplatViewer extends HTMLElement {
|
|||
|
||||
private loadDemoData() {
|
||||
this._splats = [
|
||||
{ id: "s1", slug: "matterhorn-scan", title: "Matterhorn Summit", description: "Photogrammetry capture of the Matterhorn peak from drone footage, 42 source images.", file_format: "splat", file_size_bytes: 18_874_368, view_count: 284, contributor_name: "Alpine Explorer Team", processing_status: "ready", created_at: "2026-02-10" },
|
||||
{ id: "s2", slug: "community-garden", title: "Community Garden Plot", description: "3D scan of the shared garden space — beds, paths, tool shed.", file_format: "ply", file_size_bytes: 24_117_248, view_count: 156, contributor_name: "Garden Collective", processing_status: "ready", created_at: "2026-02-15" },
|
||||
{ id: "s3", slug: "print-shop-interior", title: "Print Shop Interior", description: "Interior scan of Druckwerkstatt Berlin — letterpress, risograph, binding station.", file_format: "spz", file_size_bytes: 31_457_280, view_count: 93, contributor_name: "Druckwerkstatt", processing_status: "ready", created_at: "2026-02-18" },
|
||||
{ id: "s4", slug: "chamonix-trailhead", title: "Chamonix Trailhead", description: "360° capture of the Lac Blanc trailhead parking area and signage.", file_format: "splat", file_size_bytes: 12_582_912, view_count: 67, processing_status: "ready", created_at: "2026-02-20" },
|
||||
{ id: "s5", slug: "zermatt-bridge", title: "Zermatt Suspension Bridge", description: "Charles Kuonen bridge scan from 18 photos. Processing complete.", file_format: "ply", file_size_bytes: 0, view_count: 0, source_file_count: 18, processing_status: "processing", created_at: "2026-02-25" },
|
||||
{ id: "s6", slug: "mycorrhiza-sculpture", title: "Mycorrhiza Sculpture", description: "Uploaded for 3D reconstruction from video. Queued.", file_format: "splat", file_size_bytes: 0, view_count: 0, source_file_count: 1, processing_status: "pending", created_at: "2026-02-27" },
|
||||
{ id: "s1", slug: "train", title: "Train", description: "Classic Gaussian splatting demo scene — a model train on a track.", file_format: "splat", file_size_bytes: 6_291_456, view_count: 1842, contributor_name: "MipNeRF 360", processing_status: "ready", created_at: "2025-06-15", demoUrl: "https://huggingface.co/cakewalk/splat-data/resolve/main/train.splat" },
|
||||
{ id: "s2", slug: "truck", title: "Truck", description: "Photogrammetry capture of a pickup truck — 360° Gaussian splat reconstruction.", file_format: "splat", file_size_bytes: 6_291_456, view_count: 1536, contributor_name: "MipNeRF 360", processing_status: "ready", created_at: "2025-06-15", demoUrl: "https://huggingface.co/cakewalk/splat-data/resolve/main/truck.splat" },
|
||||
{ id: "s3", slug: "garden", title: "Garden", description: "Outdoor garden scene — dense foliage and complex lighting captured as Gaussian splats.", file_format: "splat", file_size_bytes: 6_291_456, view_count: 1203, contributor_name: "MipNeRF 360", processing_status: "ready", created_at: "2025-06-15", demoUrl: "https://huggingface.co/cakewalk/splat-data/resolve/main/garden.splat" },
|
||||
];
|
||||
}
|
||||
|
||||
|
|
@ -132,9 +131,13 @@ export class FolkSplatViewer extends HTMLElement {
|
|||
const cards = this._splats.map((s) => {
|
||||
const status = s.processing_status || "ready";
|
||||
const isReady = status === "ready";
|
||||
const tag = isReady ? "a" : "div";
|
||||
const href = isReady ? ` href="/${this._spaceSlug}/rsplat/view/${s.slug}"` : "";
|
||||
const isDemo = !!s.demoUrl;
|
||||
// Demo cards use a button instead of a link to avoid server round-trip
|
||||
const tag = isReady ? (isDemo ? "div" : "a") : "div";
|
||||
const href = isReady && !isDemo ? ` href="/${this._spaceSlug}/rsplat/view/${s.slug}"` : "";
|
||||
const demoAttr = isDemo ? ` data-demo-url="${esc(s.demoUrl!)}" data-demo-title="${esc(s.title)}" data-demo-desc="${esc(s.description || "")}" role="button" tabindex="0"` : "";
|
||||
const statusClass = !isReady ? ` splat-card--${status}` : "";
|
||||
const demoClass = isDemo ? " splat-card--demo" : "";
|
||||
|
||||
let overlay = "";
|
||||
if (status === "pending") {
|
||||
|
|
@ -150,7 +153,7 @@ export class FolkSplatViewer extends HTMLElement {
|
|||
: `<span>${s.view_count} views</span>`;
|
||||
|
||||
return `
|
||||
<${tag} class="splat-card${statusClass}" data-collab-id="splat:${s.id}"${href}>
|
||||
<${tag} class="splat-card${statusClass}${demoClass}" data-collab-id="splat:${s.id}"${href}${demoAttr}>
|
||||
<div class="splat-card__preview">
|
||||
${overlay}
|
||||
<span>${isReady ? "🔮" : "📸"}</span>
|
||||
|
|
@ -177,12 +180,17 @@ export class FolkSplatViewer extends HTMLElement {
|
|||
|
||||
this.innerHTML = `
|
||||
<div class="splat-gallery">
|
||||
<div class="splat-gallery__header">
|
||||
<h1>rSplat</h1>
|
||||
<p class="splat-gallery__subtitle">Explore and create 3D Gaussian splat scenes</p>
|
||||
</div>
|
||||
${empty}
|
||||
<div class="splat-grid">${cards}</div>
|
||||
<div class="splat-upload" id="splat-drop">
|
||||
<div class="splat-upload__toggle">
|
||||
<button class="splat-upload__toggle-btn active" data-mode="splat">Upload Splat</button>
|
||||
<button class="splat-upload__toggle-btn" data-mode="media">Upload Photos/Video</button>
|
||||
<button class="splat-upload__toggle-btn" data-mode="generate">Generate from Image</button>
|
||||
</div>
|
||||
|
||||
<!-- Splat upload mode -->
|
||||
|
|
@ -220,27 +228,50 @@ export class FolkSplatViewer extends HTMLElement {
|
|||
<div class="splat-upload__status" id="media-status"></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Generate from image mode -->
|
||||
<div class="splat-upload__mode" id="splat-mode-generate" style="display:none">
|
||||
<div class="splat-upload__icon">✨</div>
|
||||
<p class="splat-upload__text">
|
||||
Upload a single <strong>image</strong> to generate a 3D Gaussian splat using AI
|
||||
<br>or <strong id="generate-browse">browse</strong> to select an image
|
||||
</p>
|
||||
<input type="file" id="generate-file" accept=".jpg,.jpeg,.png,.webp" hidden>
|
||||
<div class="splat-generate__preview" id="generate-preview"></div>
|
||||
<div class="splat-generate__actions" id="generate-actions" style="display:none">
|
||||
<button class="splat-upload__btn" id="generate-submit">Generate 3D Splat</button>
|
||||
</div>
|
||||
<div class="splat-generate__progress" id="generate-progress" style="display:none">
|
||||
<div class="splat-generate__progress-bar"></div>
|
||||
<div class="splat-generate__progress-text">Generating 3D model...</div>
|
||||
</div>
|
||||
<div class="splat-upload__status" id="generate-status"></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
|
||||
this.setupUploadHandlers();
|
||||
this.setupMediaHandlers();
|
||||
this.setupGenerateHandlers();
|
||||
this.setupToggle();
|
||||
this.setupDemoCardHandlers();
|
||||
}
|
||||
|
||||
private setupToggle() {
|
||||
const buttons = this.querySelectorAll<HTMLButtonElement>(".splat-upload__toggle-btn");
|
||||
const splatMode = this.querySelector("#splat-mode-splat") as HTMLElement;
|
||||
const mediaMode = this.querySelector("#splat-mode-media") as HTMLElement;
|
||||
const generateMode = this.querySelector("#splat-mode-generate") as HTMLElement;
|
||||
|
||||
buttons.forEach((btn) => {
|
||||
btn.addEventListener("click", () => {
|
||||
const mode = btn.dataset.mode as "splat" | "media";
|
||||
const mode = btn.dataset.mode as "splat" | "media" | "generate";
|
||||
this._uploadMode = mode;
|
||||
buttons.forEach((b) => b.classList.toggle("active", b.dataset.mode === mode));
|
||||
splatMode.style.display = mode === "splat" ? "" : "none";
|
||||
mediaMode.style.display = mode === "media" ? "" : "none";
|
||||
if (generateMode) generateMode.style.display = mode === "generate" ? "" : "none";
|
||||
});
|
||||
});
|
||||
}
|
||||
|
|
@ -436,9 +467,118 @@ export class FolkSplatViewer extends HTMLElement {
|
|||
});
|
||||
}
|
||||
|
||||
private setupDemoCardHandlers() {
|
||||
this.querySelectorAll<HTMLElement>(".splat-card--demo").forEach((card) => {
|
||||
card.style.cursor = "pointer";
|
||||
card.addEventListener("click", (e) => {
|
||||
e.preventDefault();
|
||||
const url = card.dataset.demoUrl;
|
||||
const title = card.dataset.demoTitle || "";
|
||||
const desc = card.dataset.demoDesc || "";
|
||||
if (!url) return;
|
||||
this._mode = "viewer";
|
||||
this._splatUrl = url;
|
||||
this._splatTitle = title;
|
||||
this._splatDesc = desc;
|
||||
this._inlineViewer = true;
|
||||
this.renderViewer();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private setupGenerateHandlers() {
|
||||
const browse = this.querySelector("#generate-browse") as HTMLElement;
|
||||
const fileInput = this.querySelector("#generate-file") as HTMLInputElement;
|
||||
const preview = this.querySelector("#generate-preview") as HTMLElement;
|
||||
const actions = this.querySelector("#generate-actions") as HTMLElement;
|
||||
const submitBtn = this.querySelector("#generate-submit") as HTMLButtonElement;
|
||||
const progress = this.querySelector("#generate-progress") as HTMLElement;
|
||||
const status = this.querySelector("#generate-status") as HTMLElement;
|
||||
|
||||
if (!fileInput) return;
|
||||
|
||||
let selectedFile: File | null = null;
|
||||
|
||||
browse?.addEventListener("click", () => fileInput.click());
|
||||
|
||||
fileInput.addEventListener("change", () => {
|
||||
if (fileInput.files?.[0]) {
|
||||
selectedFile = fileInput.files[0];
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => {
|
||||
preview.innerHTML = `<img src="${reader.result}" alt="Preview">`;
|
||||
preview.style.display = "block";
|
||||
actions.style.display = "flex";
|
||||
status.textContent = "";
|
||||
};
|
||||
reader.readAsDataURL(selectedFile);
|
||||
}
|
||||
});
|
||||
|
||||
submitBtn?.addEventListener("click", async () => {
|
||||
if (!selectedFile) return;
|
||||
|
||||
submitBtn.disabled = true;
|
||||
actions.style.display = "none";
|
||||
progress.style.display = "block";
|
||||
status.textContent = "";
|
||||
|
||||
try {
|
||||
const reader = new FileReader();
|
||||
const dataUrl = await new Promise<string>((resolve) => {
|
||||
reader.onload = () => resolve(reader.result as string);
|
||||
reader.readAsDataURL(selectedFile!);
|
||||
});
|
||||
|
||||
const res = await fetch("/api/3d-gen", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ image_url: dataUrl }),
|
||||
});
|
||||
|
||||
if (res.status === 503) {
|
||||
status.textContent = "AI generation not available — FAL_KEY not configured";
|
||||
progress.style.display = "none";
|
||||
actions.style.display = "flex";
|
||||
submitBtn.disabled = false;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!res.ok) {
|
||||
const err = await res.json().catch(() => ({ error: "Generation failed" }));
|
||||
status.textContent = (err as any).error || "Generation failed";
|
||||
progress.style.display = "none";
|
||||
actions.style.display = "flex";
|
||||
submitBtn.disabled = false;
|
||||
return;
|
||||
}
|
||||
|
||||
const data = await res.json() as { url: string; format: string };
|
||||
progress.style.display = "none";
|
||||
|
||||
// Open inline viewer with generated splat
|
||||
this._mode = "viewer";
|
||||
this._splatUrl = data.url;
|
||||
this._splatTitle = selectedFile.name.replace(/\.[^.]+$/, "");
|
||||
this._splatDesc = "AI-generated 3D model";
|
||||
this._inlineViewer = true;
|
||||
this.renderViewer();
|
||||
} catch (e) {
|
||||
status.textContent = "Network error — could not reach server";
|
||||
progress.style.display = "none";
|
||||
actions.style.display = "flex";
|
||||
submitBtn.disabled = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// ── Viewer ──
|
||||
|
||||
private renderViewer() {
|
||||
const backEl = this._inlineViewer
|
||||
? `<button class="splat-viewer__back" id="splat-back-btn">← Back to Gallery</button>`
|
||||
: `<a class="splat-viewer__back" href="/${this._spaceSlug}/rsplat">← Gallery</a>`;
|
||||
|
||||
this.innerHTML = `
|
||||
<div class="splat-viewer">
|
||||
<div class="splat-loading" id="splat-loading">
|
||||
|
|
@ -446,7 +586,7 @@ export class FolkSplatViewer extends HTMLElement {
|
|||
<div class="splat-loading__text">Loading splat...</div>
|
||||
</div>
|
||||
<div class="splat-viewer__controls">
|
||||
<a class="splat-viewer__back" href="/${this._spaceSlug}/rsplat">← Gallery</a>
|
||||
${backEl}
|
||||
</div>
|
||||
${this._splatTitle ? `
|
||||
<div class="splat-viewer__info">
|
||||
|
|
@ -458,6 +598,22 @@ export class FolkSplatViewer extends HTMLElement {
|
|||
</div>
|
||||
`;
|
||||
|
||||
if (this._inlineViewer) {
|
||||
this.querySelector("#splat-back-btn")?.addEventListener("click", () => {
|
||||
if (this._viewer) {
|
||||
try { this._viewer.dispose(); } catch {}
|
||||
this._viewer = null;
|
||||
}
|
||||
this._mode = "gallery";
|
||||
this._inlineViewer = false;
|
||||
this._splatUrl = "";
|
||||
this._splatTitle = "";
|
||||
this._splatDesc = "";
|
||||
if (this._spaceSlug === "demo") this.loadDemoData();
|
||||
this.renderGallery();
|
||||
});
|
||||
}
|
||||
|
||||
this.initThreeViewer();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -313,6 +313,74 @@
|
|||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
/* ── Generate mode ── */
|
||||
|
||||
.splat-generate__preview {
|
||||
display: none;
|
||||
margin: 1rem auto;
|
||||
max-width: 240px;
|
||||
}
|
||||
|
||||
.splat-generate__preview img {
|
||||
width: 100%;
|
||||
border-radius: 8px;
|
||||
border: 1px solid var(--splat-border);
|
||||
}
|
||||
|
||||
.splat-generate__actions {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.splat-generate__progress {
|
||||
max-width: 300px;
|
||||
margin: 1rem auto;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.splat-generate__progress-bar {
|
||||
height: 4px;
|
||||
border-radius: 2px;
|
||||
background: var(--splat-border);
|
||||
overflow: hidden;
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.splat-generate__progress-bar::after {
|
||||
content: "";
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
background: var(--splat-accent);
|
||||
width: 40%;
|
||||
border-radius: 2px;
|
||||
animation: splat-progress-slide 1.2s ease-in-out infinite;
|
||||
}
|
||||
|
||||
@keyframes splat-progress-slide {
|
||||
0% { transform: translateX(-100%); }
|
||||
100% { transform: translateX(350%); }
|
||||
}
|
||||
|
||||
.splat-generate__progress-text {
|
||||
margin-top: 0.75rem;
|
||||
color: var(--splat-text-muted);
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
|
||||
/* ── Demo card ── */
|
||||
|
||||
.splat-card--demo {
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
/* ── Back button (inline viewer) ── */
|
||||
|
||||
button.splat-viewer__back {
|
||||
cursor: pointer;
|
||||
font-family: inherit;
|
||||
}
|
||||
|
||||
/* ── Viewer ── */
|
||||
|
||||
.splat-viewer {
|
||||
|
|
|
|||
|
|
@ -1069,6 +1069,51 @@ app.post("/api/video-gen/i2v", async (c) => {
|
|||
return c.json({ url: videoUrl, video_url: videoUrl });
|
||||
});
|
||||
|
||||
// Image-to-3D via fal.ai Trellis
|
||||
app.post("/api/3d-gen", async (c) => {
|
||||
if (!FAL_KEY) return c.json({ error: "FAL_KEY not configured" }, 503);
|
||||
|
||||
const { image_url } = await c.req.json();
|
||||
if (!image_url) return c.json({ error: "image_url required" }, 400);
|
||||
|
||||
try {
|
||||
const res = await fetch("https://fal.run/fal-ai/trellis", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Key ${FAL_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ image_url }),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const err = await res.text();
|
||||
console.error("[3d-gen] fal.ai error:", err);
|
||||
return c.json({ error: "3D generation failed" }, 502);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
// Trellis returns glb_url and/or model_mesh — download the GLB
|
||||
const modelUrl = data.glb_url || data.model_mesh?.url || data.output?.url;
|
||||
if (!modelUrl) return c.json({ error: "No 3D model returned" }, 502);
|
||||
|
||||
// Download the model file
|
||||
const modelRes = await fetch(modelUrl);
|
||||
if (!modelRes.ok) return c.json({ error: "Failed to download model" }, 502);
|
||||
|
||||
const modelBuf = await modelRes.arrayBuffer();
|
||||
const ext = modelUrl.includes(".ply") ? "ply" : "glb";
|
||||
const filename = `splat-${Date.now()}-${Math.random().toString(36).slice(2, 6)}.${ext}`;
|
||||
const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
|
||||
await Bun.write(resolve(dir, filename), modelBuf);
|
||||
|
||||
return c.json({ url: `/data/files/generated/${filename}`, format: ext });
|
||||
} catch (e: any) {
|
||||
console.error("[3d-gen] error:", e.message);
|
||||
return c.json({ error: "3D generation failed" }, 502);
|
||||
}
|
||||
});
|
||||
|
||||
// Blender 3D generation via LLM + RunPod
|
||||
const RUNPOD_API_KEY = process.env.RUNPOD_API_KEY || "";
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue