From 0f1090db44f46f1c44c6d14e337c702a4f1cadde Mon Sep 17 00:00:00 2001
From: Jeff Emmett <jeffemmett@gmail.com>
Date: Mon, 16 Mar 2026 14:03:39 -0700
Subject: [PATCH] feat(rsplat): switch AI generation from Trellis 2 to SAM 3D

SAM 3D outputs native Gaussian splat .ply files (rendered via existing
initSplatViewer) instead of GLB meshes, with full-scene support including
people and backgrounds. Faster generation (5-30s vs 45-75s), $0.02/gen.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 .../rsplat/components/folk-splat-viewer.ts    | 20 +++++++++----------
 modules/rsplat/mod.ts                         |  4 ++--
 server/index.ts                               | 13 +++++++-----
 3 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/modules/rsplat/components/folk-splat-viewer.ts b/modules/rsplat/components/folk-splat-viewer.ts
index c735168..af66f97 100644
--- a/modules/rsplat/components/folk-splat-viewer.ts
+++ b/modules/rsplat/components/folk-splat-viewer.ts
@@ -249,7 +249,7 @@ export class FolkSplatViewer extends HTMLElement {
           <div class="splat-upload__mode" id="splat-mode-generate" style="display:none">
             <div class="splat-upload__icon">✨</div>
             <p class="splat-upload__text">
-              Upload a single <strong>image</strong> to generate a 3D model using AI (Trellis 2)
+              Upload a single <strong>image</strong> to generate a 3D Gaussian splat using AI (SAM 3D)
               <br>or <strong id="generate-browse">browse</strong> to select an image
             </p>
             <input type="file" id="generate-file" accept=".jpg,.jpeg,.png,.webp" hidden>
@@ -489,15 +489,15 @@ export class FolkSplatViewer extends HTMLElement {
       let hiddenAt = 0;
       const phases = [
         { t: 0, msg: "Staging image..." },
-        { t: 3, msg: "Uploading to Trellis 2..." },
-        { t: 8, msg: "Reconstructing 3D geometry..." },
-        { t: 20, msg: "Generating mesh and textures..." },
-        { t: 45, msg: "Finalizing model..." },
-        { t: 75, msg: "Almost there..." },
+        { t: 2, msg: "Uploading to SAM 3D..." },
+        { t: 5, msg: "Segmenting scene..." },
+        { t: 10, msg: "Generating Gaussian splats..." },
+        { t: 18, msg: "Finalizing model..." },
+        { t: 30, msg: "Almost there..." },
       ];
 
-      // Realistic progress curve — typical Trellis 2 takes 45-75s
-      const EXPECTED_SECONDS = 60;
+      // Realistic progress curve — typical SAM 3D takes 5-30s
+      const EXPECTED_SECONDS = 20;
       const progressBar = progress.querySelector(".splat-generate__progress-bar") as HTMLElement;
 
       const estimatePercent = (elapsed: number): number => {
@@ -618,7 +618,7 @@ export class FolkSplatViewer extends HTMLElement {
         body: JSON.stringify({
           url: this._generatedUrl,
           title: this._generatedTitle || "AI Generated Model",
-          description: "AI-generated 3D model via Trellis 2",
+          description: "AI-generated 3D model via SAM 3D",
         }),
       });
 
@@ -786,7 +786,7 @@ export class FolkSplatViewer extends HTMLElement {
         body: JSON.stringify({
           url: this._generatedUrl,
           title: this._generatedTitle || "AI Generated Model",
-          description: "AI-generated 3D model via Trellis 2",
+          description: "AI-generated 3D model via SAM 3D",
         }),
       });
 
diff --git a/modules/rsplat/mod.ts b/modules/rsplat/mod.ts
index 9b7fb9e..6cf92da 100644
--- a/modules/rsplat/mod.ts
+++ b/modules/rsplat/mod.ts
@@ -717,7 +717,7 @@ routes.get("/", async (c) => {
 		`,
 		scripts: `
 			<script type="module">
-				import { FolkSplatViewer } from '/modules/rsplat/folk-splat-viewer.js?v=6';
+				import { FolkSplatViewer } from '/modules/rsplat/folk-splat-viewer.js?v=7';
 				const gallery = document.getElementById('gallery');
 				gallery.splats = ${splatsJSON};
 				gallery.spaceSlug = '${spaceSlug}';
@@ -777,7 +777,7 @@ function renderViewerPage(spaceSlug: string, dataSpace: string, idOrSlug: string
 		`,
 		scripts: `
 			<script type="module">
-				import { FolkSplatViewer } from '/modules/rsplat/folk-splat-viewer.js?v=6';
+				import { FolkSplatViewer } from '/modules/rsplat/folk-splat-viewer.js?v=7';
 			</script>
 		`,
 	});
diff --git a/server/index.ts b/server/index.ts
index 9730b47..496baee 100644
--- a/server/index.ts
+++ b/server/index.ts
@@ -1019,7 +1019,7 @@ app.post("/api/image-stage", async (c) => {
 	return c.json({ url: `${PUBLIC_ORIGIN}/data/files/generated/${filename}` });
 });
 
-// Image-to-3D via fal.ai Trellis
+// Image-to-3D via fal.ai SAM 3D
 app.post("/api/3d-gen", async (c) => {
 	if (!FAL_KEY) return c.json({ error: "FAL_KEY not configured" }, 503);
 
@@ -1029,13 +1029,13 @@ app.post("/api/3d-gen", async (c) => {
 	try {
 		const controller = new AbortController();
 		const timeout = setTimeout(() => controller.abort(), 120_000); // 120s server-side timeout
-		const res = await fetch("https://fal.run/fal-ai/trellis-2", {
+		const res = await fetch("https://fal.run/fal-ai/sam-3/3d-objects", {
 			method: "POST",
 			headers: {
 				Authorization: `Key ${FAL_KEY}`,
 				"Content-Type": "application/json",
 			},
-			body: JSON.stringify({ image_url, resolution: 1024 }),
+			body: JSON.stringify({ image_url, export_textured_glb: true }),
 			signal: controller.signal,
 		});
 		clearTimeout(timeout);
@@ -1056,7 +1056,10 @@ app.post("/api/3d-gen", async (c) => {
 		}
 
 		const data = await res.json();
-		const modelUrl = data.model_glb?.url || data.glb_url || data.model_mesh?.url;
+		// SAM 3D: prefer Gaussian splat (.ply), fallback to GLB mesh
+		const splatUrl = data.gaussian_splat?.url;
+		const glbUrl = data.model_glb?.url || data.glb_url || data.model_mesh?.url;
+		const modelUrl = splatUrl || glbUrl;
 		if (!modelUrl) return c.json({ error: "No 3D model returned" }, 502);
 
 		// Download the model file
@@ -1064,7 +1067,7 @@ app.post("/api/3d-gen", async (c) => {
 		if (!modelRes.ok) return c.json({ error: "Failed to download model" }, 502);
 
 		const modelBuf = await modelRes.arrayBuffer();
-		const ext = modelUrl.includes(".ply") ? "ply" : "glb";
+		const ext = splatUrl ? "ply" : "glb";
 		const filename = `splat-${Date.now()}-${Math.random().toString(36).slice(2, 6)}.${ext}`;
 		const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
 		await Bun.write(resolve(dir, filename), modelBuf);