Merge branch 'dev'

docker-compose.yml

@@ -63,6 +63,8 @@ services:
       - TRANSAK_ENV=PRODUCTION
       - SCRIBUS_BRIDGE_SECRET=${SCRIBUS_BRIDGE_SECRET}
       - SCRIBUS_NOVNC_URL=https://design.rspace.online
+      - IPFS_API_URL=http://kubo:5001
+      - IPFS_GATEWAY_URL=https://ipfs.jeffemmett.com
     depends_on:
       rspace-db:
         condition: service_healthy

server/index.ts

@@ -93,6 +93,8 @@ import { renderMainLanding, renderSpaceDashboard } from "./landing";
 import { syncServer } from "./sync-instance";
 import { loadAllDocs } from "./local-first/doc-persistence";
 import { backupRouter } from "./local-first/backup-routes";
+import { ipfsRouter } from "./ipfs-routes";
+import { isIPFSEnabled, pinToIPFS } from "./ipfs";
 import { oauthRouter, setOAuthStatusSyncServer } from "./oauth/index";
 import { setNotionOAuthSyncServer } from "./oauth/notion";
 import { setGoogleOAuthSyncServer } from "./oauth/google";
@@ -245,6 +247,53 @@ app.get("/data/files/generated/:subdir/:filename", serveGeneratedFile);
 app.get("/api/files/generated/:filename", serveGeneratedFile);
 app.get("/api/files/generated/:subdir/:filename", serveGeneratedFile);
 
+// ── IPFS background pinning for generated files ──
+
+/** In-memory cache of filePath → CID. Populated from .cid sidecar files on startup. */
+const generatedCidCache = new Map<string, string>();
+
+/** Load existing .cid sidecar files into cache on startup. */
+async function loadGeneratedCids() {
+	const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
+	try {
+		const { readdir, readFile } = await import("node:fs/promises");
+		const files = await readdir(dir);
+		for (const f of files) {
+			if (!f.endsWith(".cid")) continue;
+			try {
+				const cid = (await readFile(resolve(dir, f), "utf-8")).trim();
+				const originalFile = f.replace(/\.cid$/, "");
+				generatedCidCache.set(originalFile, cid);
+			} catch {}
+		}
+		if (generatedCidCache.size > 0) {
+			console.log(`[ipfs] Loaded ${generatedCidCache.size} CID sidecar files`);
+		}
+	} catch {}
+}
+loadGeneratedCids();
+
+/**
+ * Pin a generated file to IPFS in the background.
+ * Writes a .cid sidecar file alongside the original.
+ * Safe to call fire-and-forget — failures are logged and swallowed.
+ */
+async function pinGeneratedFile(filePath: string, filename: string) {
+	if (!isIPFSEnabled()) return;
+	if (generatedCidCache.has(filename)) return;
+	try {
+		const file = Bun.file(filePath);
+		const buf = new Uint8Array(await file.arrayBuffer());
+		const cid = await pinToIPFS(buf, filename);
+		generatedCidCache.set(filename, cid);
+		const { writeFile } = await import("node:fs/promises");
+		await writeFile(`${filePath}.cid`, cid);
+		console.log(`[ipfs] Pinned ${filename} → ${cid}`);
+	} catch (err: any) {
+		console.warn(`[ipfs] Pin failed for ${filename} (non-fatal):`, err.message);
+	}
+}
+
 // ── Link preview / unfurl API ──
 const linkPreviewCache = new Map<string, { title: string; description: string; image: string | null; domain: string; fetchedAt: number }>();
 
@@ -453,6 +502,9 @@ app.route("/api/spaces", spaces);
 // ── Backup API (encrypted blob storage) ──
 app.route("/api/backup", backupRouter);
 
+// ── IPFS API (pinning + gateway proxy) ──
+app.route("/api/ipfs", ipfsRouter);
+
 // ── OAuth API (Notion, Google integrations) ──
 app.route("/api/oauth", oauthRouter);
 
@@ -1172,6 +1224,7 @@ async function process3DGenJob(job: Gen3DJob) {
 		const filename = `splat-${Date.now()}-${Math.random().toString(36).slice(2, 6)}.glb`;
 		const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
 		await Bun.write(resolve(dir, filename), modelBuf);
+		pinGeneratedFile(resolve(dir, filename), filename);
 
 		job.status = "complete";
 		job.resultUrl = `/data/files/generated/${filename}`;
@@ -1211,6 +1264,7 @@ async function saveDataUrlToDisk(dataUrl: string, prefix: string): Promise<strin
 	const filename = `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 6)}.${ext}`;
 	const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
 	await Bun.write(resolve(dir, filename), Buffer.from(b64, "base64"));
+	pinGeneratedFile(resolve(dir, filename), filename);
 	return `/data/files/generated/${filename}`;
 }
 
@@ -1364,6 +1418,7 @@ app.post("/api/image-gen/img2img", async (c) => {
 			const filename = `img2img-${Date.now()}.png`;
 			const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
 			await Bun.write(resolve(dir, filename), Buffer.from(imgBuf));
+			pinGeneratedFile(resolve(dir, filename), filename);
 			const localUrl = `/data/files/generated/${filename}`;
 			return c.json({ url: localUrl, image_url: localUrl });
 		} catch (e: any) {
@@ -1423,6 +1478,7 @@ app.post("/api/image-gen/img2img", async (c) => {
 					const filename = `img2img-gemini-${Date.now()}.${ext}`;
 					const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
 					await Bun.write(resolve(dir, filename), Buffer.from(b64, "base64"));
+					pinGeneratedFile(resolve(dir, filename), filename);
 					const url = `/data/files/generated/${filename}`;
 					return c.json({ url, image_url: url });
 				}
@@ -2178,6 +2234,7 @@ app.post("/api/gemini/image", async (c) => {
 						const filename = `gemini-${Date.now()}.${ext}`;
 						const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
 						await Bun.write(resolve(dir, filename), Buffer.from(b64, "base64"));
+						pinGeneratedFile(resolve(dir, filename), filename);
 						const url = `/data/files/generated/${filename}`;
 						return c.json({ url, image_url: url });
 					}
@@ -2197,6 +2254,7 @@ app.post("/api/gemini/image", async (c) => {
 					const filename = `imagen-${Date.now()}.png`;
 					const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
 					await Bun.write(resolve(dir, filename), Buffer.from(img.image.imageBytes, "base64"));
+					pinGeneratedFile(resolve(dir, filename), filename);
 					const url = `/data/files/generated/${filename}`;
 					return c.json({ url, image_url: url });
 				}
@@ -2328,6 +2386,7 @@ app.post("/api/zine/page", async (c) => {
 					const filename = `zine-p${outline.pageNumber}-${section.id}-${Date.now()}.${ext}`;
 					const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
 					await Bun.write(resolve(dir, filename), Buffer.from(b64, "base64"));
+					pinGeneratedFile(resolve(dir, filename), filename);
 					generatedImages[section.id] = `/data/files/generated/${filename}`;
 					break;
 				}
@@ -2374,6 +2433,7 @@ app.post("/api/zine/regenerate-section", async (c) => {
 					const filename = `zine-regen-${section.id}-${Date.now()}.${ext}`;
 					const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
 					await Bun.write(resolve(dir, filename), Buffer.from(b64, "base64"));
+					pinGeneratedFile(resolve(dir, filename), filename);
 					return c.json({ sectionId: section.id, type: "image", url: `/data/files/generated/${filename}` });
 				}
 			}

server/ipfs-routes.ts

@@ -0,0 +1,123 @@
+/**
+ * IPFS API Routes — Hono router mounted at /api/ipfs.
+ *
+ * GET  /api/ipfs/status     — IPFS node health + stats (authenticated)
+ * GET  /api/ipfs/:cid       — Proxy gateway fetch (avoids CORS for clients)
+ * POST /api/ipfs/pin        — Pin arbitrary blob (authenticated, 50MB limit)
+ * DELETE /api/ipfs/:cid     — Unpin a CID (authenticated)
+ */
+
+import { Hono } from "hono";
+import type { Context, Next } from "hono";
+import { verifyToken, extractToken } from "./auth";
+import type { EncryptIDClaims } from "./auth";
+import {
+	isIPFSEnabled,
+	pinToIPFS,
+	unpinFromIPFS,
+	fetchFromIPFS,
+	getIPFSStatus,
+	ipfsGatewayUrl,
+} from "./ipfs";
+
+const MAX_PIN_SIZE = 50 * 1024 * 1024; // 50 MB
+
+type IPFSEnv = {
+	Variables: {
+		userId: string;
+		claims: EncryptIDClaims;
+	};
+};
+
+export const ipfsRouter = new Hono<IPFSEnv>();
+
+/** Auth middleware for mutating endpoints. */
+async function requireAuth(c: Context<IPFSEnv>, next: Next) {
+	const token = extractToken(c.req.raw.headers);
+	if (!token) return c.json({ error: "Authentication required" }, 401);
+	try {
+		const claims = await verifyToken(token);
+		c.set("userId", claims.sub);
+		c.set("claims", claims);
+	} catch {
+		return c.json({ error: "Invalid or expired token" }, 401);
+	}
+	await next();
+}
+
+/** GET /status — IPFS node health + stats */
+ipfsRouter.get("/status", requireAuth, async (c) => {
+	if (!isIPFSEnabled()) {
+		return c.json({ enabled: false }, 503);
+	}
+	try {
+		const status = await getIPFSStatus();
+		return c.json({ enabled: true, ...status });
+	} catch (e: any) {
+		return c.json({ enabled: true, error: e.message }, 502);
+	}
+});
+
+/** POST /pin — Pin a blob to IPFS */
+ipfsRouter.post("/pin", requireAuth, async (c) => {
+	if (!isIPFSEnabled()) {
+		return c.json({ error: "IPFS not enabled" }, 503);
+	}
+
+	const blob = await c.req.arrayBuffer();
+	if (blob.byteLength === 0) {
+		return c.json({ error: "Empty body" }, 400);
+	}
+	if (blob.byteLength > MAX_PIN_SIZE) {
+		return c.json({ error: `Blob too large (max ${MAX_PIN_SIZE} bytes)` }, 413);
+	}
+
+	const filename =
+		c.req.header("X-Filename") || `upload-${Date.now()}.bin`;
+
+	try {
+		const cid = await pinToIPFS(new Uint8Array(blob), filename);
+		return c.json({ cid, gateway: ipfsGatewayUrl(cid), size: blob.byteLength });
+	} catch (e: any) {
+		console.error("[ipfs/pin]", e.message);
+		return c.json({ error: "IPFS pin failed" }, 502);
+	}
+});
+
+/** DELETE /:cid — Unpin a CID */
+ipfsRouter.delete("/:cid", requireAuth, async (c) => {
+	if (!isIPFSEnabled()) {
+		return c.json({ error: "IPFS not enabled" }, 503);
+	}
+
+	const cid = c.req.param("cid");
+	try {
+		await unpinFromIPFS(cid);
+		return c.json({ ok: true, cid });
+	} catch (e: any) {
+		console.error("[ipfs/unpin]", e.message);
+		return c.json({ error: "IPFS unpin failed" }, 502);
+	}
+});
+
+/** GET /:cid — Proxy gateway fetch (public, no auth needed) */
+ipfsRouter.get("/:cid", async (c) => {
+	const cid = c.req.param("cid");
+	try {
+		const res = await fetchFromIPFS(cid);
+		if (!res.ok) {
+			return c.json({ error: "Not found on IPFS" }, 404);
+		}
+		const body = await res.arrayBuffer();
+		return new Response(body, {
+			headers: {
+				"Content-Type":
+					res.headers.get("Content-Type") || "application/octet-stream",
+				"Cache-Control": "public, max-age=31536000, immutable",
+			},
+		});
+	} catch (e: any) {
+		console.error("[ipfs/proxy]", e.message);
+		return c.json({ error: "IPFS fetch failed" }, 502);
+	}
+});

server/ipfs.ts

@@ -0,0 +1,102 @@
+/**
+ * IPFS Client — Server-side pinning and gateway access via Kubo API.
+ *
+ * No encryption here — backups are already encrypted by the client,
+ * and generated files are intentionally public.
+ */
+
+const IPFS_API_URL = process.env.IPFS_API_URL || "http://kubo:5001";
+const IPFS_GATEWAY_URL =
+	process.env.IPFS_GATEWAY_URL || "https://ipfs.jeffemmett.com";
+
+/** Returns true if IPFS_API_URL is configured (always true with default). */
+export function isIPFSEnabled(): boolean {
+	return !!IPFS_API_URL;
+}
+
+/** Public gateway URL for a CID. */
+export function ipfsGatewayUrl(cid: string): string {
+	return `${IPFS_GATEWAY_URL}/ipfs/${cid}`;
+}
+
+/**
+ * Pin a blob to IPFS via Kubo's /api/v0/add endpoint.
+ * Returns the CID (content hash).
+ */
+export async function pinToIPFS(
+	data: Uint8Array,
+	filename: string,
+): Promise<string> {
+	const formData = new FormData();
+	formData.append(
+		"file",
+		new Blob([data.buffer as ArrayBuffer]),
+		filename,
+	);
+
+	const res = await fetch(`${IPFS_API_URL}/api/v0/add?pin=true`, {
+		method: "POST",
+		body: formData,
+	});
+
+	if (!res.ok) {
+		const err = await res.text();
+		throw new Error(`IPFS pin failed (${res.status}): ${err}`);
+	}
+
+	const result = (await res.json()) as { Hash: string; Size: string };
+	return result.Hash;
+}
+
+/**
+ * Unpin a CID from the local Kubo node.
+ */
+export async function unpinFromIPFS(cid: string): Promise<void> {
+	const res = await fetch(
+		`${IPFS_API_URL}/api/v0/pin/rm?arg=${encodeURIComponent(cid)}`,
+		{ method: "POST" },
+	);
+	// 500 with "not pinned" is fine — idempotent unpin
+	if (!res.ok) {
+		const err = await res.text();
+		if (!err.includes("not pinned")) {
+			throw new Error(`IPFS unpin failed (${res.status}): ${err}`);
+		}
+	}
+}
+
+/**
+ * Fetch content from IPFS by CID (via gateway).
+ */
+export async function fetchFromIPFS(cid: string): Promise<Response> {
+	return fetch(`${IPFS_GATEWAY_URL}/ipfs/${cid}`);
+}
+
+/**
+ * Get IPFS node status — peer ID, repo size, number of objects.
+ */
+export async function getIPFSStatus(): Promise<{
+	peerId: string;
+	repoSize: number;
+	numObjects: number;
+}> {
+	const [idRes, statRes] = await Promise.all([
+		fetch(`${IPFS_API_URL}/api/v0/id`, { method: "POST" }),
+		fetch(`${IPFS_API_URL}/api/v0/repo/stat`, { method: "POST" }),
+	]);
+
+	if (!idRes.ok) throw new Error(`IPFS id failed: ${idRes.status}`);
+	if (!statRes.ok) throw new Error(`IPFS repo/stat failed: ${statRes.status}`);
+
+	const id = (await idRes.json()) as { ID: string };
+	const stat = (await statRes.json()) as {
+		RepoSize: number;
+		NumObjects: number;
+	};
+
+	return {
+		peerId: id.ID,
+		repoSize: stat.RepoSize,
+		numObjects: stat.NumObjects,
+	};
+}

server/local-first/backup-routes.ts

@@ -17,6 +17,7 @@ import {
 	deleteAllBackups,
 	getUsage,
 } from "./backup-store";
+import { ipfsGatewayUrl } from "../ipfs";
 
 const MAX_BLOB_SIZE = 10 * 1024 * 1024; // 10 MB per blob
 
@@ -93,6 +94,20 @@ backupRouter.get("/:space", async (c) => {
 	return c.json(manifest);
 });
 
+/** GET /api/backup/:space/:docId/ipfs — IPFS gateway URL for a backup blob */
+backupRouter.get("/:space/:docId/ipfs", async (c) => {
+	const userId = c.get("userId");
+	const space = c.req.param("space");
+	const docId = decodeURIComponent(c.req.param("docId"));
+
+	const manifest = await listBackups(userId, space);
+	const entry = manifest.entries.find((e) => e.docId === docId);
+	if (!entry) return c.json({ error: "Not found" }, 404);
+	if (!entry.cid) return c.json({ error: "Not pinned to IPFS" }, 404);
+
+	return c.json({ cid: entry.cid, gateway: ipfsGatewayUrl(entry.cid) });
+});
+
 /** DELETE /api/backup/:space/:docId — delete specific backup */
 backupRouter.delete("/:space/:docId", async (c) => {
 	const userId = c.get("userId");

server/local-first/backup-store.ts

@@ -11,6 +11,7 @@
 import { resolve, dirname } from "node:path";
 import { mkdir, readdir, readFile, writeFile, unlink, stat, rm } from "node:fs/promises";
 import { createHash } from "node:crypto";
+import { isIPFSEnabled, pinToIPFS, unpinFromIPFS } from "../ipfs";
 
 const BACKUPS_DIR = process.env.BACKUPS_DIR || "/data/backups";
 
@@ -19,6 +20,7 @@ export interface BackupManifestEntry {
 	hash: string;
 	size: number;
 	updatedAt: string;
+	cid?: string;
 }
 
 export interface BackupManifest {
@@ -106,6 +108,22 @@ export async function putBackup(
 	}
 	manifest.updatedAt = new Date().toISOString();
 	await saveManifest(userId, spaceSlug, manifest);
+
+	// Fire-and-forget IPFS pin — filesystem is primary, IPFS is redundancy
+	if (isIPFSEnabled()) {
+		pinToIPFS(blob, `${docIdHash(docId)}.enc`)
+			.then((cid) => {
+				// Update manifest entry with CID
+				const idx = manifest.entries.findIndex((e) => e.docId === docId);
+				if (idx >= 0) {
+					manifest.entries[idx].cid = cid;
+					saveManifest(userId, spaceSlug, manifest).catch(() => {});
+				}
+			})
+			.catch((err) => {
+				console.warn("[backup] IPFS pin failed (non-fatal):", err.message);
+			});
+	}
 }
 
 /**
@@ -148,11 +166,21 @@ export async function deleteBackup(
 	docId: string,
 ): Promise<boolean> {
 	try {
+		// Check for IPFS CID before removing from manifest
+		const manifest = await loadManifest(userId, spaceSlug);
+		const entry = manifest.entries.find((e) => e.docId === docId);
+
 		const path = blobPath(userId, spaceSlug, docId);
 		await unlink(path);
 
+		// Fire-and-forget IPFS unpin
+		if (entry?.cid && isIPFSEnabled()) {
+			unpinFromIPFS(entry.cid).catch((err) => {
+				console.warn("[backup] IPFS unpin failed (non-fatal):", err.message);
+			});
+		}
+
 		// Update manifest
-		const manifest = await loadManifest(userId, spaceSlug);
 		manifest.entries = manifest.entries.filter((e) => e.docId !== docId);
 		manifest.updatedAt = new Date().toISOString();
 		await saveManifest(userId, spaceSlug, manifest);