diff --git a/docker-compose.yml b/docker-compose.yml index 0174b9e..002534a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,6 +63,8 @@ services: - TRANSAK_ENV=PRODUCTION - SCRIBUS_BRIDGE_SECRET=${SCRIBUS_BRIDGE_SECRET} - SCRIBUS_NOVNC_URL=https://design.rspace.online + - IPFS_API_URL=http://kubo:5001 + - IPFS_GATEWAY_URL=https://ipfs.jeffemmett.com depends_on: rspace-db: condition: service_healthy diff --git a/server/index.ts b/server/index.ts index 10e0b1c..f819e2c 100644 --- a/server/index.ts +++ b/server/index.ts @@ -93,6 +93,8 @@ import { renderMainLanding, renderSpaceDashboard } from "./landing"; import { syncServer } from "./sync-instance"; import { loadAllDocs } from "./local-first/doc-persistence"; import { backupRouter } from "./local-first/backup-routes"; +import { ipfsRouter } from "./ipfs-routes"; +import { isIPFSEnabled, pinToIPFS } from "./ipfs"; import { oauthRouter, setOAuthStatusSyncServer } from "./oauth/index"; import { setNotionOAuthSyncServer } from "./oauth/notion"; import { setGoogleOAuthSyncServer } from "./oauth/google"; @@ -245,6 +247,53 @@ app.get("/data/files/generated/:subdir/:filename", serveGeneratedFile); app.get("/api/files/generated/:filename", serveGeneratedFile); app.get("/api/files/generated/:subdir/:filename", serveGeneratedFile); +// ── IPFS background pinning for generated files ── + +/** In-memory cache of filePath → CID. Populated from .cid sidecar files on startup. */ +const generatedCidCache = new Map(); + +/** Load existing .cid sidecar files into cache on startup. */ +async function loadGeneratedCids() { + const dir = resolve(process.env.FILES_DIR || "./data/files", "generated"); + try { + const { readdir, readFile } = await import("node:fs/promises"); + const files = await readdir(dir); + for (const f of files) { + if (!f.endsWith(".cid")) continue; + try { + const cid = (await readFile(resolve(dir, f), "utf-8")).trim(); + const originalFile = f.replace(/\.cid$/, ""); + generatedCidCache.set(originalFile, cid); + } catch {} + } + if (generatedCidCache.size > 0) { + console.log(`[ipfs] Loaded ${generatedCidCache.size} CID sidecar files`); + } + } catch {} +} +loadGeneratedCids(); + +/** + * Pin a generated file to IPFS in the background. + * Writes a .cid sidecar file alongside the original. + * Safe to call fire-and-forget — failures are logged and swallowed. + */ +async function pinGeneratedFile(filePath: string, filename: string) { + if (!isIPFSEnabled()) return; + if (generatedCidCache.has(filename)) return; + try { + const file = Bun.file(filePath); + const buf = new Uint8Array(await file.arrayBuffer()); + const cid = await pinToIPFS(buf, filename); + generatedCidCache.set(filename, cid); + const { writeFile } = await import("node:fs/promises"); + await writeFile(`${filePath}.cid`, cid); + console.log(`[ipfs] Pinned ${filename} → ${cid}`); + } catch (err: any) { + console.warn(`[ipfs] Pin failed for ${filename} (non-fatal):`, err.message); + } +} + // ── Link preview / unfurl API ── const linkPreviewCache = new Map(); @@ -453,6 +502,9 @@ app.route("/api/spaces", spaces); // ── Backup API (encrypted blob storage) ── app.route("/api/backup", backupRouter); +// ── IPFS API (pinning + gateway proxy) ── +app.route("/api/ipfs", ipfsRouter); + // ── OAuth API (Notion, Google integrations) ── app.route("/api/oauth", oauthRouter); @@ -1172,6 +1224,7 @@ async function process3DGenJob(job: Gen3DJob) { const filename = `splat-${Date.now()}-${Math.random().toString(36).slice(2, 6)}.glb`; const dir = resolve(process.env.FILES_DIR || "./data/files", "generated"); await Bun.write(resolve(dir, filename), modelBuf); + pinGeneratedFile(resolve(dir, filename), filename); job.status = "complete"; job.resultUrl = `/data/files/generated/${filename}`; @@ -1211,6 +1264,7 @@ async function saveDataUrlToDisk(dataUrl: string, prefix: string): Promise { const filename = `img2img-${Date.now()}.png`; const dir = resolve(process.env.FILES_DIR || "./data/files", "generated"); await Bun.write(resolve(dir, filename), Buffer.from(imgBuf)); + pinGeneratedFile(resolve(dir, filename), filename); const localUrl = `/data/files/generated/${filename}`; return c.json({ url: localUrl, image_url: localUrl }); } catch (e: any) { @@ -1423,6 +1478,7 @@ app.post("/api/image-gen/img2img", async (c) => { const filename = `img2img-gemini-${Date.now()}.${ext}`; const dir = resolve(process.env.FILES_DIR || "./data/files", "generated"); await Bun.write(resolve(dir, filename), Buffer.from(b64, "base64")); + pinGeneratedFile(resolve(dir, filename), filename); const url = `/data/files/generated/${filename}`; return c.json({ url, image_url: url }); } @@ -2178,6 +2234,7 @@ app.post("/api/gemini/image", async (c) => { const filename = `gemini-${Date.now()}.${ext}`; const dir = resolve(process.env.FILES_DIR || "./data/files", "generated"); await Bun.write(resolve(dir, filename), Buffer.from(b64, "base64")); + pinGeneratedFile(resolve(dir, filename), filename); const url = `/data/files/generated/${filename}`; return c.json({ url, image_url: url }); } @@ -2197,6 +2254,7 @@ app.post("/api/gemini/image", async (c) => { const filename = `imagen-${Date.now()}.png`; const dir = resolve(process.env.FILES_DIR || "./data/files", "generated"); await Bun.write(resolve(dir, filename), Buffer.from(img.image.imageBytes, "base64")); + pinGeneratedFile(resolve(dir, filename), filename); const url = `/data/files/generated/${filename}`; return c.json({ url, image_url: url }); } @@ -2328,6 +2386,7 @@ app.post("/api/zine/page", async (c) => { const filename = `zine-p${outline.pageNumber}-${section.id}-${Date.now()}.${ext}`; const dir = resolve(process.env.FILES_DIR || "./data/files", "generated"); await Bun.write(resolve(dir, filename), Buffer.from(b64, "base64")); + pinGeneratedFile(resolve(dir, filename), filename); generatedImages[section.id] = `/data/files/generated/${filename}`; break; } @@ -2374,6 +2433,7 @@ app.post("/api/zine/regenerate-section", async (c) => { const filename = `zine-regen-${section.id}-${Date.now()}.${ext}`; const dir = resolve(process.env.FILES_DIR || "./data/files", "generated"); await Bun.write(resolve(dir, filename), Buffer.from(b64, "base64")); + pinGeneratedFile(resolve(dir, filename), filename); return c.json({ sectionId: section.id, type: "image", url: `/data/files/generated/${filename}` }); } } diff --git a/server/ipfs-routes.ts b/server/ipfs-routes.ts new file mode 100644 index 0000000..e848e1d --- /dev/null +++ b/server/ipfs-routes.ts @@ -0,0 +1,123 @@ +/** + * IPFS API Routes — Hono router mounted at /api/ipfs. + * + * GET /api/ipfs/status — IPFS node health + stats (authenticated) + * GET /api/ipfs/:cid — Proxy gateway fetch (avoids CORS for clients) + * POST /api/ipfs/pin — Pin arbitrary blob (authenticated, 50MB limit) + * DELETE /api/ipfs/:cid — Unpin a CID (authenticated) + */ + +import { Hono } from "hono"; +import type { Context, Next } from "hono"; +import { verifyToken, extractToken } from "./auth"; +import type { EncryptIDClaims } from "./auth"; +import { + isIPFSEnabled, + pinToIPFS, + unpinFromIPFS, + fetchFromIPFS, + getIPFSStatus, + ipfsGatewayUrl, +} from "./ipfs"; + +const MAX_PIN_SIZE = 50 * 1024 * 1024; // 50 MB + +type IPFSEnv = { + Variables: { + userId: string; + claims: EncryptIDClaims; + }; +}; + +export const ipfsRouter = new Hono(); + +/** Auth middleware for mutating endpoints. */ +async function requireAuth(c: Context, next: Next) { + const token = extractToken(c.req.raw.headers); + if (!token) return c.json({ error: "Authentication required" }, 401); + try { + const claims = await verifyToken(token); + c.set("userId", claims.sub); + c.set("claims", claims); + } catch { + return c.json({ error: "Invalid or expired token" }, 401); + } + await next(); +} + +/** GET /status — IPFS node health + stats */ +ipfsRouter.get("/status", requireAuth, async (c) => { + if (!isIPFSEnabled()) { + return c.json({ enabled: false }, 503); + } + try { + const status = await getIPFSStatus(); + return c.json({ enabled: true, ...status }); + } catch (e: any) { + return c.json({ enabled: true, error: e.message }, 502); + } +}); + +/** POST /pin — Pin a blob to IPFS */ +ipfsRouter.post("/pin", requireAuth, async (c) => { + if (!isIPFSEnabled()) { + return c.json({ error: "IPFS not enabled" }, 503); + } + + const blob = await c.req.arrayBuffer(); + if (blob.byteLength === 0) { + return c.json({ error: "Empty body" }, 400); + } + if (blob.byteLength > MAX_PIN_SIZE) { + return c.json({ error: `Blob too large (max ${MAX_PIN_SIZE} bytes)` }, 413); + } + + const filename = + c.req.header("X-Filename") || `upload-${Date.now()}.bin`; + + try { + const cid = await pinToIPFS(new Uint8Array(blob), filename); + return c.json({ cid, gateway: ipfsGatewayUrl(cid), size: blob.byteLength }); + } catch (e: any) { + console.error("[ipfs/pin]", e.message); + return c.json({ error: "IPFS pin failed" }, 502); + } +}); + +/** DELETE /:cid — Unpin a CID */ +ipfsRouter.delete("/:cid", requireAuth, async (c) => { + if (!isIPFSEnabled()) { + return c.json({ error: "IPFS not enabled" }, 503); + } + + const cid = c.req.param("cid"); + try { + await unpinFromIPFS(cid); + return c.json({ ok: true, cid }); + } catch (e: any) { + console.error("[ipfs/unpin]", e.message); + return c.json({ error: "IPFS unpin failed" }, 502); + } +}); + +/** GET /:cid — Proxy gateway fetch (public, no auth needed) */ +ipfsRouter.get("/:cid", async (c) => { + const cid = c.req.param("cid"); + try { + const res = await fetchFromIPFS(cid); + if (!res.ok) { + return c.json({ error: "Not found on IPFS" }, 404); + } + const body = await res.arrayBuffer(); + return new Response(body, { + headers: { + "Content-Type": + res.headers.get("Content-Type") || "application/octet-stream", + "Cache-Control": "public, max-age=31536000, immutable", + }, + }); + } catch (e: any) { + console.error("[ipfs/proxy]", e.message); + return c.json({ error: "IPFS fetch failed" }, 502); + } +}); diff --git a/server/ipfs.ts b/server/ipfs.ts new file mode 100644 index 0000000..e916869 --- /dev/null +++ b/server/ipfs.ts @@ -0,0 +1,102 @@ +/** + * IPFS Client — Server-side pinning and gateway access via Kubo API. + * + * No encryption here — backups are already encrypted by the client, + * and generated files are intentionally public. + */ + +const IPFS_API_URL = process.env.IPFS_API_URL || "http://kubo:5001"; +const IPFS_GATEWAY_URL = + process.env.IPFS_GATEWAY_URL || "https://ipfs.jeffemmett.com"; + +/** Returns true if IPFS_API_URL is configured (always true with default). */ +export function isIPFSEnabled(): boolean { + return !!IPFS_API_URL; +} + +/** Public gateway URL for a CID. */ +export function ipfsGatewayUrl(cid: string): string { + return `${IPFS_GATEWAY_URL}/ipfs/${cid}`; +} + +/** + * Pin a blob to IPFS via Kubo's /api/v0/add endpoint. + * Returns the CID (content hash). + */ +export async function pinToIPFS( + data: Uint8Array, + filename: string, +): Promise { + const formData = new FormData(); + formData.append( + "file", + new Blob([data.buffer as ArrayBuffer]), + filename, + ); + + const res = await fetch(`${IPFS_API_URL}/api/v0/add?pin=true`, { + method: "POST", + body: formData, + }); + + if (!res.ok) { + const err = await res.text(); + throw new Error(`IPFS pin failed (${res.status}): ${err}`); + } + + const result = (await res.json()) as { Hash: string; Size: string }; + return result.Hash; +} + +/** + * Unpin a CID from the local Kubo node. + */ +export async function unpinFromIPFS(cid: string): Promise { + const res = await fetch( + `${IPFS_API_URL}/api/v0/pin/rm?arg=${encodeURIComponent(cid)}`, + { method: "POST" }, + ); + // 500 with "not pinned" is fine — idempotent unpin + if (!res.ok) { + const err = await res.text(); + if (!err.includes("not pinned")) { + throw new Error(`IPFS unpin failed (${res.status}): ${err}`); + } + } +} + +/** + * Fetch content from IPFS by CID (via gateway). + */ +export async function fetchFromIPFS(cid: string): Promise { + return fetch(`${IPFS_GATEWAY_URL}/ipfs/${cid}`); +} + +/** + * Get IPFS node status — peer ID, repo size, number of objects. + */ +export async function getIPFSStatus(): Promise<{ + peerId: string; + repoSize: number; + numObjects: number; +}> { + const [idRes, statRes] = await Promise.all([ + fetch(`${IPFS_API_URL}/api/v0/id`, { method: "POST" }), + fetch(`${IPFS_API_URL}/api/v0/repo/stat`, { method: "POST" }), + ]); + + if (!idRes.ok) throw new Error(`IPFS id failed: ${idRes.status}`); + if (!statRes.ok) throw new Error(`IPFS repo/stat failed: ${statRes.status}`); + + const id = (await idRes.json()) as { ID: string }; + const stat = (await statRes.json()) as { + RepoSize: number; + NumObjects: number; + }; + + return { + peerId: id.ID, + repoSize: stat.RepoSize, + numObjects: stat.NumObjects, + }; +} diff --git a/server/local-first/backup-routes.ts b/server/local-first/backup-routes.ts index 39a5e08..8c236eb 100644 --- a/server/local-first/backup-routes.ts +++ b/server/local-first/backup-routes.ts @@ -17,6 +17,7 @@ import { deleteAllBackups, getUsage, } from "./backup-store"; +import { ipfsGatewayUrl } from "../ipfs"; const MAX_BLOB_SIZE = 10 * 1024 * 1024; // 10 MB per blob @@ -93,6 +94,20 @@ backupRouter.get("/:space", async (c) => { return c.json(manifest); }); +/** GET /api/backup/:space/:docId/ipfs — IPFS gateway URL for a backup blob */ +backupRouter.get("/:space/:docId/ipfs", async (c) => { + const userId = c.get("userId"); + const space = c.req.param("space"); + const docId = decodeURIComponent(c.req.param("docId")); + + const manifest = await listBackups(userId, space); + const entry = manifest.entries.find((e) => e.docId === docId); + if (!entry) return c.json({ error: "Not found" }, 404); + if (!entry.cid) return c.json({ error: "Not pinned to IPFS" }, 404); + + return c.json({ cid: entry.cid, gateway: ipfsGatewayUrl(entry.cid) }); +}); + /** DELETE /api/backup/:space/:docId — delete specific backup */ backupRouter.delete("/:space/:docId", async (c) => { const userId = c.get("userId"); diff --git a/server/local-first/backup-store.ts b/server/local-first/backup-store.ts index 4c22716..367e593 100644 --- a/server/local-first/backup-store.ts +++ b/server/local-first/backup-store.ts @@ -11,6 +11,7 @@ import { resolve, dirname } from "node:path"; import { mkdir, readdir, readFile, writeFile, unlink, stat, rm } from "node:fs/promises"; import { createHash } from "node:crypto"; +import { isIPFSEnabled, pinToIPFS, unpinFromIPFS } from "../ipfs"; const BACKUPS_DIR = process.env.BACKUPS_DIR || "/data/backups"; @@ -19,6 +20,7 @@ export interface BackupManifestEntry { hash: string; size: number; updatedAt: string; + cid?: string; } export interface BackupManifest { @@ -106,6 +108,22 @@ export async function putBackup( } manifest.updatedAt = new Date().toISOString(); await saveManifest(userId, spaceSlug, manifest); + + // Fire-and-forget IPFS pin — filesystem is primary, IPFS is redundancy + if (isIPFSEnabled()) { + pinToIPFS(blob, `${docIdHash(docId)}.enc`) + .then((cid) => { + // Update manifest entry with CID + const idx = manifest.entries.findIndex((e) => e.docId === docId); + if (idx >= 0) { + manifest.entries[idx].cid = cid; + saveManifest(userId, spaceSlug, manifest).catch(() => {}); + } + }) + .catch((err) => { + console.warn("[backup] IPFS pin failed (non-fatal):", err.message); + }); + } } /** @@ -148,11 +166,21 @@ export async function deleteBackup( docId: string, ): Promise { try { + // Check for IPFS CID before removing from manifest + const manifest = await loadManifest(userId, spaceSlug); + const entry = manifest.entries.find((e) => e.docId === docId); + const path = blobPath(userId, spaceSlug, docId); await unlink(path); + // Fire-and-forget IPFS unpin + if (entry?.cid && isIPFSEnabled()) { + unpinFromIPFS(entry.cid).catch((err) => { + console.warn("[backup] IPFS unpin failed (non-fatal):", err.message); + }); + } + // Update manifest - const manifest = await loadManifest(userId, spaceSlug); manifest.entries = manifest.entries.filter((e) => e.docId !== docId); manifest.updatedAt = new Date().toISOString(); await saveManifest(userId, spaceSlug, manifest);