From 999502464fa4a2e5d5e76ad4e49237b726898f72 Mon Sep 17 00:00:00 2001 From: Jeff Emmett Date: Sun, 22 Mar 2026 16:42:08 -0700 Subject: [PATCH] feat(prompt): add canvas tool use via Gemini function calling folk-prompt can now spawn shapes on the canvas when Tools mode is enabled. Gemini calls functions (create_map, create_note, create_embed, create_image, create_bookmark, create_image_gen) and the client executes them via window.__canvasApi. Multi-turn loop on server (max 5 rounds) with synthetic success responses. Extensible via registerCanvasTool(). Co-Authored-By: Claude Opus 4.6 --- lib/canvas-tools.ts | 156 ++++++++++++++++++++++++++++++++++++++++++++ lib/folk-prompt.ts | 134 ++++++++++++++++++++++++++++++++----- server/index.ts | 94 +++++++++++++++++++++----- 3 files changed, 351 insertions(+), 33 deletions(-) create mode 100644 lib/canvas-tools.ts diff --git a/lib/canvas-tools.ts b/lib/canvas-tools.ts new file mode 100644 index 0000000..449776e --- /dev/null +++ b/lib/canvas-tools.ts @@ -0,0 +1,156 @@ +/** + * Canvas Tool Registry — shared by server (Gemini function declarations) and client (shape spawning). + * Pure TypeScript, no DOM or server dependencies. + */ + +export interface CanvasToolDefinition { + declaration: { + name: string; + description: string; + parameters: { + type: "object"; + properties: Record; + required: string[]; + }; + }; + tagName: string; + buildProps: (args: Record) => Record; + actionLabel: (args: Record) => string; +} + +const registry: CanvasToolDefinition[] = [ + { + declaration: { + name: "create_map", + description: "Create an interactive map centered on a location. Use when the user wants to see a place, get directions, or explore a geographic area.", + parameters: { + type: "object", + properties: { + latitude: { type: "number", description: "Latitude of the center point" }, + longitude: { type: "number", description: "Longitude of the center point" }, + zoom: { type: "number", description: "Zoom level (1-18, default 12)" }, + location_name: { type: "string", description: "Human-readable name of the location" }, + }, + required: ["latitude", "longitude", "location_name"], + }, + }, + tagName: "folk-map", + buildProps: (args) => ({ + center: [args.longitude, args.latitude], + zoom: args.zoom || 12, + }), + actionLabel: (args) => `Created map: ${args.location_name}`, + }, + { + declaration: { + name: "create_note", + description: "Create a markdown note on the canvas. Use for text content, lists, summaries, instructions, or any written information.", + parameters: { + type: "object", + properties: { + content: { type: "string", description: "Markdown content for the note" }, + title: { type: "string", description: "Optional title for the note" }, + }, + required: ["content"], + }, + }, + tagName: "folk-markdown", + buildProps: (args) => ({ + value: args.title ? `# ${args.title}\n\n${args.content}` : args.content, + }), + actionLabel: (args) => `Created note${args.title ? `: ${args.title}` : ""}`, + }, + { + declaration: { + name: "create_embed", + description: "Embed a webpage or web app on the canvas. Use for websites, search results, booking sites, videos, or any URL the user wants to view inline.", + parameters: { + type: "object", + properties: { + url: { type: "string", description: "The URL to embed" }, + title: { type: "string", description: "Descriptive title for the embed" }, + }, + required: ["url"], + }, + }, + tagName: "folk-embed", + buildProps: (args) => ({ + url: args.url, + }), + actionLabel: (args) => `Embedded: ${args.title || args.url}`, + }, + { + declaration: { + name: "create_image", + description: "Display an image on the canvas from a URL. Use when showing an existing image, photo, diagram, or any direct image link.", + parameters: { + type: "object", + properties: { + src: { type: "string", description: "Image URL" }, + alt: { type: "string", description: "Alt text describing the image" }, + }, + required: ["src"], + }, + }, + tagName: "folk-image", + buildProps: (args) => ({ + src: args.src, + alt: args.alt || "", + }), + actionLabel: (args) => `Created image${args.alt ? `: ${args.alt}` : ""}`, + }, + { + declaration: { + name: "create_bookmark", + description: "Create a bookmark card for a URL. Use when the user wants to save or reference a link without embedding the full page.", + parameters: { + type: "object", + properties: { + url: { type: "string", description: "The URL to bookmark" }, + }, + required: ["url"], + }, + }, + tagName: "folk-bookmark", + buildProps: (args) => ({ + url: args.url, + }), + actionLabel: (args) => `Bookmarked: ${args.url}`, + }, + { + declaration: { + name: "create_image_gen", + description: "Generate an AI image from a text prompt. Use when the user wants to create, generate, or imagine a new image that doesn't exist yet.", + parameters: { + type: "object", + properties: { + prompt: { type: "string", description: "Text prompt describing the image to generate" }, + style: { + type: "string", + description: "Visual style for the generated image", + enum: ["photorealistic", "illustration", "painting", "sketch", "punk-zine", "collage", "vintage", "minimalist"], + }, + }, + required: ["prompt"], + }, + }, + tagName: "folk-image-gen", + buildProps: (args) => ({ + prompt: args.prompt, + style: args.style || "photorealistic", + }), + actionLabel: (args) => `Generating image: ${args.prompt.slice(0, 50)}${args.prompt.length > 50 ? "..." : ""}`, + }, +]; + +export const CANVAS_TOOLS: CanvasToolDefinition[] = [...registry]; + +export const CANVAS_TOOL_DECLARATIONS = CANVAS_TOOLS.map((t) => t.declaration); + +export function findTool(name: string): CanvasToolDefinition | undefined { + return CANVAS_TOOLS.find((t) => t.declaration.name === name); +} + +export function registerCanvasTool(def: CanvasToolDefinition): void { + CANVAS_TOOLS.push(def); +} diff --git a/lib/folk-prompt.ts b/lib/folk-prompt.ts index 15c3f93..bdb5d31 100644 --- a/lib/folk-prompt.ts +++ b/lib/folk-prompt.ts @@ -1,6 +1,7 @@ import { FolkShape } from "./folk-shape"; import { css, html } from "./tags"; import { SpeechDictation } from "./speech-dictation"; +import { findTool } from "./canvas-tools"; const styles = css` :host { @@ -304,11 +305,50 @@ const styles = css` code { font-family: "Monaco", "Consolas", monospace; } + + .input-controls { + display: flex; + gap: 6px; + align-items: center; + } + + .tools-btn { + padding: 5px 10px; + border: 2px solid var(--rs-input-border, #e2e8f0); + border-radius: 6px; + font-size: 12px; + background: var(--rs-input-bg, #fff); + color: var(--rs-text-muted, #94a3b8); + cursor: pointer; + transition: all 0.2s; + white-space: nowrap; + } + + .tools-btn:hover { + border-color: #6366f1; + } + + .tools-btn.active { + background: #6366f1; + border-color: #6366f1; + color: white; + } + + .message.tool-action { + align-self: center; + background: #ecfdf5; + color: #065f46; + border-radius: 20px; + padding: 6px 14px; + font-size: 12px; + font-weight: 500; + max-width: 90%; + } `; export interface ChatMessage { id: string; - role: "user" | "assistant"; + role: "user" | "assistant" | "tool-action"; content: string; images?: string[]; timestamp: Date; @@ -345,6 +385,7 @@ export class FolkPrompt extends FolkShape { #error: string | null = null; #model = "gemini-flash"; #pendingImages: string[] = []; + #toolsEnabled = false; #messagesEl: HTMLElement | null = null; #promptInput: HTMLTextAreaElement | null = null; @@ -352,6 +393,7 @@ export class FolkPrompt extends FolkShape { #sendBtn: HTMLButtonElement | null = null; #attachInput: HTMLInputElement | null = null; #pendingImagesEl: HTMLElement | null = null; + #toolsBtn: HTMLButtonElement | null = null; get messages() { return this.#messages; @@ -381,18 +423,21 @@ export class FolkPrompt extends FolkShape {
- +
+ + +
@@ -418,10 +463,23 @@ export class FolkPrompt extends FolkShape { this.#sendBtn = wrapper.querySelector(".send-btn"); this.#attachInput = wrapper.querySelector(".attach-input"); this.#pendingImagesEl = wrapper.querySelector(".pending-images"); + this.#toolsBtn = wrapper.querySelector(".tools-btn") as HTMLButtonElement; const clearBtn = wrapper.querySelector(".clear-btn") as HTMLButtonElement; const closeBtn = wrapper.querySelector(".close-btn") as HTMLButtonElement; const attachBtn = wrapper.querySelector(".attach-btn") as HTMLButtonElement | null; + // Tools toggle + this.#toolsBtn?.addEventListener("click", (e) => { + e.stopPropagation(); + this.#toolsEnabled = !this.#toolsEnabled; + this.#toolsBtn!.classList.toggle("active", this.#toolsEnabled); + if (this.#promptInput) { + this.#promptInput.placeholder = this.#toolsEnabled + ? "Ask me to create maps, notes, images, or embeds on the canvas..." + : "Type your message..."; + } + }); + // Attach button attachBtn?.addEventListener("click", (e) => { e.stopPropagation(); @@ -566,16 +624,18 @@ export class FolkPrompt extends FolkShape { this.#renderMessages(true); try { + const useTools = this.#toolsEnabled && this.#model.startsWith("gemini"); const response = await fetch("/api/prompt", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ - messages: this.#messages.map((m) => ({ + messages: this.#messages.filter((m) => m.role !== "tool-action").map((m) => ({ role: m.role, content: m.content, ...(m.images?.length ? { images: m.images } : {}), })), model: this.#model, + ...(useTools ? { useTools: true } : {}), }), }); @@ -585,6 +645,11 @@ export class FolkPrompt extends FolkShape { const result = await response.json(); + // Execute tool calls if any + if (result.toolCalls?.length) { + this.#executeToolCalls(result.toolCalls); + } + const assistantMessage: ChatMessage = { id: crypto.randomUUID(), role: "assistant", @@ -607,6 +672,42 @@ export class FolkPrompt extends FolkShape { } } + #executeToolCalls(toolCalls: { name: string; args: Record; label: string }[]) { + const api = (window as any).__canvasApi; + if (!api) { + console.warn("[folk-prompt] Canvas API not available — cannot spawn shapes"); + return; + } + + let offsetY = 0; + for (const tc of toolCalls) { + const tool = findTool(tc.name); + if (!tool) continue; + + const props = tool.buildProps(tc.args); + const defaults = api.SHAPE_DEFAULTS?.[tool.tagName] || { width: 300, height: 200 }; + const preferX = this.x + this.width + 40 + defaults.width / 2; + const preferY = this.y + offsetY + defaults.height / 2; + const pos = api.findFreePosition(defaults.width, defaults.height, preferX, preferY, this); + + try { + api.newShape(tool.tagName, props, { x: pos.x + defaults.width / 2, y: pos.y + defaults.height / 2 }); + } catch (e) { + console.error(`[folk-prompt] Failed to create ${tool.tagName}:`, e); + } + + // Tool action card in chat + this.#messages.push({ + id: crypto.randomUUID(), + role: "tool-action", + content: tc.label, + timestamp: new Date(), + }); + + offsetY += defaults.height + 20; + } + } + #clearChat() { this.#messages = []; this.#error = null; @@ -629,6 +730,9 @@ export class FolkPrompt extends FolkShape { let messagesHtml = this.#messages .map((msg) => { + if (msg.role === "tool-action") { + return `
${this.#escapeHtml(msg.content)}
`; + } let imgHtml = ""; if (msg.images?.length) { imgHtml = `
${msg.images.map((src) => ``).join("")}
`; @@ -684,7 +788,7 @@ export class FolkPrompt extends FolkShape { ...super.toJSON(), type: "folk-prompt", model: this.#model, - messages: this.messages.map((msg) => ({ + messages: this.messages.filter((m) => m.role !== "tool-action").map((msg) => ({ role: msg.role, content: msg.content, ...(msg.images?.length ? { images: msg.images } : {}), diff --git a/server/index.ts b/server/index.ts index a669967..45e22f9 100644 --- a/server/index.ts +++ b/server/index.ts @@ -35,12 +35,12 @@ import { seedTemplateShapes } from "./seed-template"; // Campaign demo moved to rsocials module — see modules/rsocials/campaign-data.ts import type { SpaceVisibility } from "./community-store"; import { - verifyEncryptIDToken, evaluateSpaceAccess, - extractToken, authenticateWSUpgrade, } from "@encryptid/sdk/server"; -import type { EncryptIDClaims, SpaceAuthConfig } from "@encryptid/sdk/server"; +import type { SpaceAuthConfig } from "@encryptid/sdk/server"; +import { verifyToken, extractToken } from "./auth"; +import type { EncryptIDClaims } from "./auth"; // ── Module system ── import { registerModule, getAllModules, getModuleInfoList, getModule } from "../shared/module"; @@ -518,7 +518,7 @@ app.post("/api/communities", async (c) => { let claims: EncryptIDClaims; try { - claims = await verifyEncryptIDToken(token); + claims = await verifyToken(token); } catch { return c.json({ error: "Invalid or expired authentication token" }, 401); } @@ -653,7 +653,7 @@ app.get("/api/space-access/:slug", async (c) => { const token = extractToken(c.req.raw.headers); if (!token) return c.json({ access: false, reason: "not-authenticated" }); let claims: EncryptIDClaims | null = null; - try { claims = await verifyEncryptIDToken(token); } catch {} + try { claims = await verifyToken(token); } catch {} if (!claims) return c.json({ access: false, reason: "not-authenticated" }); const config = await getSpaceConfig(slug); @@ -711,7 +711,7 @@ import { getBalance, getTokenDoc, transferTokens, mintFromOnChain } from "./toke // Wire EncryptID JWT verifier into CRDT scheme setTokenVerifier(async (token: string) => { - const claims = await verifyEncryptIDToken(token); + const claims = await verifyToken(token); return { sub: claims.sub, did: claims.did as string | undefined, username: claims.username }; }); @@ -735,7 +735,7 @@ const x402Test = setupX402FromEnv({ console.warn("[x402 bridge] No JWT — skipping cUSDC mint (on-chain payment still valid)"); return; } - const claims = await verifyEncryptIDToken(token); + const claims = await verifyToken(token); const did = (claims.did as string) || `did:key:${(claims.sub as string).slice(0, 32)}`; const label = claims.username || did; const amount = process.env.X402_UPLOAD_PRICE || "0.01"; @@ -1619,8 +1619,13 @@ const OLLAMA_MODELS: Record = { "mistral-small": "mistral-small:24b", }; +const CANVAS_TOOLS_SYSTEM_PROMPT = `You are a helpful AI assistant in rSpace, a collaborative canvas workspace. +When the user asks to create, show, display, or visualize something, use the available tools to spawn shapes on the canvas. +After creating shapes, give a brief summary of what you placed. Only create shapes directly relevant to the request. +For text-only questions (explanations, coding help, math), respond with text — don't create shapes unless asked.`; + app.post("/api/prompt", async (c) => { - const { messages, model = "gemini-flash" } = await c.req.json(); + const { messages, model = "gemini-flash", useTools = false, systemPrompt } = await c.req.json(); if (!messages?.length) return c.json({ error: "messages required" }, 400); // Determine provider @@ -1629,7 +1634,15 @@ app.post("/api/prompt", async (c) => { const { GoogleGenerativeAI } = await import("@google/generative-ai"); const genAI = new GoogleGenerativeAI(GEMINI_API_KEY); - const geminiModel = genAI.getGenerativeModel({ model: GEMINI_MODELS[model] }); + + // Build model config with optional tools + const modelConfig: any = { model: GEMINI_MODELS[model] }; + if (useTools) { + const { CANVAS_TOOL_DECLARATIONS } = await import("../lib/canvas-tools"); + modelConfig.tools = [{ functionDeclarations: CANVAS_TOOL_DECLARATIONS }]; + modelConfig.systemInstruction = systemPrompt || CANVAS_TOOLS_SYSTEM_PROMPT; + } + const geminiModel = genAI.getGenerativeModel(modelConfig); // Convert chat messages to Gemini contents format (with optional images) const contents = messages.map((m: { role: string; content: string; images?: string[] }) => { @@ -1646,9 +1659,54 @@ app.post("/api/prompt", async (c) => { }); try { - const result = await geminiModel.generateContent({ contents }); - const text = result.response.text(); - return c.json({ content: text }); + if (!useTools) { + const result = await geminiModel.generateContent({ contents }); + const text = result.response.text(); + return c.json({ content: text }); + } + + // Multi-turn tool loop (max 5 rounds) + const toolCalls: { name: string; args: Record; label: string }[] = []; + const { findTool } = await import("../lib/canvas-tools"); + let loopContents = [...contents]; + + for (let turn = 0; turn < 5; turn++) { + const result = await geminiModel.generateContent({ contents: loopContents }); + const response = result.response; + const candidate = response.candidates?.[0]; + if (!candidate) break; + + const parts = candidate.content?.parts || []; + const fnCalls = parts.filter((p: any) => p.functionCall); + + if (fnCalls.length === 0) { + // No more tool calls — extract final text + const text = response.text(); + return c.json({ content: text, toolCalls }); + } + + // Record tool calls and build function responses + const fnResponseParts: any[] = []; + for (const part of fnCalls) { + const fc = part.functionCall; + const tool = findTool(fc.name); + const label = tool?.actionLabel(fc.args) || fc.name; + toolCalls.push({ name: fc.name, args: fc.args, label }); + fnResponseParts.push({ + functionResponse: { + name: fc.name, + response: { success: true, message: `${label} — shape will be created on the canvas.` }, + }, + }); + } + + // Append model turn + function responses for next iteration + loopContents.push({ role: "model", parts }); + loopContents.push({ role: "user", parts: fnResponseParts }); + } + + // Exhausted loop — return what we have + return c.json({ content: "I've set up the requested items on your canvas.", toolCalls }); } catch (e: any) { console.error("[prompt] Gemini error:", e.message); return c.json({ error: "Gemini request failed" }, 502); @@ -1985,7 +2043,7 @@ app.post("/api/spaces/auto-provision", async (c) => { let claims: EncryptIDClaims; try { - claims = await verifyEncryptIDToken(token); + claims = await verifyToken(token); } catch { return c.json({ error: "Invalid or expired token" }, 401); } @@ -2111,7 +2169,7 @@ for (const mod of getAllModules()) { return c.json({ error: "Authentication required" }, 401); } let claims: EncryptIDClaims | null = null; - try { claims = await verifyEncryptIDToken(token); } catch {} + try { claims = await verifyToken(token); } catch {} if (!claims) { return c.json({ error: "Authentication required" }, 401); } @@ -2135,7 +2193,7 @@ for (const mod of getAllModules()) { const token = extractToken(c.req.raw.headers); let claims: EncryptIDClaims | null = null; if (token) { - try { claims = await verifyEncryptIDToken(token); } catch {} + try { claims = await verifyToken(token); } catch {} } const resolved = await resolveCallerRole(space, claims); if (resolved) { @@ -2242,7 +2300,7 @@ app.get("/admin-data", async (c) => { const token = extractToken(c.req.raw.headers); if (!token) return c.json({ error: "Authentication required" }, 401); let claims: EncryptIDClaims; - try { claims = await verifyEncryptIDToken(token); } catch { return c.json({ error: "Invalid token" }, 401); } + try { claims = await verifyToken(token); } catch { return c.json({ error: "Invalid token" }, 401); } if (!isAdminDID(claims.sub)) return c.json({ error: "Admin access required" }, 403); const STORAGE_DIR = process.env.STORAGE_DIR || "./data/communities"; @@ -2309,7 +2367,7 @@ app.post("/admin-action", async (c) => { const token = extractToken(c.req.raw.headers); if (!token) return c.json({ error: "Authentication required" }, 401); let claims: EncryptIDClaims; - try { claims = await verifyEncryptIDToken(token); } catch { return c.json({ error: "Invalid token" }, 401); } + try { claims = await verifyToken(token); } catch { return c.json({ error: "Invalid token" }, 401); } if (!isAdminDID(claims.sub)) return c.json({ error: "Admin access required" }, 403); const body = await c.req.json(); @@ -2721,7 +2779,7 @@ const server = Bun.serve({ if (jwtUsername === subdomain && !provisioningInProgress.has(subdomain)) { provisioningInProgress.add(subdomain); try { - const claims = await verifyEncryptIDToken(token); + const claims = await verifyToken(token); const username = claims.username?.toLowerCase(); if (username === subdomain && !(await communityExists(subdomain))) { await createSpace({