feat(prompt): add canvas tool use via Gemini function calling

folk-prompt can now spawn shapes on the canvas when Tools mode is
enabled. Gemini calls functions (create_map, create_note, create_embed,
create_image, create_bookmark, create_image_gen) and the client
executes them via window.__canvasApi. Multi-turn loop on server (max 5
rounds) with synthetic success responses. Extensible via registerCanvasTool().

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jeff Emmett 2026-03-22 16:42:08 -07:00
parent 7618433498
commit 999502464f
3 changed files with 351 additions and 33 deletions

156
lib/canvas-tools.ts Normal file
View File

@ -0,0 +1,156 @@
/**
* Canvas Tool Registry shared by server (Gemini function declarations) and client (shape spawning).
* Pure TypeScript, no DOM or server dependencies.
*/
export interface CanvasToolDefinition {
declaration: {
name: string;
description: string;
parameters: {
type: "object";
properties: Record<string, { type: string; description: string; enum?: string[] }>;
required: string[];
};
};
tagName: string;
buildProps: (args: Record<string, any>) => Record<string, any>;
actionLabel: (args: Record<string, any>) => string;
}
const registry: CanvasToolDefinition[] = [
{
declaration: {
name: "create_map",
description: "Create an interactive map centered on a location. Use when the user wants to see a place, get directions, or explore a geographic area.",
parameters: {
type: "object",
properties: {
latitude: { type: "number", description: "Latitude of the center point" },
longitude: { type: "number", description: "Longitude of the center point" },
zoom: { type: "number", description: "Zoom level (1-18, default 12)" },
location_name: { type: "string", description: "Human-readable name of the location" },
},
required: ["latitude", "longitude", "location_name"],
},
},
tagName: "folk-map",
buildProps: (args) => ({
center: [args.longitude, args.latitude],
zoom: args.zoom || 12,
}),
actionLabel: (args) => `Created map: ${args.location_name}`,
},
{
declaration: {
name: "create_note",
description: "Create a markdown note on the canvas. Use for text content, lists, summaries, instructions, or any written information.",
parameters: {
type: "object",
properties: {
content: { type: "string", description: "Markdown content for the note" },
title: { type: "string", description: "Optional title for the note" },
},
required: ["content"],
},
},
tagName: "folk-markdown",
buildProps: (args) => ({
value: args.title ? `# ${args.title}\n\n${args.content}` : args.content,
}),
actionLabel: (args) => `Created note${args.title ? `: ${args.title}` : ""}`,
},
{
declaration: {
name: "create_embed",
description: "Embed a webpage or web app on the canvas. Use for websites, search results, booking sites, videos, or any URL the user wants to view inline.",
parameters: {
type: "object",
properties: {
url: { type: "string", description: "The URL to embed" },
title: { type: "string", description: "Descriptive title for the embed" },
},
required: ["url"],
},
},
tagName: "folk-embed",
buildProps: (args) => ({
url: args.url,
}),
actionLabel: (args) => `Embedded: ${args.title || args.url}`,
},
{
declaration: {
name: "create_image",
description: "Display an image on the canvas from a URL. Use when showing an existing image, photo, diagram, or any direct image link.",
parameters: {
type: "object",
properties: {
src: { type: "string", description: "Image URL" },
alt: { type: "string", description: "Alt text describing the image" },
},
required: ["src"],
},
},
tagName: "folk-image",
buildProps: (args) => ({
src: args.src,
alt: args.alt || "",
}),
actionLabel: (args) => `Created image${args.alt ? `: ${args.alt}` : ""}`,
},
{
declaration: {
name: "create_bookmark",
description: "Create a bookmark card for a URL. Use when the user wants to save or reference a link without embedding the full page.",
parameters: {
type: "object",
properties: {
url: { type: "string", description: "The URL to bookmark" },
},
required: ["url"],
},
},
tagName: "folk-bookmark",
buildProps: (args) => ({
url: args.url,
}),
actionLabel: (args) => `Bookmarked: ${args.url}`,
},
{
declaration: {
name: "create_image_gen",
description: "Generate an AI image from a text prompt. Use when the user wants to create, generate, or imagine a new image that doesn't exist yet.",
parameters: {
type: "object",
properties: {
prompt: { type: "string", description: "Text prompt describing the image to generate" },
style: {
type: "string",
description: "Visual style for the generated image",
enum: ["photorealistic", "illustration", "painting", "sketch", "punk-zine", "collage", "vintage", "minimalist"],
},
},
required: ["prompt"],
},
},
tagName: "folk-image-gen",
buildProps: (args) => ({
prompt: args.prompt,
style: args.style || "photorealistic",
}),
actionLabel: (args) => `Generating image: ${args.prompt.slice(0, 50)}${args.prompt.length > 50 ? "..." : ""}`,
},
];
export const CANVAS_TOOLS: CanvasToolDefinition[] = [...registry];
export const CANVAS_TOOL_DECLARATIONS = CANVAS_TOOLS.map((t) => t.declaration);
export function findTool(name: string): CanvasToolDefinition | undefined {
return CANVAS_TOOLS.find((t) => t.declaration.name === name);
}
export function registerCanvasTool(def: CanvasToolDefinition): void {
CANVAS_TOOLS.push(def);
}

View File

@ -1,6 +1,7 @@
import { FolkShape } from "./folk-shape";
import { css, html } from "./tags";
import { SpeechDictation } from "./speech-dictation";
import { findTool } from "./canvas-tools";
const styles = css`
:host {
@ -304,11 +305,50 @@ const styles = css`
code {
font-family: "Monaco", "Consolas", monospace;
}
.input-controls {
display: flex;
gap: 6px;
align-items: center;
}
.tools-btn {
padding: 5px 10px;
border: 2px solid var(--rs-input-border, #e2e8f0);
border-radius: 6px;
font-size: 12px;
background: var(--rs-input-bg, #fff);
color: var(--rs-text-muted, #94a3b8);
cursor: pointer;
transition: all 0.2s;
white-space: nowrap;
}
.tools-btn:hover {
border-color: #6366f1;
}
.tools-btn.active {
background: #6366f1;
border-color: #6366f1;
color: white;
}
.message.tool-action {
align-self: center;
background: #ecfdf5;
color: #065f46;
border-radius: 20px;
padding: 6px 14px;
font-size: 12px;
font-weight: 500;
max-width: 90%;
}
`;
export interface ChatMessage {
id: string;
role: "user" | "assistant";
role: "user" | "assistant" | "tool-action";
content: string;
images?: string[];
timestamp: Date;
@ -345,6 +385,7 @@ export class FolkPrompt extends FolkShape {
#error: string | null = null;
#model = "gemini-flash";
#pendingImages: string[] = [];
#toolsEnabled = false;
#messagesEl: HTMLElement | null = null;
#promptInput: HTMLTextAreaElement | null = null;
@ -352,6 +393,7 @@ export class FolkPrompt extends FolkShape {
#sendBtn: HTMLButtonElement | null = null;
#attachInput: HTMLInputElement | null = null;
#pendingImagesEl: HTMLElement | null = null;
#toolsBtn: HTMLButtonElement | null = null;
get messages() {
return this.#messages;
@ -381,18 +423,21 @@ export class FolkPrompt extends FolkShape {
</div>
</div>
<div class="input-area">
<select class="model-select">
<optgroup label="Gemini">
<option value="gemini-flash">Gemini 2.5 Flash</option>
<option value="gemini-pro">Gemini 2.5 Pro</option>
</optgroup>
<optgroup label="Local (Ollama)">
<option value="llama3.2">Llama 3.2 (3B)</option>
<option value="llama3.1">Llama 3.1 (8B)</option>
<option value="qwen2.5-coder">Qwen Coder (7B)</option>
<option value="mistral-small">Mistral Small (24B)</option>
</optgroup>
</select>
<div class="input-controls">
<select class="model-select">
<optgroup label="Gemini">
<option value="gemini-flash">Gemini 2.5 Flash</option>
<option value="gemini-pro">Gemini 2.5 Pro</option>
</optgroup>
<optgroup label="Local (Ollama)">
<option value="llama3.2">Llama 3.2 (3B)</option>
<option value="llama3.1">Llama 3.1 (8B)</option>
<option value="qwen2.5-coder">Qwen Coder (7B)</option>
<option value="mistral-small">Mistral Small (24B)</option>
</optgroup>
</select>
<button class="tools-btn" title="Enable canvas tools — AI can create maps, notes, embeds, and more">Tools</button>
</div>
<div class="pending-images"></div>
<div class="prompt-row">
<textarea class="prompt-input" placeholder="Type your message..." rows="2"></textarea>
@ -418,10 +463,23 @@ export class FolkPrompt extends FolkShape {
this.#sendBtn = wrapper.querySelector(".send-btn");
this.#attachInput = wrapper.querySelector(".attach-input");
this.#pendingImagesEl = wrapper.querySelector(".pending-images");
this.#toolsBtn = wrapper.querySelector(".tools-btn") as HTMLButtonElement;
const clearBtn = wrapper.querySelector(".clear-btn") as HTMLButtonElement;
const closeBtn = wrapper.querySelector(".close-btn") as HTMLButtonElement;
const attachBtn = wrapper.querySelector(".attach-btn") as HTMLButtonElement | null;
// Tools toggle
this.#toolsBtn?.addEventListener("click", (e) => {
e.stopPropagation();
this.#toolsEnabled = !this.#toolsEnabled;
this.#toolsBtn!.classList.toggle("active", this.#toolsEnabled);
if (this.#promptInput) {
this.#promptInput.placeholder = this.#toolsEnabled
? "Ask me to create maps, notes, images, or embeds on the canvas..."
: "Type your message...";
}
});
// Attach button
attachBtn?.addEventListener("click", (e) => {
e.stopPropagation();
@ -566,16 +624,18 @@ export class FolkPrompt extends FolkShape {
this.#renderMessages(true);
try {
const useTools = this.#toolsEnabled && this.#model.startsWith("gemini");
const response = await fetch("/api/prompt", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
messages: this.#messages.map((m) => ({
messages: this.#messages.filter((m) => m.role !== "tool-action").map((m) => ({
role: m.role,
content: m.content,
...(m.images?.length ? { images: m.images } : {}),
})),
model: this.#model,
...(useTools ? { useTools: true } : {}),
}),
});
@ -585,6 +645,11 @@ export class FolkPrompt extends FolkShape {
const result = await response.json();
// Execute tool calls if any
if (result.toolCalls?.length) {
this.#executeToolCalls(result.toolCalls);
}
const assistantMessage: ChatMessage = {
id: crypto.randomUUID(),
role: "assistant",
@ -607,6 +672,42 @@ export class FolkPrompt extends FolkShape {
}
}
#executeToolCalls(toolCalls: { name: string; args: Record<string, any>; label: string }[]) {
const api = (window as any).__canvasApi;
if (!api) {
console.warn("[folk-prompt] Canvas API not available — cannot spawn shapes");
return;
}
let offsetY = 0;
for (const tc of toolCalls) {
const tool = findTool(tc.name);
if (!tool) continue;
const props = tool.buildProps(tc.args);
const defaults = api.SHAPE_DEFAULTS?.[tool.tagName] || { width: 300, height: 200 };
const preferX = this.x + this.width + 40 + defaults.width / 2;
const preferY = this.y + offsetY + defaults.height / 2;
const pos = api.findFreePosition(defaults.width, defaults.height, preferX, preferY, this);
try {
api.newShape(tool.tagName, props, { x: pos.x + defaults.width / 2, y: pos.y + defaults.height / 2 });
} catch (e) {
console.error(`[folk-prompt] Failed to create ${tool.tagName}:`, e);
}
// Tool action card in chat
this.#messages.push({
id: crypto.randomUUID(),
role: "tool-action",
content: tc.label,
timestamp: new Date(),
});
offsetY += defaults.height + 20;
}
}
#clearChat() {
this.#messages = [];
this.#error = null;
@ -629,6 +730,9 @@ export class FolkPrompt extends FolkShape {
let messagesHtml = this.#messages
.map((msg) => {
if (msg.role === "tool-action") {
return `<div class="message tool-action">${this.#escapeHtml(msg.content)}</div>`;
}
let imgHtml = "";
if (msg.images?.length) {
imgHtml = `<div class="msg-images">${msg.images.map((src) => `<img src="${this.#escapeHtml(src)}" />`).join("")}</div>`;
@ -684,7 +788,7 @@ export class FolkPrompt extends FolkShape {
...super.toJSON(),
type: "folk-prompt",
model: this.#model,
messages: this.messages.map((msg) => ({
messages: this.messages.filter((m) => m.role !== "tool-action").map((msg) => ({
role: msg.role,
content: msg.content,
...(msg.images?.length ? { images: msg.images } : {}),

View File

@ -35,12 +35,12 @@ import { seedTemplateShapes } from "./seed-template";
// Campaign demo moved to rsocials module — see modules/rsocials/campaign-data.ts
import type { SpaceVisibility } from "./community-store";
import {
verifyEncryptIDToken,
evaluateSpaceAccess,
extractToken,
authenticateWSUpgrade,
} from "@encryptid/sdk/server";
import type { EncryptIDClaims, SpaceAuthConfig } from "@encryptid/sdk/server";
import type { SpaceAuthConfig } from "@encryptid/sdk/server";
import { verifyToken, extractToken } from "./auth";
import type { EncryptIDClaims } from "./auth";
// ── Module system ──
import { registerModule, getAllModules, getModuleInfoList, getModule } from "../shared/module";
@ -518,7 +518,7 @@ app.post("/api/communities", async (c) => {
let claims: EncryptIDClaims;
try {
claims = await verifyEncryptIDToken(token);
claims = await verifyToken(token);
} catch {
return c.json({ error: "Invalid or expired authentication token" }, 401);
}
@ -653,7 +653,7 @@ app.get("/api/space-access/:slug", async (c) => {
const token = extractToken(c.req.raw.headers);
if (!token) return c.json({ access: false, reason: "not-authenticated" });
let claims: EncryptIDClaims | null = null;
try { claims = await verifyEncryptIDToken(token); } catch {}
try { claims = await verifyToken(token); } catch {}
if (!claims) return c.json({ access: false, reason: "not-authenticated" });
const config = await getSpaceConfig(slug);
@ -711,7 +711,7 @@ import { getBalance, getTokenDoc, transferTokens, mintFromOnChain } from "./toke
// Wire EncryptID JWT verifier into CRDT scheme
setTokenVerifier(async (token: string) => {
const claims = await verifyEncryptIDToken(token);
const claims = await verifyToken(token);
return { sub: claims.sub, did: claims.did as string | undefined, username: claims.username };
});
@ -735,7 +735,7 @@ const x402Test = setupX402FromEnv({
console.warn("[x402 bridge] No JWT — skipping cUSDC mint (on-chain payment still valid)");
return;
}
const claims = await verifyEncryptIDToken(token);
const claims = await verifyToken(token);
const did = (claims.did as string) || `did:key:${(claims.sub as string).slice(0, 32)}`;
const label = claims.username || did;
const amount = process.env.X402_UPLOAD_PRICE || "0.01";
@ -1619,8 +1619,13 @@ const OLLAMA_MODELS: Record<string, string> = {
"mistral-small": "mistral-small:24b",
};
const CANVAS_TOOLS_SYSTEM_PROMPT = `You are a helpful AI assistant in rSpace, a collaborative canvas workspace.
When the user asks to create, show, display, or visualize something, use the available tools to spawn shapes on the canvas.
After creating shapes, give a brief summary of what you placed. Only create shapes directly relevant to the request.
For text-only questions (explanations, coding help, math), respond with text don't create shapes unless asked.`;
app.post("/api/prompt", async (c) => {
const { messages, model = "gemini-flash" } = await c.req.json();
const { messages, model = "gemini-flash", useTools = false, systemPrompt } = await c.req.json();
if (!messages?.length) return c.json({ error: "messages required" }, 400);
// Determine provider
@ -1629,7 +1634,15 @@ app.post("/api/prompt", async (c) => {
const { GoogleGenerativeAI } = await import("@google/generative-ai");
const genAI = new GoogleGenerativeAI(GEMINI_API_KEY);
const geminiModel = genAI.getGenerativeModel({ model: GEMINI_MODELS[model] });
// Build model config with optional tools
const modelConfig: any = { model: GEMINI_MODELS[model] };
if (useTools) {
const { CANVAS_TOOL_DECLARATIONS } = await import("../lib/canvas-tools");
modelConfig.tools = [{ functionDeclarations: CANVAS_TOOL_DECLARATIONS }];
modelConfig.systemInstruction = systemPrompt || CANVAS_TOOLS_SYSTEM_PROMPT;
}
const geminiModel = genAI.getGenerativeModel(modelConfig);
// Convert chat messages to Gemini contents format (with optional images)
const contents = messages.map((m: { role: string; content: string; images?: string[] }) => {
@ -1646,9 +1659,54 @@ app.post("/api/prompt", async (c) => {
});
try {
const result = await geminiModel.generateContent({ contents });
const text = result.response.text();
return c.json({ content: text });
if (!useTools) {
const result = await geminiModel.generateContent({ contents });
const text = result.response.text();
return c.json({ content: text });
}
// Multi-turn tool loop (max 5 rounds)
const toolCalls: { name: string; args: Record<string, any>; label: string }[] = [];
const { findTool } = await import("../lib/canvas-tools");
let loopContents = [...contents];
for (let turn = 0; turn < 5; turn++) {
const result = await geminiModel.generateContent({ contents: loopContents });
const response = result.response;
const candidate = response.candidates?.[0];
if (!candidate) break;
const parts = candidate.content?.parts || [];
const fnCalls = parts.filter((p: any) => p.functionCall);
if (fnCalls.length === 0) {
// No more tool calls — extract final text
const text = response.text();
return c.json({ content: text, toolCalls });
}
// Record tool calls and build function responses
const fnResponseParts: any[] = [];
for (const part of fnCalls) {
const fc = part.functionCall;
const tool = findTool(fc.name);
const label = tool?.actionLabel(fc.args) || fc.name;
toolCalls.push({ name: fc.name, args: fc.args, label });
fnResponseParts.push({
functionResponse: {
name: fc.name,
response: { success: true, message: `${label} — shape will be created on the canvas.` },
},
});
}
// Append model turn + function responses for next iteration
loopContents.push({ role: "model", parts });
loopContents.push({ role: "user", parts: fnResponseParts });
}
// Exhausted loop — return what we have
return c.json({ content: "I've set up the requested items on your canvas.", toolCalls });
} catch (e: any) {
console.error("[prompt] Gemini error:", e.message);
return c.json({ error: "Gemini request failed" }, 502);
@ -1985,7 +2043,7 @@ app.post("/api/spaces/auto-provision", async (c) => {
let claims: EncryptIDClaims;
try {
claims = await verifyEncryptIDToken(token);
claims = await verifyToken(token);
} catch {
return c.json({ error: "Invalid or expired token" }, 401);
}
@ -2111,7 +2169,7 @@ for (const mod of getAllModules()) {
return c.json({ error: "Authentication required" }, 401);
}
let claims: EncryptIDClaims | null = null;
try { claims = await verifyEncryptIDToken(token); } catch {}
try { claims = await verifyToken(token); } catch {}
if (!claims) {
return c.json({ error: "Authentication required" }, 401);
}
@ -2135,7 +2193,7 @@ for (const mod of getAllModules()) {
const token = extractToken(c.req.raw.headers);
let claims: EncryptIDClaims | null = null;
if (token) {
try { claims = await verifyEncryptIDToken(token); } catch {}
try { claims = await verifyToken(token); } catch {}
}
const resolved = await resolveCallerRole(space, claims);
if (resolved) {
@ -2242,7 +2300,7 @@ app.get("/admin-data", async (c) => {
const token = extractToken(c.req.raw.headers);
if (!token) return c.json({ error: "Authentication required" }, 401);
let claims: EncryptIDClaims;
try { claims = await verifyEncryptIDToken(token); } catch { return c.json({ error: "Invalid token" }, 401); }
try { claims = await verifyToken(token); } catch { return c.json({ error: "Invalid token" }, 401); }
if (!isAdminDID(claims.sub)) return c.json({ error: "Admin access required" }, 403);
const STORAGE_DIR = process.env.STORAGE_DIR || "./data/communities";
@ -2309,7 +2367,7 @@ app.post("/admin-action", async (c) => {
const token = extractToken(c.req.raw.headers);
if (!token) return c.json({ error: "Authentication required" }, 401);
let claims: EncryptIDClaims;
try { claims = await verifyEncryptIDToken(token); } catch { return c.json({ error: "Invalid token" }, 401); }
try { claims = await verifyToken(token); } catch { return c.json({ error: "Invalid token" }, 401); }
if (!isAdminDID(claims.sub)) return c.json({ error: "Admin access required" }, 403);
const body = await c.req.json();
@ -2721,7 +2779,7 @@ const server = Bun.serve<WSData>({
if (jwtUsername === subdomain && !provisioningInProgress.has(subdomain)) {
provisioningInProgress.add(subdomain);
try {
const claims = await verifyEncryptIDToken(token);
const claims = await verifyToken(token);
const username = claims.username?.toLowerCase();
if (username === subdomain && !(await communityExists(subdomain))) {
await createSpace({