diff --git a/lib/index.ts b/lib/index.ts index ac2d22e..f56c0c8 100644 --- a/lib/index.ts +++ b/lib/index.ts @@ -85,3 +85,10 @@ export * from "./presence"; // Offline support export * from "./offline-store"; + +// MI (Mycelial Intelligence) Canvas Integration +export * from "./mi-canvas-bridge"; +export * from "./mi-actions"; +export * from "./mi-action-executor"; +export * from "./mi-selection-transforms"; +export * from "./mi-tool-schema"; diff --git a/lib/mi-action-executor.ts b/lib/mi-action-executor.ts new file mode 100644 index 0000000..e416fa9 --- /dev/null +++ b/lib/mi-action-executor.ts @@ -0,0 +1,157 @@ +/** + * MiActionExecutor — Executes parsed MI actions against the live canvas. + * + * Relies on `window.__canvasApi` exposed by canvas.html, which provides: + * { newShape, findFreePosition, SHAPE_DEFAULTS, setupShapeEventListeners, sync, canvasContent } + */ + +import type { MiAction } from "./mi-actions"; + +export interface ExecutionResult { + action: MiAction; + ok: boolean; + shapeId?: string; + error?: string; +} + +interface CanvasApi { + newShape: (tagName: string, props?: Record) => any; + findFreePosition: (width: number, height: number) => { x: number; y: number }; + SHAPE_DEFAULTS: Record; + setupShapeEventListeners: (shape: any) => void; + sync: any; + canvasContent: HTMLElement; +} + +function getCanvasApi(): CanvasApi | null { + return (window as any).__canvasApi || null; +} + +/** Resolve `$N` backreferences in an id string. */ +function resolveRef(id: string, refMap: Map): string { + if (id.startsWith("$")) { + return refMap.get(id) || id; + } + return id; +} + +export class MiActionExecutor { + static instance: MiActionExecutor | null = null; + + constructor() { + if (MiActionExecutor.instance) return MiActionExecutor.instance; + MiActionExecutor.instance = this; + } + + execute(actions: MiAction[]): ExecutionResult[] { + const api = getCanvasApi(); + if (!api) { + return actions.map((a) => ({ action: a, ok: false, error: "Canvas API not available" })); + } + + const results: ExecutionResult[] = []; + const refMap = new Map(); + + for (const action of actions) { + try { + const result = this.#executeOne(action, api, refMap); + results.push(result); + } catch (e: any) { + results.push({ action, ok: false, error: e.message }); + } + } + return results; + } + + #executeOne(action: MiAction, api: CanvasApi, refMap: Map): ExecutionResult { + switch (action.type) { + case "create-shape": { + const shape = api.newShape(action.tagName, action.props || {}); + if (!shape) { + return { action, ok: false, error: `Failed to create ${action.tagName}` }; + } + if (action.ref) { + refMap.set(action.ref, shape.id); + } + return { action, ok: true, shapeId: shape.id }; + } + + case "update-shape": { + const el = api.canvasContent.querySelector(`#${CSS.escape(action.shapeId)}`); + if (!el) { + return { action, ok: false, error: `Shape ${action.shapeId} not found` }; + } + for (const [key, value] of Object.entries(action.fields)) { + (el as any)[key] = value; + } + api.sync.updateShape?.(action.shapeId); + return { action, ok: true, shapeId: action.shapeId }; + } + + case "delete-shape": { + const el = api.canvasContent.querySelector(`#${CSS.escape(action.shapeId)}`); + if (!el) { + return { action, ok: false, error: `Shape ${action.shapeId} not found` }; + } + api.sync.deleteShape(action.shapeId); + el.remove(); + return { action, ok: true, shapeId: action.shapeId }; + } + + case "connect": { + const sourceId = resolveRef(action.sourceId, refMap); + const targetId = resolveRef(action.targetId, refMap); + + const arrow = document.createElement("folk-arrow"); + const arrowId = `arrow-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`; + arrow.id = arrowId; + (arrow as any).sourceId = sourceId; + (arrow as any).targetId = targetId; + if (action.color) (arrow as any).color = action.color; + + api.canvasContent.appendChild(arrow); + api.sync.registerShape(arrow); + return { action, ok: true, shapeId: arrowId }; + } + + case "move-shape": { + const el = api.canvasContent.querySelector(`#${CSS.escape(action.shapeId)}`); + if (!el) { + return { action, ok: false, error: `Shape ${action.shapeId} not found` }; + } + (el as any).x = action.x; + (el as any).y = action.y; + api.sync.updateShape?.(action.shapeId); + return { action, ok: true, shapeId: action.shapeId }; + } + + case "transform": { + // Delegate to mi-selection-transforms if available + const transforms = (window as any).__miSelectionTransforms; + if (transforms && transforms[action.transform]) { + const shapeEls = action.shapeIds + .map((id) => api.canvasContent.querySelector(`#${CSS.escape(id)}`)) + .filter(Boolean) as HTMLElement[]; + if (shapeEls.length === 0) { + return { action, ok: false, error: "No matching shapes found" }; + } + transforms[action.transform](shapeEls); + // Persist positions + for (const el of shapeEls) { + api.sync.updateShape?.(el.id); + } + return { action, ok: true }; + } + return { action, ok: false, error: `Unknown transform: ${action.transform}` }; + } + + case "navigate": { + window.location.href = action.path; + return { action, ok: true }; + } + + default: + return { action, ok: false, error: `Unknown action type` }; + } + } +} diff --git a/lib/mi-actions.ts b/lib/mi-actions.ts new file mode 100644 index 0000000..e515df1 --- /dev/null +++ b/lib/mi-actions.ts @@ -0,0 +1,68 @@ +/** + * MI Action Protocol — types and parser for [MI_ACTION:{...}] markers + * embedded in LLM responses. + * + * The LLM outputs prose interleaved with action markers. The parser + * extracts actions and returns clean display text for the user. + */ + +export type MiAction = + | { type: "create-shape"; tagName: string; props: Record; ref?: string } + | { type: "update-shape"; shapeId: string; fields: Record } + | { type: "delete-shape"; shapeId: string } + | { type: "connect"; sourceId: string; targetId: string; color?: string } + | { type: "move-shape"; shapeId: string; x: number; y: number } + | { type: "navigate"; path: string } + | { + type: "transform"; + transform: string; + shapeIds: string[]; + }; + +export interface ParsedMiResponse { + displayText: string; + actions: MiAction[]; +} + +const ACTION_PATTERN = /\[MI_ACTION:([\s\S]*?)\]/g; + +/** + * Parse [MI_ACTION:{...}] markers from streamed text. + * Returns the clean display text (markers stripped) and an array of actions. + */ +export function parseMiActions(text: string): ParsedMiResponse { + const actions: MiAction[] = []; + const displayText = text.replace(ACTION_PATTERN, (_, json) => { + try { + const action = JSON.parse(json.trim()) as MiAction; + if (action && action.type) { + actions.push(action); + } + } catch { + // Malformed action — skip silently + } + return ""; + }); + + return { + displayText: displayText.replace(/\n{3,}/g, "\n\n").trim(), + actions, + }; +} + +/** Summarise executed actions for a confirmation chip. */ +export function summariseActions(actions: MiAction[]): string { + const counts: Record = {}; + for (const a of actions) { + counts[a.type] = (counts[a.type] || 0) + 1; + } + const parts: string[] = []; + if (counts["create-shape"]) parts.push(`Created ${counts["create-shape"]} shape(s)`); + if (counts["update-shape"]) parts.push(`Updated ${counts["update-shape"]} shape(s)`); + if (counts["delete-shape"]) parts.push(`Deleted ${counts["delete-shape"]} shape(s)`); + if (counts["connect"]) parts.push(`Connected ${counts["connect"]} pair(s)`); + if (counts["move-shape"]) parts.push(`Moved ${counts["move-shape"]} shape(s)`); + if (counts["transform"]) parts.push(`Applied ${counts["transform"]} transform(s)`); + if (counts["navigate"]) parts.push(`Navigating`); + return parts.join(", ") || ""; +} diff --git a/lib/mi-canvas-bridge.ts b/lib/mi-canvas-bridge.ts new file mode 100644 index 0000000..4961a83 --- /dev/null +++ b/lib/mi-canvas-bridge.ts @@ -0,0 +1,144 @@ +/** + * MiCanvasBridge — Singleton that connects canvas state to the MI assistant. + * + * Listens to selection, viewport, and connection changes on the canvas + * and exposes a structured context snapshot for the MI system prompt. + */ + +export interface ShapeInfo { + id: string; + type: string; + x: number; + y: number; + width: number; + height: number; + content?: string; + title?: string; +} + +export interface Connection { + arrowId: string; + sourceId: string; + targetId: string; +} + +export interface ShapeGroup { + shapeIds: string[]; +} + +export interface CanvasContext { + selectedShapes: ShapeInfo[]; + allShapes: ShapeInfo[]; + connections: Connection[]; + viewport: { x: number; y: number; scale: number }; + shapeGroups: ShapeGroup[]; + shapeCountByType: Record; +} + +export class MiCanvasBridge { + static instance: MiCanvasBridge | null = null; + + selectedShapeIds: string[] = []; + viewport = { x: 0, y: 0, scale: 1 }; + + private canvasContent: HTMLElement | null = null; + + constructor(canvasContent?: HTMLElement) { + if (MiCanvasBridge.instance) return MiCanvasBridge.instance; + this.canvasContent = canvasContent || document.getElementById("canvas-content"); + MiCanvasBridge.instance = this; + } + + setSelection(ids: string[]) { + this.selectedShapeIds = ids; + } + + setViewport(x: number, y: number, scale: number) { + this.viewport = { x, y, scale }; + } + + /** Build full context snapshot for the MI system prompt. */ + getCanvasContext(): CanvasContext { + const allShapes = this.#collectShapes(); + const connections = this.#collectConnections(); + const selectedShapes = allShapes.filter((s) => this.selectedShapeIds.includes(s.id)); + const shapeGroups = this.#buildShapeGroups(allShapes, connections); + const shapeCountByType: Record = {}; + for (const s of allShapes) { + shapeCountByType[s.type] = (shapeCountByType[s.type] || 0) + 1; + } + + return { + selectedShapes, + allShapes, + connections, + viewport: { ...this.viewport }, + shapeGroups, + shapeCountByType, + }; + } + + #collectShapes(): ShapeInfo[] { + if (!this.canvasContent) return []; + return [...this.canvasContent.children] + .filter( + (el) => + el.tagName?.includes("-") && + el.id && + !el.tagName.toLowerCase().includes("arrow"), + ) + .map((el: any) => ({ + id: el.id, + type: el.tagName.toLowerCase(), + x: el.x ?? 0, + y: el.y ?? 0, + width: el.width ?? 0, + height: el.height ?? 0, + ...(el.content ? { content: String(el.content).slice(0, 120) } : {}), + ...(el.title ? { title: String(el.title).slice(0, 80) } : {}), + })); + } + + #collectConnections(): Connection[] { + if (!this.canvasContent) return []; + return [...this.canvasContent.querySelectorAll("folk-arrow")] + .filter((el: any) => el.id && el.sourceId && el.targetId) + .map((el: any) => ({ + arrowId: el.id, + sourceId: el.sourceId, + targetId: el.targetId, + })); + } + + /** BFS on the arrow graph to find clusters of connected shapes. */ + #buildShapeGroups(shapes: ShapeInfo[], connections: Connection[]): ShapeGroup[] { + const adj = new Map>(); + for (const c of connections) { + if (!adj.has(c.sourceId)) adj.set(c.sourceId, new Set()); + if (!adj.has(c.targetId)) adj.set(c.targetId, new Set()); + adj.get(c.sourceId)!.add(c.targetId); + adj.get(c.targetId)!.add(c.sourceId); + } + + const visited = new Set(); + const groups: ShapeGroup[] = []; + + for (const shape of shapes) { + if (visited.has(shape.id) || !adj.has(shape.id)) continue; + const group: string[] = []; + const queue = [shape.id]; + while (queue.length) { + const id = queue.shift()!; + if (visited.has(id)) continue; + visited.add(id); + group.push(id); + for (const neighbor of adj.get(id) || []) { + if (!visited.has(neighbor)) queue.push(neighbor); + } + } + if (group.length > 1) groups.push({ shapeIds: group }); + } + + return groups; + } +} diff --git a/lib/mi-selection-transforms.ts b/lib/mi-selection-transforms.ts new file mode 100644 index 0000000..b16cd4d --- /dev/null +++ b/lib/mi-selection-transforms.ts @@ -0,0 +1,193 @@ +/** + * Selection Transforms — align, distribute, arrange, and match-size + * operations on canvas shape elements. + * + * Each function accepts an array of shape elements (with x, y, width, height) + * and mutates their positions in place. + * + * Exposed on `window.__miSelectionTransforms` so the MiActionExecutor can + * invoke them by name. + */ + +interface ShapeEl { + x: number; + y: number; + width: number; + height: number; +} + +// ── Align ── + +export function alignLeft(shapes: ShapeEl[]) { + if (shapes.length < 2) return; + const minX = Math.min(...shapes.map((s) => s.x)); + for (const s of shapes) s.x = minX; +} + +export function alignRight(shapes: ShapeEl[]) { + if (shapes.length < 2) return; + const maxRight = Math.max(...shapes.map((s) => s.x + s.width)); + for (const s of shapes) s.x = maxRight - s.width; +} + +export function alignCenterH(shapes: ShapeEl[]) { + if (shapes.length < 2) return; + const centers = shapes.map((s) => s.x + s.width / 2); + const avg = centers.reduce((a, b) => a + b, 0) / centers.length; + for (const s of shapes) s.x = avg - s.width / 2; +} + +export function alignTop(shapes: ShapeEl[]) { + if (shapes.length < 2) return; + const minY = Math.min(...shapes.map((s) => s.y)); + for (const s of shapes) s.y = minY; +} + +export function alignBottom(shapes: ShapeEl[]) { + if (shapes.length < 2) return; + const maxBottom = Math.max(...shapes.map((s) => s.y + s.height)); + for (const s of shapes) s.y = maxBottom - s.height; +} + +export function alignCenterV(shapes: ShapeEl[]) { + if (shapes.length < 2) return; + const centers = shapes.map((s) => s.y + s.height / 2); + const avg = centers.reduce((a, b) => a + b, 0) / centers.length; + for (const s of shapes) s.y = avg - s.height / 2; +} + +// ── Distribute ── + +export function distributeH(shapes: ShapeEl[]) { + if (shapes.length < 3) return; + const sorted = [...shapes].sort((a, b) => a.x - b.x); + const first = sorted[0]; + const last = sorted[sorted.length - 1]; + const totalWidth = sorted.reduce((s, el) => s + el.width, 0); + const span = last.x + last.width - first.x; + const gap = (span - totalWidth) / (sorted.length - 1); + + let cursor = first.x; + for (const s of sorted) { + s.x = cursor; + cursor += s.width + gap; + } +} + +export function distributeV(shapes: ShapeEl[]) { + if (shapes.length < 3) return; + const sorted = [...shapes].sort((a, b) => a.y - b.y); + const first = sorted[0]; + const last = sorted[sorted.length - 1]; + const totalHeight = sorted.reduce((s, el) => s + el.height, 0); + const span = last.y + last.height - first.y; + const gap = (span - totalHeight) / (sorted.length - 1); + + let cursor = first.y; + for (const s of sorted) { + s.y = cursor; + cursor += s.height + gap; + } +} + +// ── Arrange ── + +export function arrangeRow(shapes: ShapeEl[]) { + if (shapes.length < 2) return; + const gap = 30; + const baseY = Math.min(...shapes.map((s) => s.y)); + let cursor = shapes[0].x; + for (const s of shapes) { + s.x = cursor; + s.y = baseY; + cursor += s.width + gap; + } +} + +export function arrangeColumn(shapes: ShapeEl[]) { + if (shapes.length < 2) return; + const gap = 30; + const baseX = Math.min(...shapes.map((s) => s.x)); + let cursor = shapes[0].y; + for (const s of shapes) { + s.x = baseX; + s.y = cursor; + cursor += s.height + gap; + } +} + +export function arrangeGrid(shapes: ShapeEl[]) { + if (shapes.length < 2) return; + const cols = Math.ceil(Math.sqrt(shapes.length)); + const gap = 30; + const baseX = Math.min(...shapes.map((s) => s.x)); + const baseY = Math.min(...shapes.map((s) => s.y)); + const maxW = Math.max(...shapes.map((s) => s.width)); + const maxH = Math.max(...shapes.map((s) => s.height)); + + for (let i = 0; i < shapes.length; i++) { + const col = i % cols; + const row = Math.floor(i / cols); + shapes[i].x = baseX + col * (maxW + gap); + shapes[i].y = baseY + row * (maxH + gap); + } +} + +export function arrangeCircle(shapes: ShapeEl[]) { + if (shapes.length < 2) return; + const cx = shapes.reduce((s, el) => s + el.x + el.width / 2, 0) / shapes.length; + const cy = shapes.reduce((s, el) => s + el.y + el.height / 2, 0) / shapes.length; + const maxDim = Math.max(...shapes.map((s) => Math.max(s.width, s.height))); + const radius = Math.max(maxDim * shapes.length / (2 * Math.PI), 150); + const step = (2 * Math.PI) / shapes.length; + + for (let i = 0; i < shapes.length; i++) { + const angle = step * i - Math.PI / 2; + shapes[i].x = cx + radius * Math.cos(angle) - shapes[i].width / 2; + shapes[i].y = cy + radius * Math.sin(angle) - shapes[i].height / 2; + } +} + +// ── Match Size ── + +export function matchWidth(shapes: ShapeEl[]) { + if (shapes.length < 2) return; + const maxW = Math.max(...shapes.map((s) => s.width)); + for (const s of shapes) s.width = maxW; +} + +export function matchHeight(shapes: ShapeEl[]) { + if (shapes.length < 2) return; + const maxH = Math.max(...shapes.map((s) => s.height)); + for (const s of shapes) s.height = maxH; +} + +export function matchSize(shapes: ShapeEl[]) { + matchWidth(shapes); + matchHeight(shapes); +} + +// ── Registry: kebab-name → function ── + +const TRANSFORM_MAP: Record void> = { + "align-left": alignLeft, + "align-right": alignRight, + "align-center-h": alignCenterH, + "align-top": alignTop, + "align-bottom": alignBottom, + "align-center-v": alignCenterV, + "distribute-h": distributeH, + "distribute-v": distributeV, + "arrange-row": arrangeRow, + "arrange-column": arrangeColumn, + "arrange-grid": arrangeGrid, + "arrange-circle": arrangeCircle, + "match-width": matchWidth, + "match-height": matchHeight, + "match-size": matchSize, +}; + +/** Install the transform map on window so the action executor can find them. */ +export function installSelectionTransforms() { + (window as any).__miSelectionTransforms = TRANSFORM_MAP; +} diff --git a/lib/mi-tool-schema.ts b/lib/mi-tool-schema.ts new file mode 100644 index 0000000..22aedc6 --- /dev/null +++ b/lib/mi-tool-schema.ts @@ -0,0 +1,59 @@ +/** + * MI Tool Schema — lightweight registry of canvas shape types with keyword + * matching, so MI can suggest relevant tools as clickable chips. + */ + +export interface ToolHint { + tagName: string; + label: string; + icon: string; + keywords: string[]; +} + +const TOOL_HINTS: ToolHint[] = [ + { tagName: "folk-markdown", label: "Note", icon: "📝", keywords: ["note", "text", "markdown", "write", "document"] }, + { tagName: "folk-wrapper", label: "Card", icon: "📋", keywords: ["card", "wrapper", "container", "group"] }, + { tagName: "folk-slide", label: "Slide", icon: "🖼️", keywords: ["slide", "presentation", "deck"] }, + { tagName: "folk-chat", label: "Chat", icon: "💬", keywords: ["chat", "message", "conversation", "talk"] }, + { tagName: "folk-embed", label: "Embed", icon: "🔗", keywords: ["embed", "iframe", "website", "url", "link"] }, + { tagName: "folk-calendar", label: "Calendar", icon: "📅", keywords: ["calendar", "date", "schedule", "event"] }, + { tagName: "folk-map", label: "Map", icon: "🗺️", keywords: ["map", "location", "place", "geo"] }, + { tagName: "folk-image-gen", label: "AI Image", icon: "🎨", keywords: ["image", "picture", "photo", "generate", "art", "draw"] }, + { tagName: "folk-video-gen", label: "AI Video", icon: "🎬", keywords: ["video", "clip", "animate", "movie", "film"] }, + { tagName: "folk-prompt", label: "AI Chat", icon: "🤖", keywords: ["ai", "prompt", "llm", "assistant", "gpt"] }, + { tagName: "folk-transcription", label: "Transcribe", icon: "🎙️", keywords: ["transcribe", "audio", "speech", "voice", "record"] }, + { tagName: "folk-video-chat", label: "Video Call", icon: "📹", keywords: ["video call", "webcam", "meeting"] }, + { tagName: "folk-obs-note", label: "Obsidian Note", icon: "📓", keywords: ["obsidian", "note", "vault"] }, + { tagName: "folk-workflow-block", label: "Workflow", icon: "⚙️", keywords: ["workflow", "automation", "block", "process"] }, + { tagName: "folk-social-post", label: "Social Post", icon: "📣", keywords: ["social", "post", "twitter", "instagram", "campaign"] }, + { tagName: "folk-splat", label: "3D Gaussian", icon: "💎", keywords: ["3d", "splat", "gaussian", "point cloud"] }, + { tagName: "folk-drawfast", label: "Drawing", icon: "✏️", keywords: ["draw", "sketch", "whiteboard", "pencil"] }, + { tagName: "folk-rapp", label: "rApp Embed", icon: "📦", keywords: ["rapp", "module", "embed", "app"] }, + { tagName: "folk-feed", label: "Feed", icon: "📡", keywords: ["feed", "data", "stream", "flow"] }, + { tagName: "folk-piano", label: "Piano", icon: "🎹", keywords: ["piano", "music", "instrument", "midi"] }, + { tagName: "folk-choice-vote", label: "Vote", icon: "🗳️", keywords: ["vote", "poll", "election", "choice"] }, + { tagName: "folk-choice-rank", label: "Ranking", icon: "📊", keywords: ["rank", "order", "priority", "sort"] }, + { tagName: "folk-choice-spider", label: "Spider Chart", icon: "🕸️", keywords: ["spider", "radar", "criteria", "evaluate"] }, +]; + +/** + * Given a user query, return matching tool hints (max 3). + * Matches if any keyword appears in the query (case-insensitive). + */ +export function suggestTools(query: string): ToolHint[] { + const q = query.toLowerCase(); + const scored: { hint: ToolHint; score: number }[] = []; + + for (const hint of TOOL_HINTS) { + let score = 0; + for (const kw of hint.keywords) { + if (q.includes(kw)) score += kw.length; // longer keyword match = higher relevance + } + if (score > 0) scored.push({ hint, score }); + } + + return scored + .sort((a, b) => b.score - a.score) + .slice(0, 3) + .map((s) => s.hint); +} diff --git a/server/index.ts b/server/index.ts index 0975a78..972a47e 100644 --- a/server/index.ts +++ b/server/index.ts @@ -148,11 +148,42 @@ app.post("/api/mi/ask", async (c) => { if (context.activeTab) contextSection += `\n- Active tab: ${context.activeTab}`; if (context.openShapes?.length) { const shapeSummary = context.openShapes - .slice(0, 10) - .map((s: any) => ` - ${s.type}${s.title ? `: ${s.title}` : ""}${s.snippet ? ` (${s.snippet})` : ""}`) + .slice(0, 15) + .map((s: any) => { + let desc = ` - ${s.type} (id: ${s.id})`; + if (s.title) desc += `: ${s.title}`; + if (s.snippet) desc += ` — "${s.snippet}"`; + if (s.x != null) desc += ` at (${s.x}, ${s.y})`; + return desc; + }) .join("\n"); contextSection += `\n- Open shapes on canvas:\n${shapeSummary}`; } + if (context.selectedShapes?.length) { + const selSummary = context.selectedShapes + .map((s: any) => ` - ${s.type} (id: ${s.id})${s.title ? `: ${s.title}` : ""}${s.snippet ? ` — "${s.snippet}"` : ""}`) + .join("\n"); + contextSection += `\n- The user currently has selected:\n${selSummary}`; + } + if (context.connections?.length) { + const connSummary = context.connections + .slice(0, 15) + .map((c: any) => ` - ${c.sourceId} → ${c.targetId}`) + .join("\n"); + contextSection += `\n- Connected shapes:\n${connSummary}`; + } + if (context.viewport) { + contextSection += `\n- Viewport: zoom ${context.viewport.scale?.toFixed?.(2) || context.viewport.scale}, pan (${Math.round(context.viewport.x)}, ${Math.round(context.viewport.y)})`; + } + if (context.shapeGroups?.length) { + contextSection += `\n- ${context.shapeGroups.length} group(s) of connected shapes`; + } + if (context.shapeCountByType && Object.keys(context.shapeCountByType).length) { + const typeCounts = Object.entries(context.shapeCountByType) + .map(([t, n]) => `${t}: ${n}`) + .join(", "); + contextSection += `\n- Shape types: ${typeCounts}`; + } const systemPrompt = `You are mi (mycelial intelligence), the intelligent assistant for rSpace — a self-hosted, community-run platform. You help users navigate, understand, and get the most out of the platform's apps (rApps). @@ -168,10 +199,32 @@ ${contextSection} - Be concise and helpful. Keep responses short (2-4 sentences) unless the user asks for detail. - When suggesting actions, reference specific rApps by name and explain how they connect. - You can suggest navigating to /:space/:moduleId paths. -- If the user has shapes open on their canvas, you can reference them and suggest connections. +- If the user has shapes open on their canvas, you can reference them by id and suggest connections. - Help with setup: guide users through creating spaces, adding content, configuring rApps. - If you don't know something specific about the user's data, say so honestly. -- Use a warm, knowledgeable tone. You're a mycelial guide, connecting knowledge across the platform.`; +- Use a warm, knowledgeable tone. You're a mycelial guide, connecting knowledge across the platform. + +## Actions +When the user asks you to create, modify, delete, connect, move, or arrange shapes on the canvas, +include action markers in your response. Each marker is on its own line: +[MI_ACTION:{"type":"create-shape","tagName":"folk-markdown","props":{"content":"# Hello"},"ref":"$1"}] +[MI_ACTION:{"type":"connect","sourceId":"$1","targetId":"shape-123"}] +[MI_ACTION:{"type":"update-shape","shapeId":"shape-123","fields":{"content":"Updated text"}}] +[MI_ACTION:{"type":"delete-shape","shapeId":"shape-123"}] +[MI_ACTION:{"type":"move-shape","shapeId":"shape-123","x":400,"y":200}] +[MI_ACTION:{"type":"navigate","path":"/myspace/canvas"}] + +Use "$1", "$2", etc. as ref values when creating shapes, then reference them in subsequent connect actions. +Available shape types: folk-markdown, folk-wrapper, folk-image-gen, folk-video-gen, folk-prompt, +folk-embed, folk-calendar, folk-map, folk-chat, folk-slide, folk-obs-note, folk-workflow-block, +folk-social-post, folk-splat, folk-drawfast, folk-rapp, folk-feed. + +## Transforms +When the user asks to align, distribute, or arrange selected shapes: +[MI_ACTION:{"type":"transform","transform":"align-left","shapeIds":["shape-1","shape-2"]}] +Available transforms: align-left, align-right, align-center-h, align-top, align-bottom, align-center-v, +distribute-h, distribute-v, arrange-row, arrange-column, arrange-grid, arrange-circle, +match-width, match-height, match-size.`; // Build conversation for Ollama const ollamaMessages = [ diff --git a/shared/components/rstack-mi.ts b/shared/components/rstack-mi.ts index 2459e4c..c80f1f3 100644 --- a/shared/components/rstack-mi.ts +++ b/shared/components/rstack-mi.ts @@ -7,10 +7,15 @@ */ import { getAccessToken } from "./rstack-identity"; +import { parseMiActions, summariseActions } from "../../lib/mi-actions"; +import { MiActionExecutor } from "../../lib/mi-action-executor"; +import { suggestTools, type ToolHint } from "../../lib/mi-tool-schema"; interface MiMessage { role: "user" | "assistant"; content: string; + actionSummary?: string; + toolHints?: ToolHint[]; } export class RStackMi extends HTMLElement { @@ -81,7 +86,7 @@ export class RStackMi extends HTMLElement { bar.addEventListener("click", (e) => e.stopPropagation()); } - /** Gather page context: open shapes, active module, tabs, etc. */ + /** Gather page context: open shapes, active module, tabs, canvas state. */ #gatherContext(): Record { const ctx: Record = {}; @@ -89,19 +94,49 @@ export class RStackMi extends HTMLElement { ctx.space = document.querySelector("rstack-space-switcher")?.getAttribute("current") || ""; ctx.module = document.querySelector("rstack-app-switcher")?.getAttribute("current") || ""; - // Open shapes on canvas (if any) - const canvasContent = document.getElementById("canvas-content"); - if (canvasContent) { - const shapes = [...canvasContent.children] - .filter((el) => el.tagName?.includes("-") && el.id) - .map((el: any) => ({ - type: el.tagName.toLowerCase(), - id: el.id, - ...(el.content ? { snippet: el.content.slice(0, 60) } : {}), - ...(el.title ? { title: el.title } : {}), - })) - .slice(0, 20); - if (shapes.length) ctx.openShapes = shapes; + // Deep canvas context from MI bridge (if available) + const bridge = (window as any).__miCanvasBridge; + if (bridge) { + const cc = bridge.getCanvasContext(); + ctx.openShapes = cc.allShapes.slice(0, 20).map((s: any) => ({ + type: s.type, + id: s.id, + x: Math.round(s.x), + y: Math.round(s.y), + width: Math.round(s.width), + height: Math.round(s.height), + ...(s.content ? { snippet: s.content.slice(0, 80) } : {}), + ...(s.title ? { title: s.title } : {}), + })); + if (cc.selectedShapes.length) { + ctx.selectedShapes = cc.selectedShapes.map((s: any) => ({ + type: s.type, + id: s.id, + x: Math.round(s.x), + y: Math.round(s.y), + ...(s.content ? { snippet: s.content.slice(0, 80) } : {}), + ...(s.title ? { title: s.title } : {}), + })); + } + if (cc.connections.length) ctx.connections = cc.connections; + ctx.viewport = cc.viewport; + if (cc.shapeGroups.length) ctx.shapeGroups = cc.shapeGroups; + ctx.shapeCountByType = cc.shapeCountByType; + } else { + // Fallback: basic shape list from DOM + const canvasContent = document.getElementById("canvas-content"); + if (canvasContent) { + const shapes = [...canvasContent.children] + .filter((el) => el.tagName?.includes("-") && el.id) + .map((el: any) => ({ + type: el.tagName.toLowerCase(), + id: el.id, + ...(el.content ? { snippet: el.content.slice(0, 60) } : {}), + ...(el.title ? { title: el.title } : {}), + })) + .slice(0, 20); + if (shapes.length) ctx.openShapes = shapes; + } } // Active tab/layer info @@ -189,6 +224,25 @@ export class RStackMi extends HTMLElement { // If still empty after stream, show fallback if (!this.#messages[assistantIdx].content) { this.#messages[assistantIdx].content = "I couldn't generate a response. Please try again."; + this.#renderMessages(messagesEl); + } else { + // Parse and execute MI actions from the response + const rawText = this.#messages[assistantIdx].content; + const { displayText, actions } = parseMiActions(rawText); + this.#messages[assistantIdx].content = displayText; + + if (actions.length) { + const executor = new MiActionExecutor(); + executor.execute(actions); + this.#messages[assistantIdx].actionSummary = summariseActions(actions); + } + + // Check for tool suggestions + const hints = suggestTools(query); + if (hints.length) { + this.#messages[assistantIdx].toolHints = hints; + } + this.#renderMessages(messagesEl); } } catch (e: any) { @@ -207,13 +261,30 @@ export class RStackMi extends HTMLElement {
${m.role === "user" ? "You" : "✧ mi"}
${m.content ? this.#formatContent(m.content) : ''}
+ ${m.actionSummary ? `
${this.#escapeHtml(m.actionSummary)}
` : ""} + ${m.toolHints?.length ? `
${m.toolHints.map((h) => ``).join("")}
` : ""}
`, ) .join(""); + + // Wire tool chip clicks + container.querySelectorAll(".mi-tool-chip").forEach((btn) => { + btn.addEventListener("click", () => { + const tag = btn.dataset.tag; + if (!tag) return; + const executor = new MiActionExecutor(); + executor.execute([{ type: "create-shape", tagName: tag, props: {} }]); + }); + }); + container.scrollTop = container.scrollHeight; } + #escapeHtml(s: string): string { + return s.replace(/&/g, "&").replace(//g, ">"); + } + #formatContent(s: string): string { // Escape HTML then convert markdown-like formatting return s @@ -327,6 +398,24 @@ const STYLES = ` 30% { transform: translateY(-4px); } } +.mi-action-chip { + display: inline-block; margin-top: 6px; padding: 3px 10px; + border-radius: 12px; font-size: 0.75rem; font-weight: 600; +} +:host-context([data-theme="dark"]) .mi-action-chip { background: rgba(6,182,212,0.15); color: #67e8f9; } +:host-context([data-theme="light"]) .mi-action-chip { background: rgba(6,182,212,0.1); color: #0891b2; } + +.mi-tool-chips { display: flex; flex-wrap: wrap; gap: 6px; margin-top: 8px; } +.mi-tool-chip { + padding: 4px 10px; border-radius: 8px; border: none; + font-size: 0.75rem; cursor: pointer; transition: background 0.15s; + font-family: inherit; +} +:host-context([data-theme="dark"]) .mi-tool-chip { background: rgba(255,255,255,0.08); color: #e2e8f0; } +:host-context([data-theme="light"]) .mi-tool-chip { background: rgba(0,0,0,0.05); color: #374151; } +:host-context([data-theme="dark"]) .mi-tool-chip:hover { background: rgba(255,255,255,0.15); } +:host-context([data-theme="light"]) .mi-tool-chip:hover { background: rgba(0,0,0,0.1); } + @media (max-width: 640px) { .mi { max-width: 200px; } .mi-panel { min-width: 300px; left: -60px; } diff --git a/shared/components/rstack-tab-bar.ts b/shared/components/rstack-tab-bar.ts index 83addb3..382dcf7 100644 --- a/shared/components/rstack-tab-bar.ts +++ b/shared/components/rstack-tab-bar.ts @@ -357,39 +357,40 @@ export class RStackTabBar extends HTMLElement { const isActive = layer.id === this.active; const containedFeeds = this.#getContainedFeeds(layer.id); - // Feed port indicators — output kinds (right side) and input kinds (left side) - const outputKinds = this.#getModuleOutputKinds(layer.moduleId); - const inputKinds = this.#getModuleInputKinds(layer.moduleId); + // Build I/O chip markup — output feeds (right) and input accepts (left) + const mod = this.#modules.find(m => m.id === layer.moduleId); + const outFeeds = mod?.feeds || []; + const inKinds = mod?.acceptsFeeds || []; + const containedSet = new Set(containedFeeds.map(f => f.id)); - const outPorts = [...outputKinds].map(k => - `` - ).join(""); - const inPorts = [...inputKinds].map(k => - `` + const outChips = outFeeds.map(f => { + const contained = containedSet.has(f.id); + return ` + ${f.name}${contained ? '\uD83D\uDD12' : ""} + `; + }).join(""); + + const inChips = inKinds.map(k => + ` + ${FLOW_LABELS[k]} + ` ).join(""); - const containedHtml = containedFeeds.length > 0 ? ` -
- ${containedFeeds.map(f => ` - - \uD83D\uDD12 - ${f.name} - - `).join("")} -
- ` : ""; + const hasIO = outFeeds.length > 0 || inKinds.length > 0; layersHtml += `
-
${inPorts}
${badge?.badge || layer.moduleId.slice(0, 2)} ${layer.label} -
${outPorts}
- ${containedHtml} + ${hasIO ? `
+
${inChips}
+
${outChips}
+
` : ""}
`; }); @@ -1204,7 +1205,7 @@ const STYLES = ` .layer-plane { position: absolute; width: 320px; - min-height: 70px; + min-height: 44px; border-radius: 10px; border: 1px solid var(--layer-color); padding: 10px 14px; @@ -1282,44 +1283,81 @@ const STYLES = ` white-space: nowrap; } -/* Feed port indicators */ -.layer-ports { +/* ── I/O chip system ── */ +.layer-io { display: flex; + justify-content: space-between; + gap: 6px; + margin-top: 5px; +} + +.io-col { + display: flex; + flex-direction: column; gap: 3px; + min-width: 0; +} +.io-col--in { align-items: flex-start; } +.io-col--out { align-items: flex-end; } + +.io-chip { + display: inline-flex; + align-items: center; + gap: 4px; + font-size: 0.55rem; + font-weight: 600; + padding: 2px 7px; + border-radius: 9px; + white-space: nowrap; + max-width: 140px; + overflow: hidden; + text-overflow: ellipsis; + transition: opacity 0.15s, box-shadow 0.15s; + cursor: default; +} + +.io-chip--out { + background: color-mix(in srgb, var(--chip-color) 18%, transparent); + border: 1px solid color-mix(in srgb, var(--chip-color) 40%, transparent); + color: var(--chip-color); +} + +.io-chip--in { + background: transparent; + border: 1px dashed color-mix(in srgb, var(--chip-color) 35%, transparent); + color: color-mix(in srgb, var(--chip-color) 70%, #e2e8f0); +} + +.io-chip--contained { + opacity: 0.5; +} + +.io-chip:hover { + opacity: 1; + box-shadow: 0 0 8px color-mix(in srgb, var(--chip-color) 30%, transparent); +} + +.io-dot { + width: 5px; + height: 5px; + border-radius: 50%; flex-shrink: 0; } -.feed-port { - width: 6px; - height: 6px; - border-radius: 50%; - opacity: 0.7; -} -.feed-port--in { box-shadow: inset 0 0 0 1.5px rgba(0,0,0,0.3); } -.feed-port--out { box-shadow: 0 0 3px currentColor; } - -/* Containment indicators */ -.layer-contained { - display: flex; - flex-wrap: wrap; - gap: 4px; - margin-top: 6px; +.io-chip--out .io-dot { + background: var(--chip-color); + box-shadow: 0 0 4px var(--chip-color); } -.contained-feed { - display: inline-flex; - align-items: center; - gap: 3px; - font-size: 0.6rem; - padding: 2px 6px; - border-radius: 4px; - background: color-mix(in srgb, var(--feed-color) 10%, transparent); - border: 1px solid color-mix(in srgb, var(--feed-color) 25%, transparent); - opacity: 0.7; +.io-chip--in .io-dot { + background: transparent; + box-shadow: inset 0 0 0 1.5px var(--chip-color); } -.contained-lock { - font-size: 0.55rem; +.io-lock { + font-size: 0.5rem; + margin-left: 2px; + opacity: 0.6; } /* ── Flow particles ── */ @@ -1605,7 +1643,8 @@ const STYLES = ` .stack-view { max-height: 40vh; } .stack-view-3d { height: 260px; } .stack-scene { width: 240px; } - .layer-plane { width: 240px; min-height: 56px; padding: 8px 10px; } + .layer-plane { width: 240px; min-height: 40px; padding: 8px 10px; } + .io-chip { font-size: 0.5rem; padding: 1px 5px; max-width: 100px; } .flow-dialog { width: 240px; } } `; diff --git a/website/canvas.html b/website/canvas.html index a956962..d9e02f8 100644 --- a/website/canvas.html +++ b/website/canvas.html @@ -803,7 +803,9 @@ CommunitySync, PresenceManager, generatePeerId, - OfflineStore + OfflineStore, + MiCanvasBridge, + installSelectionTransforms } from "@lib"; import { RStackIdentity } from "@shared/components/rstack-identity"; import { RStackAppSwitcher } from "@shared/components/rstack-app-switcher"; @@ -1438,6 +1440,12 @@ } }); + // Track selection for MI bridge + shape.addEventListener("pointerdown", () => { + selectedShapeId = shape.id; + __miCanvasBridge.setSelection([shape.id]); + }); + // Close button shape.addEventListener("close", () => { sync.deleteShape(shape.id); @@ -1592,6 +1600,12 @@ return shape; } + // ── MI Canvas Bridge + Selection Transforms ── + const __miCanvasBridge = new MiCanvasBridge(canvasContent); + window.__miCanvasBridge = __miCanvasBridge; + window.__canvasApi = { newShape, findFreePosition, SHAPE_DEFAULTS, setupShapeEventListeners, sync, canvasContent }; + installSelectionTransforms(); + // Toolbar button handlers document.getElementById("new-markdown").addEventListener("click", () => { newShape("folk-markdown", { content: "# New Note\n\nStart typing..." }); @@ -2140,6 +2154,8 @@ const gridSize = 20 * scale; canvas.style.backgroundSize = `${gridSize}px ${gridSize}px`; canvas.style.backgroundPosition = `${panX - 1}px ${panY - 1}px`; + // Keep MI bridge in sync + __miCanvasBridge.setViewport(panX, panY, scale); } document.getElementById("zoom-in").addEventListener("click", () => { @@ -2301,6 +2317,9 @@ canvas.addEventListener("pointerdown", (e) => { if (e.target !== canvas && e.target !== canvasContent) return; if (connectMode) return; + // Clicking canvas background clears MI selection + selectedShapeId = null; + __miCanvasBridge.setSelection([]); isPanning = true; panPointerId = e.pointerId; panStartX = e.clientX;