diff --git a/lib/mi-action-executor.ts b/lib/mi-action-executor.ts index 32fef92..b86658b 100644 --- a/lib/mi-action-executor.ts +++ b/lib/mi-action-executor.ts @@ -151,6 +151,12 @@ export class MiActionExecutor { case "delete-content": return this.#executeDeleteContent(action); + // Server-side actions (media gen, data queries) — delegate to server + case "generate-image": + case "generate-video": + case "query-content": + return this.#executeServerAction(action); + // Admin actions case "enable-module": case "disable-module": @@ -168,6 +174,30 @@ export class MiActionExecutor { } } + /** Delegate server-side actions to the MI API (fallback for non-agentic path). */ + async #executeServerAction(action: MiAction): Promise { + try { + const res = await fetch("/api/mi/execute-server-action", { + method: "POST", + headers: { + "Content-Type": "application/json", + ...(this.#token ? { Authorization: `Bearer ${this.#token}` } : {}), + }, + body: JSON.stringify({ action, space: this.#space }), + }); + + if (!res.ok) { + const err = await res.json().catch(() => ({ error: "Request failed" })); + return { action, ok: false, error: err.error || `HTTP ${res.status}` }; + } + + const data = await res.json(); + return { action, ok: data.ok !== false, contentId: data.url || data.summary }; + } catch (e: any) { + return { action, ok: false, error: e.message }; + } + } + #executeCanvasAction(action: MiAction, refMap: Map): ExecutionResult { const api = getCanvasApi(); if (!api) { diff --git a/lib/mi-actions.ts b/lib/mi-actions.ts index 8e5c7bc..69206bf 100644 --- a/lib/mi-actions.ts +++ b/lib/mi-actions.ts @@ -23,6 +23,11 @@ export type MiAction = | { type: "enable-module"; moduleId: string } | { type: "disable-module"; moduleId: string } | { type: "configure-module"; moduleId: string; settings: Record } + // Media generation (server-side) + | { type: "generate-image"; prompt: string; style?: string; ref?: string } + | { type: "generate-video"; prompt: string; source_image?: string; ref?: string } + // Data queries (server-side) + | { type: "query-content"; module: string; queryType: "recent" | "summary" | "count"; limit?: number; ref?: string } // Composite actions | { type: "scaffold"; name: string; steps: MiAction[] } | { type: "batch"; actions: MiAction[]; requireConfirm?: boolean }; @@ -94,6 +99,9 @@ export function summariseActions(actions: MiAction[]): string { if (counts["enable-module"]) parts.push(`Enabled ${counts["enable-module"]} module(s)`); if (counts["disable-module"]) parts.push(`Disabled ${counts["disable-module"]} module(s)`); if (counts["configure-module"]) parts.push(`Configured ${counts["configure-module"]} module(s)`); + if (counts["generate-image"]) parts.push(`Generated ${counts["generate-image"]} image(s)`); + if (counts["generate-video"]) parts.push(`Generated ${counts["generate-video"]} video(s)`); + if (counts["query-content"]) parts.push(`Queried ${counts["query-content"]} module(s)`); if (counts["scaffold"]) parts.push(`Scaffolded ${counts["scaffold"]} setup(s)`); if (counts["batch"]) parts.push(`Batch: ${counts["batch"]} group(s)`); return parts.join(", ") || ""; @@ -128,6 +136,15 @@ export function detailedActionSummary(actions: MiAction[]): string[] { case "navigate": details.push(`Navigate to ${a.path}`); break; + case "generate-image": + details.push(`Generate image: "${a.prompt.slice(0, 60)}${a.prompt.length > 60 ? "…" : ""}"`); + break; + case "generate-video": + details.push(`Generate video: "${a.prompt.slice(0, 60)}${a.prompt.length > 60 ? "…" : ""}"`); + break; + case "query-content": + details.push(`Query ${a.module}: ${a.queryType}`); + break; default: details.push(`${a.type}`); } diff --git a/modules/rnotes/mod.ts b/modules/rnotes/mod.ts index 6cbad74..e65b7c3 100644 --- a/modules/rnotes/mod.ts +++ b/modules/rnotes/mod.ts @@ -1719,3 +1719,44 @@ export const notesModule: RSpaceModule = { { label: "Create a Notebook", icon: "✏️", description: "Start writing from scratch", type: 'create', href: '/{space}/rnotes' }, ], }; + +// ── MI Integration ── + +export interface MINoteItem { + id: string; + title: string; + contentPlain: string; + tags: string[]; + type: string; + updatedAt: number; +} + +/** + * Read recent notes directly from Automerge for the MI system prompt. + */ +export function getRecentNotesForMI(space: string, limit = 3): MINoteItem[] { + if (!_syncServer) return []; + const allNotes: MINoteItem[] = []; + const prefix = `${space}:notes:notebooks:`; + + for (const docId of _syncServer.listDocs()) { + if (!docId.startsWith(prefix)) continue; + const doc = _syncServer.getDoc(docId); + if (!doc?.items) continue; + + for (const item of Object.values(doc.items)) { + allNotes.push({ + id: item.id, + title: item.title, + contentPlain: (item.contentPlain || "").slice(0, 300), + tags: item.tags ? Array.from(item.tags) : [], + type: item.type, + updatedAt: item.updatedAt, + }); + } + } + + return allNotes + .sort((a, b) => b.updatedAt - a.updatedAt) + .slice(0, limit); +} diff --git a/modules/rtasks/mod.ts b/modules/rtasks/mod.ts index aa92874..3c6ffa3 100644 --- a/modules/rtasks/mod.ts +++ b/modules/rtasks/mod.ts @@ -828,3 +828,45 @@ export const tasksModule: RSpaceModule = { { label: "Create a Taskboard", icon: "📋", description: "Start a new kanban project board", type: 'create', href: '/{space}/rtasks' }, ], }; + +// ── MI Integration ── + +export interface MITaskItem { + id: string; + title: string; + status: string; + priority: string | null; + description: string; + createdAt: number; +} + +/** + * Read recent/open tasks directly from Automerge for the MI system prompt. + */ +export function getRecentTasksForMI(space: string, limit = 5): MITaskItem[] { + if (!_syncServer) return []; + const allTasks: MITaskItem[] = []; + + for (const docId of _syncServer.getDocIds()) { + if (!docId.startsWith(`${space}:tasks:boards:`)) continue; + const doc = _syncServer.getDoc(docId); + if (!doc?.tasks) continue; + + for (const task of Object.values(doc.tasks)) { + allTasks.push({ + id: task.id, + title: task.title, + status: task.status, + priority: task.priority, + description: (task.description || "").slice(0, 200), + createdAt: task.createdAt, + }); + } + } + + // Prioritize non-DONE tasks, then sort by creation date + return allTasks + .filter((t) => t.status !== "DONE") + .sort((a, b) => b.createdAt - a.createdAt) + .slice(0, limit); +} diff --git a/server/index.ts b/server/index.ts index 90dade4..ca673fb 100644 --- a/server/index.ts +++ b/server/index.ts @@ -1182,47 +1182,15 @@ async function saveDataUrlToDisk(dataUrl: string, prefix: string): Promise { - if (!FAL_KEY) return c.json({ error: "FAL_KEY not configured" }, 503); - const { prompt, style } = await c.req.json(); if (!prompt) return c.json({ error: "prompt required" }, 400); - const stylePrompts: Record = { - illustration: "digital illustration style, ", - photorealistic: "photorealistic, high detail, ", - painting: "oil painting style, artistic, ", - sketch: "pencil sketch style, hand-drawn, ", - "punk-zine": "punk zine aesthetic, cut-and-paste collage, bold contrast, ", - }; - const styledPrompt = (stylePrompts[style] || "") + prompt; - - const res = await fetch("https://fal.run/fal-ai/flux-pro/v1.1", { - method: "POST", - headers: { - Authorization: `Key ${FAL_KEY}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - prompt: styledPrompt, - image_size: "landscape_4_3", - num_images: 1, - safety_tolerance: "2", - }), - }); - - if (!res.ok) { - const err = await res.text(); - console.error("[image-gen] fal.ai error:", err); - return c.json({ error: "Image generation failed" }, 502); - } - - const data = await res.json(); - const imageUrl = data.images?.[0]?.url || data.output?.url; - if (!imageUrl) return c.json({ error: "No image returned" }, 502); - - return c.json({ url: imageUrl, image_url: imageUrl }); + const { generateImageViaFal } = await import("./mi-media"); + const result = await generateImageViaFal(prompt, style); + if (!result.ok) return c.json({ error: result.error }, 502); + return c.json({ url: result.url, image_url: result.url }); }); // Upload image (data URL → disk) @@ -1438,37 +1406,15 @@ app.post("/api/image-gen/img2img", async (c) => { return c.json({ error: `Unknown provider: ${provider}` }, 400); }); -// Text-to-video via fal.ai WAN 2.1 +// Text-to-video via fal.ai WAN 2.1 (delegates to shared helper) app.post("/api/video-gen/t2v", async (c) => { - if (!FAL_KEY) return c.json({ error: "FAL_KEY not configured" }, 503); - - const { prompt, duration } = await c.req.json(); + const { prompt } = await c.req.json(); if (!prompt) return c.json({ error: "prompt required" }, 400); - const res = await fetch("https://fal.run/fal-ai/wan/v2.1", { - method: "POST", - headers: { - Authorization: `Key ${FAL_KEY}`, - "Content-Type": "application/json", - }, - body: JSON.stringify({ - prompt, - num_frames: duration === "5s" ? 81 : 49, - resolution: "480p", - }), - }); - - if (!res.ok) { - const err = await res.text(); - console.error("[video-gen/t2v] fal.ai error:", err); - return c.json({ error: "Video generation failed" }, 502); - } - - const data = await res.json(); - const videoUrl = data.video?.url || data.output?.url; - if (!videoUrl) return c.json({ error: "No video returned" }, 502); - - return c.json({ url: videoUrl, video_url: videoUrl }); + const { generateVideoViaFal } = await import("./mi-media"); + const result = await generateVideoViaFal(prompt); + if (!result.ok) return c.json({ error: result.error }, 502); + return c.json({ url: result.url, video_url: result.url }); }); // Image-to-video via fal.ai Kling diff --git a/server/mi-agent.ts b/server/mi-agent.ts new file mode 100644 index 0000000..c1d3fa7 --- /dev/null +++ b/server/mi-agent.ts @@ -0,0 +1,201 @@ +/** + * MI Agentic Loop — multi-turn streaming orchestrator. + * + * Streams LLM output, detects [MI_ACTION:{...}] markers, executes + * server-side actions (media gen, data queries), and feeds results + * back for the next LLM turn. Client-side actions (canvas shapes) + * pass through unchanged in the stream. + */ + +import type { MiProvider, MiMessage, MiStreamChunk } from "./mi-provider"; +import { parseMiActions } from "../lib/mi-actions"; +import type { MiAction } from "../lib/mi-actions"; +import { generateImage, generateVideoViaFal } from "./mi-media"; +import { queryModuleContent } from "./mi-data-queries"; + +export interface AgenticLoopOptions { + messages: MiMessage[]; + provider: MiProvider; + providerModel: string; + space: string; + maxTurns?: number; +} + +/** NDJSON line types emitted by the agentic loop. */ +interface TurnLine { type: "turn"; turn: number; maxTurns: number } +interface ActionStartLine { type: "action-start"; action: { type: string; ref?: string } } +interface ActionResultLine { + type: "action-result"; + actionType: string; + ok: boolean; + url?: string; + data?: any; + summary?: string; + error?: string; + ref?: string; +} +interface MessageLine { message: { role: string; content: string }; done: boolean } + +type NDJSONLine = TurnLine | ActionStartLine | ActionResultLine | MessageLine; + +const SERVER_ACTION_TYPES = new Set(["generate-image", "generate-video", "query-content"]); + +function isServerAction(action: MiAction): boolean { + return SERVER_ACTION_TYPES.has(action.type); +} + +/** + * Execute a single server-side MI action. + */ +async function executeServerAction( + action: MiAction, + space: string, +): Promise { + const ref = "ref" in action ? (action as any).ref : undefined; + + switch (action.type) { + case "generate-image": { + const result = await generateImage(action.prompt, action.style); + if (result.ok) { + return { type: "action-result", actionType: "generate-image", ok: true, url: result.url, ref }; + } + return { type: "action-result", actionType: "generate-image", ok: false, error: result.error, ref }; + } + + case "generate-video": { + const result = await generateVideoViaFal(action.prompt, action.source_image); + if (result.ok) { + return { type: "action-result", actionType: "generate-video", ok: true, url: result.url, ref }; + } + return { type: "action-result", actionType: "generate-video", ok: false, error: result.error, ref }; + } + + case "query-content": { + const result = queryModuleContent(space, action.module, action.queryType, action.limit); + return { + type: "action-result", + actionType: "query-content", + ok: result.ok, + data: result.data, + summary: result.summary, + ref, + }; + } + + default: + return { type: "action-result", actionType: (action as any).type, ok: false, error: "Unknown server action", ref }; + } +} + +/** + * Run the agentic loop. Returns a ReadableStream of NDJSON lines. + * + * Each turn: stream LLM → accumulate text → parse actions → + * execute server-side actions → feed results back → next turn. + * Client-side actions pass through in the text stream. + */ +export function runAgenticLoop(opts: AgenticLoopOptions): ReadableStream { + const { messages, provider, providerModel, space, maxTurns = 5 } = opts; + const encoder = new TextEncoder(); + + // Working copy of conversation + const conversation = [...messages]; + + return new ReadableStream({ + async start(controller) { + try { + for (let turn = 0; turn < maxTurns; turn++) { + // Emit turn indicator (skip turn 0 for backwards compat) + if (turn > 0) { + emit(controller, encoder, { type: "turn", turn, maxTurns }); + } + + // Stream LLM response + let fullText = ""; + const gen = provider.stream(conversation, providerModel); + + for await (const chunk of gen) { + if (chunk.content) { + fullText += chunk.content; + // Stream text to client in real-time (Ollama NDJSON format) + emit(controller, encoder, { + message: { role: "assistant", content: chunk.content }, + done: false, + }); + } + if (chunk.done) break; + } + + // Parse actions from accumulated text + const { actions } = parseMiActions(fullText); + const serverActions = actions.filter(isServerAction); + + // If no server-side actions, we're done + if (serverActions.length === 0) { + emit(controller, encoder, { + message: { role: "assistant", content: "" }, + done: true, + }); + controller.close(); + return; + } + + // Execute server-side actions + const resultSummaries: string[] = []; + for (const action of serverActions) { + // Notify client that action is starting + emit(controller, encoder, { + type: "action-start", + action: { type: action.type, ref: "ref" in action ? (action as any).ref : undefined }, + }); + + const result = await executeServerAction(action, space); + emit(controller, encoder, result); + + // Build summary for next LLM turn + if (result.ok) { + if (result.url) { + resultSummaries.push(`[${result.actionType}] Generated: ${result.url}`); + } else if (result.summary) { + resultSummaries.push(`[${result.actionType}] ${result.summary}`); + } + } else { + resultSummaries.push(`[${result.actionType}] Failed: ${result.error}`); + } + } + + // Add assistant message + action results to conversation for next turn + conversation.push({ role: "assistant", content: fullText }); + conversation.push({ + role: "user", + content: `Action results:\n${resultSummaries.join("\n")}\n\nIncorporate these results and continue. If you generated an image or video, you can now create a canvas shape referencing the URL. Do not re-generate media that already succeeded.`, + }); + } + + // Max turns reached + emit(controller, encoder, { + message: { role: "assistant", content: "" }, + done: true, + }); + controller.close(); + } catch (err: any) { + // Emit error as a message so client can display it + try { + emit(controller, encoder, { + message: { role: "assistant", content: `\n\n*Error: ${err.message}*` }, + done: true, + }); + } catch { /* controller may be closed */ } + controller.close(); + } + }, + }); +} + +function emit( + controller: ReadableStreamDefaultController, + encoder: TextEncoder, + data: NDJSONLine, +): void { + controller.enqueue(encoder.encode(JSON.stringify(data) + "\n")); +} diff --git a/server/mi-data-queries.ts b/server/mi-data-queries.ts new file mode 100644 index 0000000..fc73972 --- /dev/null +++ b/server/mi-data-queries.ts @@ -0,0 +1,66 @@ +/** + * MI Data Queries — dispatches content queries to module-specific readers. + * + * Used by the agentic loop to fetch module data server-side and feed + * results back into the LLM context. + */ + +import { getUpcomingEventsForMI } from "../modules/rcal/mod"; +import { getRecentNotesForMI } from "../modules/rnotes/mod"; +import { getRecentTasksForMI } from "../modules/rtasks/mod"; + +export interface MiQueryResult { + ok: boolean; + module: string; + queryType: string; + data: any; + summary: string; +} + +/** + * Query module content by type. Returns structured data + a text summary + * the LLM can consume. + */ +export function queryModuleContent( + space: string, + module: string, + queryType: "recent" | "summary" | "count", + limit = 5, +): MiQueryResult { + switch (module) { + case "rnotes": { + const notes = getRecentNotesForMI(space, limit); + if (queryType === "count") { + return { ok: true, module, queryType, data: { count: notes.length }, summary: `${notes.length} recent notes found.` }; + } + const lines = notes.map((n) => `- "${n.title}" (${n.type}, updated ${new Date(n.updatedAt).toLocaleDateString()})${n.tags.length ? ` [${n.tags.join(", ")}]` : ""}: ${n.contentPlain.slice(0, 100)}...`); + return { ok: true, module, queryType, data: notes, summary: lines.length ? `Recent notes:\n${lines.join("\n")}` : "No notes found." }; + } + + case "rtasks": { + const tasks = getRecentTasksForMI(space, limit); + if (queryType === "count") { + return { ok: true, module, queryType, data: { count: tasks.length }, summary: `${tasks.length} open tasks found.` }; + } + const lines = tasks.map((t) => `- "${t.title}" [${t.status}]${t.priority ? ` (${t.priority})` : ""}${t.description ? `: ${t.description.slice(0, 80)}` : ""}`); + return { ok: true, module, queryType, data: tasks, summary: lines.length ? `Open tasks:\n${lines.join("\n")}` : "No open tasks." }; + } + + case "rcal": { + const events = getUpcomingEventsForMI(space, 14, limit); + if (queryType === "count") { + return { ok: true, module, queryType, data: { count: events.length }, summary: `${events.length} upcoming events.` }; + } + const lines = events.map((e) => { + const date = e.allDay ? e.start.split("T")[0] : e.start; + let line = `- ${date}: ${e.title}`; + if (e.location) line += ` (${e.location})`; + return line; + }); + return { ok: true, module, queryType, data: events, summary: lines.length ? `Upcoming events:\n${lines.join("\n")}` : "No upcoming events." }; + } + + default: + return { ok: false, module, queryType, data: null, summary: `Module "${module}" does not support content queries.` }; + } +} diff --git a/server/mi-media.ts b/server/mi-media.ts new file mode 100644 index 0000000..3b2c696 --- /dev/null +++ b/server/mi-media.ts @@ -0,0 +1,202 @@ +/** + * MI Media Generation — shared helpers for image and video generation. + * + * Extracted from server/index.ts endpoints so the agentic loop can + * call them directly without HTTP round-trips. + */ + +import { resolve } from "path"; + +const FAL_KEY = process.env.FAL_KEY || ""; +const GEMINI_API_KEY = process.env.GEMINI_API_KEY || ""; + +const STYLE_PROMPTS: Record = { + illustration: "digital illustration style, ", + photorealistic: "photorealistic, high detail, ", + painting: "oil painting style, artistic, ", + sketch: "pencil sketch style, hand-drawn, ", + "punk-zine": "punk zine aesthetic, cut-and-paste collage, bold contrast, ", +}; + +const GEMINI_STYLE_HINTS: Record = { + photorealistic: "photorealistic, high detail, natural lighting, ", + illustration: "digital illustration, clean lines, vibrant colors, ", + painting: "oil painting style, brushstrokes visible, painterly, ", + sketch: "pencil sketch, hand-drawn, line art, ", + "punk-zine": "punk zine aesthetic, xerox texture, high contrast, DIY, rough edges, ", + collage: "cut-and-paste collage, mixed media, layered paper textures, ", + vintage: "vintage aesthetic, retro colors, aged paper texture, ", + minimalist: "minimalist design, simple shapes, limited color palette, ", +}; + +export interface MediaResult { + ok: true; + url: string; +} + +export interface MediaError { + ok: false; + error: string; +} + +export type MediaOutcome = MediaResult | MediaError; + +/** + * Generate an image via fal.ai Flux Pro. + */ +export async function generateImageViaFal(prompt: string, style?: string): Promise { + if (!FAL_KEY) return { ok: false, error: "FAL_KEY not configured" }; + + const styledPrompt = (style && STYLE_PROMPTS[style] || "") + prompt; + + const res = await fetch("https://fal.run/fal-ai/flux-pro/v1.1", { + method: "POST", + headers: { + Authorization: `Key ${FAL_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + prompt: styledPrompt, + image_size: "landscape_4_3", + num_images: 1, + safety_tolerance: "2", + }), + }); + + if (!res.ok) { + const err = await res.text(); + console.error("[mi-media] fal.ai image error:", err); + return { ok: false, error: "Image generation failed" }; + } + + const data = await res.json(); + const imageUrl = data.images?.[0]?.url || data.output?.url; + if (!imageUrl) return { ok: false, error: "No image returned" }; + + return { ok: true, url: imageUrl }; +} + +/** + * Generate an image via Gemini (gemini-2.5-flash-image or imagen-3.0). + */ +export async function generateImageViaGemini(prompt: string, style?: string): Promise { + if (!GEMINI_API_KEY) return { ok: false, error: "GEMINI_API_KEY not configured" }; + + const enhancedPrompt = (style && GEMINI_STYLE_HINTS[style] || "") + prompt; + const { GoogleGenAI } = await import("@google/genai"); + const ai = new GoogleGenAI({ apiKey: GEMINI_API_KEY }); + + const models = ["gemini-2.5-flash-image", "imagen-3.0-generate-002"]; + for (const modelName of models) { + try { + if (modelName.startsWith("gemini")) { + const result = await ai.models.generateContent({ + model: modelName, + contents: enhancedPrompt, + config: { responseModalities: ["Text", "Image"] }, + }); + + const parts = result.candidates?.[0]?.content?.parts || []; + for (const part of parts) { + if ((part as any).inlineData) { + const { data: b64, mimeType } = (part as any).inlineData; + const ext = mimeType?.includes("png") ? "png" : "jpg"; + const filename = `gemini-${Date.now()}.${ext}`; + const dir = resolve(process.env.FILES_DIR || "./data/files", "generated"); + await Bun.write(resolve(dir, filename), Buffer.from(b64, "base64")); + return { ok: true, url: `/data/files/generated/${filename}` }; + } + } + } else { + const result = await ai.models.generateImages({ + model: modelName, + prompt: enhancedPrompt, + config: { numberOfImages: 1, aspectRatio: "3:4" }, + }); + const img = (result as any).generatedImages?.[0]; + if (img?.image?.imageBytes) { + const filename = `imagen-${Date.now()}.png`; + const dir = resolve(process.env.FILES_DIR || "./data/files", "generated"); + await Bun.write(resolve(dir, filename), Buffer.from(img.image.imageBytes, "base64")); + return { ok: true, url: `/data/files/generated/${filename}` }; + } + } + } catch (e: any) { + console.error(`[mi-media] ${modelName} error:`, e.message); + continue; + } + } + + return { ok: false, error: "All Gemini image models failed" }; +} + +/** + * Generate a text-to-video via fal.ai WAN 2.1. + */ +export async function generateVideoViaFal(prompt: string, source_image?: string): Promise { + if (!FAL_KEY) return { ok: false, error: "FAL_KEY not configured" }; + + if (source_image) { + // Image-to-video via Kling + const res = await fetch("https://fal.run/fal-ai/kling-video/v1/standard/image-to-video", { + method: "POST", + headers: { + Authorization: `Key ${FAL_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + image_url: source_image, + prompt: prompt || "", + duration: "5", + aspect_ratio: "16:9", + }), + }); + + if (!res.ok) { + const err = await res.text(); + console.error("[mi-media] fal.ai i2v error:", err); + return { ok: false, error: "Video generation failed" }; + } + + const data = await res.json(); + const videoUrl = data.video?.url || data.output?.url; + if (!videoUrl) return { ok: false, error: "No video returned" }; + return { ok: true, url: videoUrl }; + } + + // Text-to-video via WAN 2.1 + const res = await fetch("https://fal.run/fal-ai/wan/v2.1", { + method: "POST", + headers: { + Authorization: `Key ${FAL_KEY}`, + "Content-Type": "application/json", + }, + body: JSON.stringify({ + prompt, + num_frames: 49, + resolution: "480p", + }), + }); + + if (!res.ok) { + const err = await res.text(); + console.error("[mi-media] fal.ai t2v error:", err); + return { ok: false, error: "Video generation failed" }; + } + + const data = await res.json(); + const videoUrl = data.video?.url || data.output?.url; + if (!videoUrl) return { ok: false, error: "No video returned" }; + + return { ok: true, url: videoUrl }; +} + +/** + * Try fal.ai first, fall back to Gemini for image generation. + */ +export async function generateImage(prompt: string, style?: string): Promise { + const falResult = await generateImageViaFal(prompt, style); + if (falResult.ok) return falResult; + + return generateImageViaGemini(prompt, style); +} diff --git a/server/mi-provider.ts b/server/mi-provider.ts index 9c56fd3..f67ef2e 100644 --- a/server/mi-provider.ts +++ b/server/mi-provider.ts @@ -45,6 +45,8 @@ const MODEL_REGISTRY: MiModelConfig[] = [ { id: "llama3.1", provider: "ollama", providerModel: "llama3.1:8b", label: "Llama 3.1 (8B)", group: "Local" }, { id: "qwen2.5-coder", provider: "ollama", providerModel: "qwen2.5-coder:7b", label: "Qwen Coder", group: "Local" }, { id: "mistral-small", provider: "ollama", providerModel: "mistral-small:24b", label: "Mistral Small", group: "Local" }, + { id: "claude-sonnet", provider: "litellm", providerModel: "claude-sonnet", label: "Claude Sonnet", group: "Claude" }, + { id: "claude-haiku", provider: "litellm", providerModel: "claude-haiku", label: "Claude Haiku", group: "Claude" }, ]; // ── Ollama Provider ── @@ -160,31 +162,79 @@ class GeminiProvider implements MiProvider { } } -// ── Anthropic Provider (stub) ── +// ── LiteLLM Provider (OpenAI-compatible SSE via LiteLLM proxy) ── -class AnthropicProvider implements MiProvider { - id = "anthropic"; +class LiteLLMProvider implements MiProvider { + id = "litellm"; + #url: string; + #apiKey: string; + + constructor() { + this.#url = process.env.LITELLM_URL || "https://llm.jeffemmett.com"; + this.#apiKey = process.env.LITELLM_API_KEY || ""; + } isAvailable(): boolean { - return !!(process.env.ANTHROPIC_API_KEY); + return !!(this.#url && this.#apiKey); } - async *stream(_messages: MiMessage[], _model: string): AsyncGenerator { - throw new Error("Anthropic provider not yet implemented — add @anthropic-ai/sdk"); - } -} + async *stream(messages: MiMessage[], model: string): AsyncGenerator { + const res = await fetch(`${this.#url}/v1/chat/completions`, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${this.#apiKey}`, + }, + body: JSON.stringify({ + model, + messages: messages.map((m) => ({ role: m.role, content: m.content })), + stream: true, + }), + }); -// ── OpenAI-Compatible Provider (stub) ── + if (!res.ok) { + const errText = await res.text().catch(() => ""); + throw new Error(`LiteLLM error ${res.status}: ${errText}`); + } -class OpenAICompatProvider implements MiProvider { - id = "openai-compat"; + if (!res.body) throw new Error("No response body from LiteLLM"); - isAvailable(): boolean { - return !!(process.env.OPENAI_COMPAT_URL && process.env.OPENAI_COMPAT_KEY); - } + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ""; - async *stream(_messages: MiMessage[], _model: string): AsyncGenerator { - throw new Error("OpenAI-compatible provider not yet implemented"); + while (true) { + const { done, value } = await reader.read(); + if (done) break; + + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split("\n"); + buffer = lines.pop() || ""; + + for (const line of lines) { + const trimmed = line.trim(); + if (!trimmed || !trimmed.startsWith("data: ")) continue; + const payload = trimmed.slice(6); + if (payload === "[DONE]") { + yield { content: "", done: true }; + return; + } + try { + const data = JSON.parse(payload); + const delta = data.choices?.[0]?.delta; + const finishReason = data.choices?.[0]?.finish_reason; + if (delta?.content) { + yield { content: delta.content, done: false }; + } + if (finishReason === "stop") { + yield { content: "", done: true }; + return; + } + } catch { + // skip malformed SSE lines + } + } + } } } @@ -196,8 +246,7 @@ export class MiProviderRegistry { constructor() { this.register(new OllamaProvider()); this.register(new GeminiProvider()); - this.register(new AnthropicProvider()); - this.register(new OpenAICompatProvider()); + this.register(new LiteLLMProvider()); } register(provider: MiProvider): void { diff --git a/server/mi-routes.ts b/server/mi-routes.ts index 0830687..5ab318d 100644 --- a/server/mi-routes.ts +++ b/server/mi-routes.ts @@ -18,6 +18,11 @@ import type { EncryptIDClaims } from "./auth"; import { buildModuleCapabilities, MODULE_ROUTES } from "../lib/mi-module-routes"; import type { MiAction } from "../lib/mi-actions"; import { getUpcomingEventsForMI } from "../modules/rcal/mod"; +import { getRecentNotesForMI } from "../modules/rnotes/mod"; +import { getRecentTasksForMI } from "../modules/rtasks/mod"; +import { runAgenticLoop } from "./mi-agent"; +import { generateImage, generateVideoViaFal } from "./mi-media"; +import { queryModuleContent } from "./mi-data-queries"; const mi = new Hono(); @@ -135,6 +140,8 @@ mi.post("/ask", async (c) => { const timeContext = `${weekdays[now.getUTCDay()]}, ${now.toISOString().split("T")[0]} at ${now.toISOString().split("T")[1].split(".")[0]} UTC`; let calendarContext = ""; + let notesContext = ""; + let tasksContext = ""; if (space) { const upcoming = getUpcomingEventsForMI(space); if (upcoming.length > 0) { @@ -148,6 +155,22 @@ mi.post("/ask", async (c) => { }); calendarContext = `\n- Upcoming events (next 14 days):\n${lines.join("\n")}`; } + + const recentNotes = getRecentNotesForMI(space, 3); + if (recentNotes.length > 0) { + const lines = recentNotes.map((n) => + `- "${n.title}" (${n.type}${n.tags.length ? `, tags: ${n.tags.join(", ")}` : ""}): ${n.contentPlain.slice(0, 100)}…` + ); + notesContext = `\n- Recent notes:\n${lines.join("\n")}`; + } + + const openTasks = getRecentTasksForMI(space, 5); + if (openTasks.length > 0) { + const lines = openTasks.map((t) => + `- "${t.title}" [${t.status}]${t.priority ? ` (${t.priority})` : ""}${t.description ? `: ${t.description.slice(0, 80)}` : ""}` + ); + tasksContext = `\n- Open tasks:\n${lines.join("\n")}`; + } } const systemPrompt = `You are mi (mycelial intelligence), the intelligent assistant for rSpace — a self-hosted, community-run platform. @@ -173,7 +196,7 @@ ${moduleCapabilities} When the user asks to create a social media campaign, use create-content with module rsocials, contentType campaign, body.rawBrief set to the user's description, body.navigateToWizard true. ## Current Context -${contextSection}${calendarContext} +${contextSection}${calendarContext}${notesContext}${tasksContext} ## Guidelines - Be concise and helpful. Keep responses short (2-4 sentences) unless the user asks for detail. @@ -216,6 +239,23 @@ When the user asks to create content in a specific rApp (not a canvas shape): For multi-step setup requests like "set up this space for a book club": [MI_ACTION:{"type":"scaffold","name":"Book Club Setup","steps":[...ordered actions...]}] +## Media Generation (server-side — MI will execute these and return the result URL) +When the user asks you to generate an image or video, use these actions: +[MI_ACTION:{"type":"generate-image","prompt":"a forest of glowing mushrooms at dusk","style":"illustration","ref":"$1"}] +[MI_ACTION:{"type":"generate-video","prompt":"timelapse of mycelium growing through soil","ref":"$2"}] +After the server generates the media, you will receive the URL in a follow-up message. +Then create a canvas shape referencing that URL: +[MI_ACTION:{"type":"create-shape","tagName":"folk-image-gen","props":{"src":""},"ref":"$3"}] +Available styles: illustration, photorealistic, painting, sketch, punk-zine. + +## Content Queries (server-side — MI will fetch and return results) +When you need to look up the user's actual data (notes, tasks, events): +[MI_ACTION:{"type":"query-content","module":"rnotes","queryType":"recent","limit":5}] +[MI_ACTION:{"type":"query-content","module":"rtasks","queryType":"recent","limit":5}] +[MI_ACTION:{"type":"query-content","module":"rcal","queryType":"recent","limit":5}] +queryType can be: "recent", "summary", or "count". +Results will be provided in a follow-up message for you to incorporate into your response. + ## Batch Actions [MI_ACTION:{"type":"batch","actions":[...actions...],"requireConfirm":true}] Use requireConfirm:true for destructive batches.`; @@ -228,8 +268,13 @@ Use requireConfirm:true for destructive batches.`; ]; try { - const gen = providerInfo.provider.stream(miMessages, providerInfo.providerModel); - const body = miRegistry.streamToNDJSON(gen); + const body = runAgenticLoop({ + messages: miMessages, + provider: providerInfo.provider, + providerModel: providerInfo.providerModel, + space: space || "", + maxTurns: 5, + }); return new Response(body, { headers: { @@ -377,8 +422,11 @@ function getRequiredRole(action: MiAction): SpaceRoleString { case "transform": case "scaffold": case "batch": + case "generate-image": + case "generate-video": return "member"; case "navigate": + case "query-content": return "viewer"; default: return "member"; @@ -413,6 +461,30 @@ mi.post("/validate-actions", async (c) => { return c.json({ validated, callerRole }); }); +// ── POST /execute-server-action — client-side fallback for server actions ── + +mi.post("/execute-server-action", async (c) => { + const { action, space } = await c.req.json(); + if (!action?.type) return c.json({ error: "action required" }, 400); + + switch (action.type) { + case "generate-image": { + const result = await generateImage(action.prompt, action.style); + return c.json(result); + } + case "generate-video": { + const result = await generateVideoViaFal(action.prompt, action.source_image); + return c.json(result); + } + case "query-content": { + const result = queryModuleContent(space || "", action.module, action.queryType, action.limit); + return c.json(result); + } + default: + return c.json({ ok: false, error: `Unknown server action: ${action.type}` }, 400); + } +}); + // ── Fallback response (when AI is unavailable) ── function generateFallbackResponse( diff --git a/shared/components/rstack-mi.ts b/shared/components/rstack-mi.ts index dd09a84..4e675a0 100644 --- a/shared/components/rstack-mi.ts +++ b/shared/components/rstack-mi.ts @@ -445,6 +445,8 @@ export class RStackMi extends HTMLElement { const reader = res.body.getReader(); const decoder = new TextDecoder(); + const serverActionResults: { type: string; url?: string; ref?: string }[] = []; + while (true) { const { done, value } = await reader.read(); if (done) break; @@ -453,12 +455,36 @@ export class RStackMi extends HTMLElement { for (const line of chunk.split("\n").filter(Boolean)) { try { const data = JSON.parse(line); + + // Standard message content (backwards-compatible) if (data.message?.content) { this.#messages[assistantIdx].content += data.message.content; } if (data.response) { this.#messages[assistantIdx].content = data.response; } + + // Agentic loop: turn indicator + if (data.type === "turn" && data.turn > 0) { + this.#messages[assistantIdx].content += `\n\n*— thinking (turn ${data.turn + 1}/${data.maxTurns}) —*\n\n`; + } + + // Agentic loop: server action starting + if (data.type === "action-start") { + this.#messages[assistantIdx].content += `\n*⏳ ${data.action?.type}…*\n`; + } + + // Agentic loop: server action result + if (data.type === "action-result") { + if (data.ok && data.url) { + serverActionResults.push({ type: data.actionType, url: data.url, ref: data.ref }); + this.#messages[assistantIdx].content += `\n*✓ ${data.actionType}: generated*\n`; + } else if (data.ok && data.summary) { + this.#messages[assistantIdx].content += `\n*✓ ${data.actionType}: ${data.summary.slice(0, 80)}*\n`; + } else if (!data.ok) { + this.#messages[assistantIdx].content += `\n*✗ ${data.actionType}: ${data.error || "failed"}*\n`; + } + } } catch { /* skip malformed lines */ } } this.#renderMessages(messagesEl); @@ -468,7 +494,7 @@ export class RStackMi extends HTMLElement { this.#messages[assistantIdx].content = "I couldn't generate a response. Please try again."; this.#renderMessages(messagesEl); } else { - // Parse and execute MI actions + // Parse and execute MI actions (client-side canvas actions) const rawText = this.#messages[assistantIdx].content; const { displayText, actions } = parseMiActions(rawText); this.#messages[assistantIdx].content = displayText; @@ -479,6 +505,23 @@ export class RStackMi extends HTMLElement { this.#messages[assistantIdx].actionDetails = detailedActionSummary(actions); } + // Create canvas shapes for server-generated media + if (serverActionResults.length) { + const mediaActions: MiAction[] = []; + for (const r of serverActionResults) { + if (r.type === "generate-image" && r.url) { + mediaActions.push({ type: "create-shape", tagName: "folk-image-gen", props: { src: r.url }, ref: r.ref }); + } else if (r.type === "generate-video" && r.url) { + mediaActions.push({ type: "create-shape", tagName: "folk-video-gen", props: { src: r.url }, ref: r.ref }); + } + } + if (mediaActions.length) { + const executor = new MiActionExecutor(); + executor.setContext(context.space || "", getAccessToken() || ""); + executor.execute(mediaActions); + } + } + // Check for tool suggestions const hints = suggestTools(query); if (hints.length) {