feat(mi): agentic upgrade — multi-turn loop, LiteLLM, media gen, live data
Transform MI from single-shot chat into an agentic system: - LiteLLM provider with Claude Sonnet/Haiku models via proxy - Agentic loop (max 5 turns): stream → parse actions → execute server-side → feed results back - Server-side media generation (fal.ai + Gemini) as first-class MI actions - Module data queries (rnotes, rtasks, rcal) read directly from Automerge - System prompt enriched with recent notes, open tasks, and calendar events - Client handles new NDJSON types (turn, action-start, action-result) - Extracted shared media helpers, refactored image/video endpoints Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
a5c7eef466
commit
dd46905e12
|
|
@ -151,6 +151,12 @@ export class MiActionExecutor {
|
|||
case "delete-content":
|
||||
return this.#executeDeleteContent(action);
|
||||
|
||||
// Server-side actions (media gen, data queries) — delegate to server
|
||||
case "generate-image":
|
||||
case "generate-video":
|
||||
case "query-content":
|
||||
return this.#executeServerAction(action);
|
||||
|
||||
// Admin actions
|
||||
case "enable-module":
|
||||
case "disable-module":
|
||||
|
|
@ -168,6 +174,30 @@ export class MiActionExecutor {
|
|||
}
|
||||
}
|
||||
|
||||
/** Delegate server-side actions to the MI API (fallback for non-agentic path). */
|
||||
async #executeServerAction(action: MiAction): Promise<ExecutionResult> {
|
||||
try {
|
||||
const res = await fetch("/api/mi/execute-server-action", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
...(this.#token ? { Authorization: `Bearer ${this.#token}` } : {}),
|
||||
},
|
||||
body: JSON.stringify({ action, space: this.#space }),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const err = await res.json().catch(() => ({ error: "Request failed" }));
|
||||
return { action, ok: false, error: err.error || `HTTP ${res.status}` };
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
return { action, ok: data.ok !== false, contentId: data.url || data.summary };
|
||||
} catch (e: any) {
|
||||
return { action, ok: false, error: e.message };
|
||||
}
|
||||
}
|
||||
|
||||
#executeCanvasAction(action: MiAction, refMap: Map<string, string>): ExecutionResult {
|
||||
const api = getCanvasApi();
|
||||
if (!api) {
|
||||
|
|
|
|||
|
|
@ -23,6 +23,11 @@ export type MiAction =
|
|||
| { type: "enable-module"; moduleId: string }
|
||||
| { type: "disable-module"; moduleId: string }
|
||||
| { type: "configure-module"; moduleId: string; settings: Record<string, any> }
|
||||
// Media generation (server-side)
|
||||
| { type: "generate-image"; prompt: string; style?: string; ref?: string }
|
||||
| { type: "generate-video"; prompt: string; source_image?: string; ref?: string }
|
||||
// Data queries (server-side)
|
||||
| { type: "query-content"; module: string; queryType: "recent" | "summary" | "count"; limit?: number; ref?: string }
|
||||
// Composite actions
|
||||
| { type: "scaffold"; name: string; steps: MiAction[] }
|
||||
| { type: "batch"; actions: MiAction[]; requireConfirm?: boolean };
|
||||
|
|
@ -94,6 +99,9 @@ export function summariseActions(actions: MiAction[]): string {
|
|||
if (counts["enable-module"]) parts.push(`Enabled ${counts["enable-module"]} module(s)`);
|
||||
if (counts["disable-module"]) parts.push(`Disabled ${counts["disable-module"]} module(s)`);
|
||||
if (counts["configure-module"]) parts.push(`Configured ${counts["configure-module"]} module(s)`);
|
||||
if (counts["generate-image"]) parts.push(`Generated ${counts["generate-image"]} image(s)`);
|
||||
if (counts["generate-video"]) parts.push(`Generated ${counts["generate-video"]} video(s)`);
|
||||
if (counts["query-content"]) parts.push(`Queried ${counts["query-content"]} module(s)`);
|
||||
if (counts["scaffold"]) parts.push(`Scaffolded ${counts["scaffold"]} setup(s)`);
|
||||
if (counts["batch"]) parts.push(`Batch: ${counts["batch"]} group(s)`);
|
||||
return parts.join(", ") || "";
|
||||
|
|
@ -128,6 +136,15 @@ export function detailedActionSummary(actions: MiAction[]): string[] {
|
|||
case "navigate":
|
||||
details.push(`Navigate to ${a.path}`);
|
||||
break;
|
||||
case "generate-image":
|
||||
details.push(`Generate image: "${a.prompt.slice(0, 60)}${a.prompt.length > 60 ? "…" : ""}"`);
|
||||
break;
|
||||
case "generate-video":
|
||||
details.push(`Generate video: "${a.prompt.slice(0, 60)}${a.prompt.length > 60 ? "…" : ""}"`);
|
||||
break;
|
||||
case "query-content":
|
||||
details.push(`Query ${a.module}: ${a.queryType}`);
|
||||
break;
|
||||
default:
|
||||
details.push(`${a.type}`);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1719,3 +1719,44 @@ export const notesModule: RSpaceModule = {
|
|||
{ label: "Create a Notebook", icon: "✏️", description: "Start writing from scratch", type: 'create', href: '/{space}/rnotes' },
|
||||
],
|
||||
};
|
||||
|
||||
// ── MI Integration ──
|
||||
|
||||
export interface MINoteItem {
|
||||
id: string;
|
||||
title: string;
|
||||
contentPlain: string;
|
||||
tags: string[];
|
||||
type: string;
|
||||
updatedAt: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read recent notes directly from Automerge for the MI system prompt.
|
||||
*/
|
||||
export function getRecentNotesForMI(space: string, limit = 3): MINoteItem[] {
|
||||
if (!_syncServer) return [];
|
||||
const allNotes: MINoteItem[] = [];
|
||||
const prefix = `${space}:notes:notebooks:`;
|
||||
|
||||
for (const docId of _syncServer.listDocs()) {
|
||||
if (!docId.startsWith(prefix)) continue;
|
||||
const doc = _syncServer.getDoc<NotebookDoc>(docId);
|
||||
if (!doc?.items) continue;
|
||||
|
||||
for (const item of Object.values(doc.items)) {
|
||||
allNotes.push({
|
||||
id: item.id,
|
||||
title: item.title,
|
||||
contentPlain: (item.contentPlain || "").slice(0, 300),
|
||||
tags: item.tags ? Array.from(item.tags) : [],
|
||||
type: item.type,
|
||||
updatedAt: item.updatedAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return allNotes
|
||||
.sort((a, b) => b.updatedAt - a.updatedAt)
|
||||
.slice(0, limit);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -828,3 +828,45 @@ export const tasksModule: RSpaceModule = {
|
|||
{ label: "Create a Taskboard", icon: "📋", description: "Start a new kanban project board", type: 'create', href: '/{space}/rtasks' },
|
||||
],
|
||||
};
|
||||
|
||||
// ── MI Integration ──
|
||||
|
||||
export interface MITaskItem {
|
||||
id: string;
|
||||
title: string;
|
||||
status: string;
|
||||
priority: string | null;
|
||||
description: string;
|
||||
createdAt: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read recent/open tasks directly from Automerge for the MI system prompt.
|
||||
*/
|
||||
export function getRecentTasksForMI(space: string, limit = 5): MITaskItem[] {
|
||||
if (!_syncServer) return [];
|
||||
const allTasks: MITaskItem[] = [];
|
||||
|
||||
for (const docId of _syncServer.getDocIds()) {
|
||||
if (!docId.startsWith(`${space}:tasks:boards:`)) continue;
|
||||
const doc = _syncServer.getDoc<BoardDoc>(docId);
|
||||
if (!doc?.tasks) continue;
|
||||
|
||||
for (const task of Object.values(doc.tasks)) {
|
||||
allTasks.push({
|
||||
id: task.id,
|
||||
title: task.title,
|
||||
status: task.status,
|
||||
priority: task.priority,
|
||||
description: (task.description || "").slice(0, 200),
|
||||
createdAt: task.createdAt,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Prioritize non-DONE tasks, then sort by creation date
|
||||
return allTasks
|
||||
.filter((t) => t.status !== "DONE")
|
||||
.sort((a, b) => b.createdAt - a.createdAt)
|
||||
.slice(0, limit);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1182,47 +1182,15 @@ async function saveDataUrlToDisk(dataUrl: string, prefix: string): Promise<strin
|
|||
return `/data/files/generated/${filename}`;
|
||||
}
|
||||
|
||||
// Image generation via fal.ai Flux Pro
|
||||
// Image generation via fal.ai Flux Pro (delegates to shared helper)
|
||||
app.post("/api/image-gen", async (c) => {
|
||||
if (!FAL_KEY) return c.json({ error: "FAL_KEY not configured" }, 503);
|
||||
|
||||
const { prompt, style } = await c.req.json();
|
||||
if (!prompt) return c.json({ error: "prompt required" }, 400);
|
||||
|
||||
const stylePrompts: Record<string, string> = {
|
||||
illustration: "digital illustration style, ",
|
||||
photorealistic: "photorealistic, high detail, ",
|
||||
painting: "oil painting style, artistic, ",
|
||||
sketch: "pencil sketch style, hand-drawn, ",
|
||||
"punk-zine": "punk zine aesthetic, cut-and-paste collage, bold contrast, ",
|
||||
};
|
||||
const styledPrompt = (stylePrompts[style] || "") + prompt;
|
||||
|
||||
const res = await fetch("https://fal.run/fal-ai/flux-pro/v1.1", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Key ${FAL_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
prompt: styledPrompt,
|
||||
image_size: "landscape_4_3",
|
||||
num_images: 1,
|
||||
safety_tolerance: "2",
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const err = await res.text();
|
||||
console.error("[image-gen] fal.ai error:", err);
|
||||
return c.json({ error: "Image generation failed" }, 502);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
const imageUrl = data.images?.[0]?.url || data.output?.url;
|
||||
if (!imageUrl) return c.json({ error: "No image returned" }, 502);
|
||||
|
||||
return c.json({ url: imageUrl, image_url: imageUrl });
|
||||
const { generateImageViaFal } = await import("./mi-media");
|
||||
const result = await generateImageViaFal(prompt, style);
|
||||
if (!result.ok) return c.json({ error: result.error }, 502);
|
||||
return c.json({ url: result.url, image_url: result.url });
|
||||
});
|
||||
|
||||
// Upload image (data URL → disk)
|
||||
|
|
@ -1438,37 +1406,15 @@ app.post("/api/image-gen/img2img", async (c) => {
|
|||
return c.json({ error: `Unknown provider: ${provider}` }, 400);
|
||||
});
|
||||
|
||||
// Text-to-video via fal.ai WAN 2.1
|
||||
// Text-to-video via fal.ai WAN 2.1 (delegates to shared helper)
|
||||
app.post("/api/video-gen/t2v", async (c) => {
|
||||
if (!FAL_KEY) return c.json({ error: "FAL_KEY not configured" }, 503);
|
||||
|
||||
const { prompt, duration } = await c.req.json();
|
||||
const { prompt } = await c.req.json();
|
||||
if (!prompt) return c.json({ error: "prompt required" }, 400);
|
||||
|
||||
const res = await fetch("https://fal.run/fal-ai/wan/v2.1", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Key ${FAL_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
prompt,
|
||||
num_frames: duration === "5s" ? 81 : 49,
|
||||
resolution: "480p",
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const err = await res.text();
|
||||
console.error("[video-gen/t2v] fal.ai error:", err);
|
||||
return c.json({ error: "Video generation failed" }, 502);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
const videoUrl = data.video?.url || data.output?.url;
|
||||
if (!videoUrl) return c.json({ error: "No video returned" }, 502);
|
||||
|
||||
return c.json({ url: videoUrl, video_url: videoUrl });
|
||||
const { generateVideoViaFal } = await import("./mi-media");
|
||||
const result = await generateVideoViaFal(prompt);
|
||||
if (!result.ok) return c.json({ error: result.error }, 502);
|
||||
return c.json({ url: result.url, video_url: result.url });
|
||||
});
|
||||
|
||||
// Image-to-video via fal.ai Kling
|
||||
|
|
|
|||
|
|
@ -0,0 +1,201 @@
|
|||
/**
|
||||
* MI Agentic Loop — multi-turn streaming orchestrator.
|
||||
*
|
||||
* Streams LLM output, detects [MI_ACTION:{...}] markers, executes
|
||||
* server-side actions (media gen, data queries), and feeds results
|
||||
* back for the next LLM turn. Client-side actions (canvas shapes)
|
||||
* pass through unchanged in the stream.
|
||||
*/
|
||||
|
||||
import type { MiProvider, MiMessage, MiStreamChunk } from "./mi-provider";
|
||||
import { parseMiActions } from "../lib/mi-actions";
|
||||
import type { MiAction } from "../lib/mi-actions";
|
||||
import { generateImage, generateVideoViaFal } from "./mi-media";
|
||||
import { queryModuleContent } from "./mi-data-queries";
|
||||
|
||||
export interface AgenticLoopOptions {
|
||||
messages: MiMessage[];
|
||||
provider: MiProvider;
|
||||
providerModel: string;
|
||||
space: string;
|
||||
maxTurns?: number;
|
||||
}
|
||||
|
||||
/** NDJSON line types emitted by the agentic loop. */
|
||||
interface TurnLine { type: "turn"; turn: number; maxTurns: number }
|
||||
interface ActionStartLine { type: "action-start"; action: { type: string; ref?: string } }
|
||||
interface ActionResultLine {
|
||||
type: "action-result";
|
||||
actionType: string;
|
||||
ok: boolean;
|
||||
url?: string;
|
||||
data?: any;
|
||||
summary?: string;
|
||||
error?: string;
|
||||
ref?: string;
|
||||
}
|
||||
interface MessageLine { message: { role: string; content: string }; done: boolean }
|
||||
|
||||
type NDJSONLine = TurnLine | ActionStartLine | ActionResultLine | MessageLine;
|
||||
|
||||
const SERVER_ACTION_TYPES = new Set(["generate-image", "generate-video", "query-content"]);
|
||||
|
||||
function isServerAction(action: MiAction): boolean {
|
||||
return SERVER_ACTION_TYPES.has(action.type);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a single server-side MI action.
|
||||
*/
|
||||
async function executeServerAction(
|
||||
action: MiAction,
|
||||
space: string,
|
||||
): Promise<ActionResultLine> {
|
||||
const ref = "ref" in action ? (action as any).ref : undefined;
|
||||
|
||||
switch (action.type) {
|
||||
case "generate-image": {
|
||||
const result = await generateImage(action.prompt, action.style);
|
||||
if (result.ok) {
|
||||
return { type: "action-result", actionType: "generate-image", ok: true, url: result.url, ref };
|
||||
}
|
||||
return { type: "action-result", actionType: "generate-image", ok: false, error: result.error, ref };
|
||||
}
|
||||
|
||||
case "generate-video": {
|
||||
const result = await generateVideoViaFal(action.prompt, action.source_image);
|
||||
if (result.ok) {
|
||||
return { type: "action-result", actionType: "generate-video", ok: true, url: result.url, ref };
|
||||
}
|
||||
return { type: "action-result", actionType: "generate-video", ok: false, error: result.error, ref };
|
||||
}
|
||||
|
||||
case "query-content": {
|
||||
const result = queryModuleContent(space, action.module, action.queryType, action.limit);
|
||||
return {
|
||||
type: "action-result",
|
||||
actionType: "query-content",
|
||||
ok: result.ok,
|
||||
data: result.data,
|
||||
summary: result.summary,
|
||||
ref,
|
||||
};
|
||||
}
|
||||
|
||||
default:
|
||||
return { type: "action-result", actionType: (action as any).type, ok: false, error: "Unknown server action", ref };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Run the agentic loop. Returns a ReadableStream of NDJSON lines.
|
||||
*
|
||||
* Each turn: stream LLM → accumulate text → parse actions →
|
||||
* execute server-side actions → feed results back → next turn.
|
||||
* Client-side actions pass through in the text stream.
|
||||
*/
|
||||
export function runAgenticLoop(opts: AgenticLoopOptions): ReadableStream {
|
||||
const { messages, provider, providerModel, space, maxTurns = 5 } = opts;
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
// Working copy of conversation
|
||||
const conversation = [...messages];
|
||||
|
||||
return new ReadableStream({
|
||||
async start(controller) {
|
||||
try {
|
||||
for (let turn = 0; turn < maxTurns; turn++) {
|
||||
// Emit turn indicator (skip turn 0 for backwards compat)
|
||||
if (turn > 0) {
|
||||
emit(controller, encoder, { type: "turn", turn, maxTurns });
|
||||
}
|
||||
|
||||
// Stream LLM response
|
||||
let fullText = "";
|
||||
const gen = provider.stream(conversation, providerModel);
|
||||
|
||||
for await (const chunk of gen) {
|
||||
if (chunk.content) {
|
||||
fullText += chunk.content;
|
||||
// Stream text to client in real-time (Ollama NDJSON format)
|
||||
emit(controller, encoder, {
|
||||
message: { role: "assistant", content: chunk.content },
|
||||
done: false,
|
||||
});
|
||||
}
|
||||
if (chunk.done) break;
|
||||
}
|
||||
|
||||
// Parse actions from accumulated text
|
||||
const { actions } = parseMiActions(fullText);
|
||||
const serverActions = actions.filter(isServerAction);
|
||||
|
||||
// If no server-side actions, we're done
|
||||
if (serverActions.length === 0) {
|
||||
emit(controller, encoder, {
|
||||
message: { role: "assistant", content: "" },
|
||||
done: true,
|
||||
});
|
||||
controller.close();
|
||||
return;
|
||||
}
|
||||
|
||||
// Execute server-side actions
|
||||
const resultSummaries: string[] = [];
|
||||
for (const action of serverActions) {
|
||||
// Notify client that action is starting
|
||||
emit(controller, encoder, {
|
||||
type: "action-start",
|
||||
action: { type: action.type, ref: "ref" in action ? (action as any).ref : undefined },
|
||||
});
|
||||
|
||||
const result = await executeServerAction(action, space);
|
||||
emit(controller, encoder, result);
|
||||
|
||||
// Build summary for next LLM turn
|
||||
if (result.ok) {
|
||||
if (result.url) {
|
||||
resultSummaries.push(`[${result.actionType}] Generated: ${result.url}`);
|
||||
} else if (result.summary) {
|
||||
resultSummaries.push(`[${result.actionType}] ${result.summary}`);
|
||||
}
|
||||
} else {
|
||||
resultSummaries.push(`[${result.actionType}] Failed: ${result.error}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Add assistant message + action results to conversation for next turn
|
||||
conversation.push({ role: "assistant", content: fullText });
|
||||
conversation.push({
|
||||
role: "user",
|
||||
content: `Action results:\n${resultSummaries.join("\n")}\n\nIncorporate these results and continue. If you generated an image or video, you can now create a canvas shape referencing the URL. Do not re-generate media that already succeeded.`,
|
||||
});
|
||||
}
|
||||
|
||||
// Max turns reached
|
||||
emit(controller, encoder, {
|
||||
message: { role: "assistant", content: "" },
|
||||
done: true,
|
||||
});
|
||||
controller.close();
|
||||
} catch (err: any) {
|
||||
// Emit error as a message so client can display it
|
||||
try {
|
||||
emit(controller, encoder, {
|
||||
message: { role: "assistant", content: `\n\n*Error: ${err.message}*` },
|
||||
done: true,
|
||||
});
|
||||
} catch { /* controller may be closed */ }
|
||||
controller.close();
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function emit(
|
||||
controller: ReadableStreamDefaultController,
|
||||
encoder: TextEncoder,
|
||||
data: NDJSONLine,
|
||||
): void {
|
||||
controller.enqueue(encoder.encode(JSON.stringify(data) + "\n"));
|
||||
}
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
/**
|
||||
* MI Data Queries — dispatches content queries to module-specific readers.
|
||||
*
|
||||
* Used by the agentic loop to fetch module data server-side and feed
|
||||
* results back into the LLM context.
|
||||
*/
|
||||
|
||||
import { getUpcomingEventsForMI } from "../modules/rcal/mod";
|
||||
import { getRecentNotesForMI } from "../modules/rnotes/mod";
|
||||
import { getRecentTasksForMI } from "../modules/rtasks/mod";
|
||||
|
||||
export interface MiQueryResult {
|
||||
ok: boolean;
|
||||
module: string;
|
||||
queryType: string;
|
||||
data: any;
|
||||
summary: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Query module content by type. Returns structured data + a text summary
|
||||
* the LLM can consume.
|
||||
*/
|
||||
export function queryModuleContent(
|
||||
space: string,
|
||||
module: string,
|
||||
queryType: "recent" | "summary" | "count",
|
||||
limit = 5,
|
||||
): MiQueryResult {
|
||||
switch (module) {
|
||||
case "rnotes": {
|
||||
const notes = getRecentNotesForMI(space, limit);
|
||||
if (queryType === "count") {
|
||||
return { ok: true, module, queryType, data: { count: notes.length }, summary: `${notes.length} recent notes found.` };
|
||||
}
|
||||
const lines = notes.map((n) => `- "${n.title}" (${n.type}, updated ${new Date(n.updatedAt).toLocaleDateString()})${n.tags.length ? ` [${n.tags.join(", ")}]` : ""}: ${n.contentPlain.slice(0, 100)}...`);
|
||||
return { ok: true, module, queryType, data: notes, summary: lines.length ? `Recent notes:\n${lines.join("\n")}` : "No notes found." };
|
||||
}
|
||||
|
||||
case "rtasks": {
|
||||
const tasks = getRecentTasksForMI(space, limit);
|
||||
if (queryType === "count") {
|
||||
return { ok: true, module, queryType, data: { count: tasks.length }, summary: `${tasks.length} open tasks found.` };
|
||||
}
|
||||
const lines = tasks.map((t) => `- "${t.title}" [${t.status}]${t.priority ? ` (${t.priority})` : ""}${t.description ? `: ${t.description.slice(0, 80)}` : ""}`);
|
||||
return { ok: true, module, queryType, data: tasks, summary: lines.length ? `Open tasks:\n${lines.join("\n")}` : "No open tasks." };
|
||||
}
|
||||
|
||||
case "rcal": {
|
||||
const events = getUpcomingEventsForMI(space, 14, limit);
|
||||
if (queryType === "count") {
|
||||
return { ok: true, module, queryType, data: { count: events.length }, summary: `${events.length} upcoming events.` };
|
||||
}
|
||||
const lines = events.map((e) => {
|
||||
const date = e.allDay ? e.start.split("T")[0] : e.start;
|
||||
let line = `- ${date}: ${e.title}`;
|
||||
if (e.location) line += ` (${e.location})`;
|
||||
return line;
|
||||
});
|
||||
return { ok: true, module, queryType, data: events, summary: lines.length ? `Upcoming events:\n${lines.join("\n")}` : "No upcoming events." };
|
||||
}
|
||||
|
||||
default:
|
||||
return { ok: false, module, queryType, data: null, summary: `Module "${module}" does not support content queries.` };
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,202 @@
|
|||
/**
|
||||
* MI Media Generation — shared helpers for image and video generation.
|
||||
*
|
||||
* Extracted from server/index.ts endpoints so the agentic loop can
|
||||
* call them directly without HTTP round-trips.
|
||||
*/
|
||||
|
||||
import { resolve } from "path";
|
||||
|
||||
const FAL_KEY = process.env.FAL_KEY || "";
|
||||
const GEMINI_API_KEY = process.env.GEMINI_API_KEY || "";
|
||||
|
||||
const STYLE_PROMPTS: Record<string, string> = {
|
||||
illustration: "digital illustration style, ",
|
||||
photorealistic: "photorealistic, high detail, ",
|
||||
painting: "oil painting style, artistic, ",
|
||||
sketch: "pencil sketch style, hand-drawn, ",
|
||||
"punk-zine": "punk zine aesthetic, cut-and-paste collage, bold contrast, ",
|
||||
};
|
||||
|
||||
const GEMINI_STYLE_HINTS: Record<string, string> = {
|
||||
photorealistic: "photorealistic, high detail, natural lighting, ",
|
||||
illustration: "digital illustration, clean lines, vibrant colors, ",
|
||||
painting: "oil painting style, brushstrokes visible, painterly, ",
|
||||
sketch: "pencil sketch, hand-drawn, line art, ",
|
||||
"punk-zine": "punk zine aesthetic, xerox texture, high contrast, DIY, rough edges, ",
|
||||
collage: "cut-and-paste collage, mixed media, layered paper textures, ",
|
||||
vintage: "vintage aesthetic, retro colors, aged paper texture, ",
|
||||
minimalist: "minimalist design, simple shapes, limited color palette, ",
|
||||
};
|
||||
|
||||
export interface MediaResult {
|
||||
ok: true;
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface MediaError {
|
||||
ok: false;
|
||||
error: string;
|
||||
}
|
||||
|
||||
export type MediaOutcome = MediaResult | MediaError;
|
||||
|
||||
/**
|
||||
* Generate an image via fal.ai Flux Pro.
|
||||
*/
|
||||
export async function generateImageViaFal(prompt: string, style?: string): Promise<MediaOutcome> {
|
||||
if (!FAL_KEY) return { ok: false, error: "FAL_KEY not configured" };
|
||||
|
||||
const styledPrompt = (style && STYLE_PROMPTS[style] || "") + prompt;
|
||||
|
||||
const res = await fetch("https://fal.run/fal-ai/flux-pro/v1.1", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Key ${FAL_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
prompt: styledPrompt,
|
||||
image_size: "landscape_4_3",
|
||||
num_images: 1,
|
||||
safety_tolerance: "2",
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const err = await res.text();
|
||||
console.error("[mi-media] fal.ai image error:", err);
|
||||
return { ok: false, error: "Image generation failed" };
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
const imageUrl = data.images?.[0]?.url || data.output?.url;
|
||||
if (!imageUrl) return { ok: false, error: "No image returned" };
|
||||
|
||||
return { ok: true, url: imageUrl };
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate an image via Gemini (gemini-2.5-flash-image or imagen-3.0).
|
||||
*/
|
||||
export async function generateImageViaGemini(prompt: string, style?: string): Promise<MediaOutcome> {
|
||||
if (!GEMINI_API_KEY) return { ok: false, error: "GEMINI_API_KEY not configured" };
|
||||
|
||||
const enhancedPrompt = (style && GEMINI_STYLE_HINTS[style] || "") + prompt;
|
||||
const { GoogleGenAI } = await import("@google/genai");
|
||||
const ai = new GoogleGenAI({ apiKey: GEMINI_API_KEY });
|
||||
|
||||
const models = ["gemini-2.5-flash-image", "imagen-3.0-generate-002"];
|
||||
for (const modelName of models) {
|
||||
try {
|
||||
if (modelName.startsWith("gemini")) {
|
||||
const result = await ai.models.generateContent({
|
||||
model: modelName,
|
||||
contents: enhancedPrompt,
|
||||
config: { responseModalities: ["Text", "Image"] },
|
||||
});
|
||||
|
||||
const parts = result.candidates?.[0]?.content?.parts || [];
|
||||
for (const part of parts) {
|
||||
if ((part as any).inlineData) {
|
||||
const { data: b64, mimeType } = (part as any).inlineData;
|
||||
const ext = mimeType?.includes("png") ? "png" : "jpg";
|
||||
const filename = `gemini-${Date.now()}.${ext}`;
|
||||
const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
|
||||
await Bun.write(resolve(dir, filename), Buffer.from(b64, "base64"));
|
||||
return { ok: true, url: `/data/files/generated/${filename}` };
|
||||
}
|
||||
}
|
||||
} else {
|
||||
const result = await ai.models.generateImages({
|
||||
model: modelName,
|
||||
prompt: enhancedPrompt,
|
||||
config: { numberOfImages: 1, aspectRatio: "3:4" },
|
||||
});
|
||||
const img = (result as any).generatedImages?.[0];
|
||||
if (img?.image?.imageBytes) {
|
||||
const filename = `imagen-${Date.now()}.png`;
|
||||
const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
|
||||
await Bun.write(resolve(dir, filename), Buffer.from(img.image.imageBytes, "base64"));
|
||||
return { ok: true, url: `/data/files/generated/${filename}` };
|
||||
}
|
||||
}
|
||||
} catch (e: any) {
|
||||
console.error(`[mi-media] ${modelName} error:`, e.message);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return { ok: false, error: "All Gemini image models failed" };
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a text-to-video via fal.ai WAN 2.1.
|
||||
*/
|
||||
export async function generateVideoViaFal(prompt: string, source_image?: string): Promise<MediaOutcome> {
|
||||
if (!FAL_KEY) return { ok: false, error: "FAL_KEY not configured" };
|
||||
|
||||
if (source_image) {
|
||||
// Image-to-video via Kling
|
||||
const res = await fetch("https://fal.run/fal-ai/kling-video/v1/standard/image-to-video", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Key ${FAL_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
image_url: source_image,
|
||||
prompt: prompt || "",
|
||||
duration: "5",
|
||||
aspect_ratio: "16:9",
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const err = await res.text();
|
||||
console.error("[mi-media] fal.ai i2v error:", err);
|
||||
return { ok: false, error: "Video generation failed" };
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
const videoUrl = data.video?.url || data.output?.url;
|
||||
if (!videoUrl) return { ok: false, error: "No video returned" };
|
||||
return { ok: true, url: videoUrl };
|
||||
}
|
||||
|
||||
// Text-to-video via WAN 2.1
|
||||
const res = await fetch("https://fal.run/fal-ai/wan/v2.1", {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Authorization: `Key ${FAL_KEY}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({
|
||||
prompt,
|
||||
num_frames: 49,
|
||||
resolution: "480p",
|
||||
}),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const err = await res.text();
|
||||
console.error("[mi-media] fal.ai t2v error:", err);
|
||||
return { ok: false, error: "Video generation failed" };
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
const videoUrl = data.video?.url || data.output?.url;
|
||||
if (!videoUrl) return { ok: false, error: "No video returned" };
|
||||
|
||||
return { ok: true, url: videoUrl };
|
||||
}
|
||||
|
||||
/**
|
||||
* Try fal.ai first, fall back to Gemini for image generation.
|
||||
*/
|
||||
export async function generateImage(prompt: string, style?: string): Promise<MediaOutcome> {
|
||||
const falResult = await generateImageViaFal(prompt, style);
|
||||
if (falResult.ok) return falResult;
|
||||
|
||||
return generateImageViaGemini(prompt, style);
|
||||
}
|
||||
|
|
@ -45,6 +45,8 @@ const MODEL_REGISTRY: MiModelConfig[] = [
|
|||
{ id: "llama3.1", provider: "ollama", providerModel: "llama3.1:8b", label: "Llama 3.1 (8B)", group: "Local" },
|
||||
{ id: "qwen2.5-coder", provider: "ollama", providerModel: "qwen2.5-coder:7b", label: "Qwen Coder", group: "Local" },
|
||||
{ id: "mistral-small", provider: "ollama", providerModel: "mistral-small:24b", label: "Mistral Small", group: "Local" },
|
||||
{ id: "claude-sonnet", provider: "litellm", providerModel: "claude-sonnet", label: "Claude Sonnet", group: "Claude" },
|
||||
{ id: "claude-haiku", provider: "litellm", providerModel: "claude-haiku", label: "Claude Haiku", group: "Claude" },
|
||||
];
|
||||
|
||||
// ── Ollama Provider ──
|
||||
|
|
@ -160,31 +162,79 @@ class GeminiProvider implements MiProvider {
|
|||
}
|
||||
}
|
||||
|
||||
// ── Anthropic Provider (stub) ──
|
||||
// ── LiteLLM Provider (OpenAI-compatible SSE via LiteLLM proxy) ──
|
||||
|
||||
class AnthropicProvider implements MiProvider {
|
||||
id = "anthropic";
|
||||
class LiteLLMProvider implements MiProvider {
|
||||
id = "litellm";
|
||||
#url: string;
|
||||
#apiKey: string;
|
||||
|
||||
constructor() {
|
||||
this.#url = process.env.LITELLM_URL || "https://llm.jeffemmett.com";
|
||||
this.#apiKey = process.env.LITELLM_API_KEY || "";
|
||||
}
|
||||
|
||||
isAvailable(): boolean {
|
||||
return !!(process.env.ANTHROPIC_API_KEY);
|
||||
return !!(this.#url && this.#apiKey);
|
||||
}
|
||||
|
||||
async *stream(_messages: MiMessage[], _model: string): AsyncGenerator<MiStreamChunk> {
|
||||
throw new Error("Anthropic provider not yet implemented — add @anthropic-ai/sdk");
|
||||
}
|
||||
}
|
||||
async *stream(messages: MiMessage[], model: string): AsyncGenerator<MiStreamChunk> {
|
||||
const res = await fetch(`${this.#url}/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"Content-Type": "application/json",
|
||||
Authorization: `Bearer ${this.#apiKey}`,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
messages: messages.map((m) => ({ role: m.role, content: m.content })),
|
||||
stream: true,
|
||||
}),
|
||||
});
|
||||
|
||||
// ── OpenAI-Compatible Provider (stub) ──
|
||||
|
||||
class OpenAICompatProvider implements MiProvider {
|
||||
id = "openai-compat";
|
||||
|
||||
isAvailable(): boolean {
|
||||
return !!(process.env.OPENAI_COMPAT_URL && process.env.OPENAI_COMPAT_KEY);
|
||||
if (!res.ok) {
|
||||
const errText = await res.text().catch(() => "");
|
||||
throw new Error(`LiteLLM error ${res.status}: ${errText}`);
|
||||
}
|
||||
|
||||
async *stream(_messages: MiMessage[], _model: string): AsyncGenerator<MiStreamChunk> {
|
||||
throw new Error("OpenAI-compatible provider not yet implemented");
|
||||
if (!res.body) throw new Error("No response body from LiteLLM");
|
||||
|
||||
const reader = res.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
const lines = buffer.split("\n");
|
||||
buffer = lines.pop() || "";
|
||||
|
||||
for (const line of lines) {
|
||||
const trimmed = line.trim();
|
||||
if (!trimmed || !trimmed.startsWith("data: ")) continue;
|
||||
const payload = trimmed.slice(6);
|
||||
if (payload === "[DONE]") {
|
||||
yield { content: "", done: true };
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const data = JSON.parse(payload);
|
||||
const delta = data.choices?.[0]?.delta;
|
||||
const finishReason = data.choices?.[0]?.finish_reason;
|
||||
if (delta?.content) {
|
||||
yield { content: delta.content, done: false };
|
||||
}
|
||||
if (finishReason === "stop") {
|
||||
yield { content: "", done: true };
|
||||
return;
|
||||
}
|
||||
} catch {
|
||||
// skip malformed SSE lines
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -196,8 +246,7 @@ export class MiProviderRegistry {
|
|||
constructor() {
|
||||
this.register(new OllamaProvider());
|
||||
this.register(new GeminiProvider());
|
||||
this.register(new AnthropicProvider());
|
||||
this.register(new OpenAICompatProvider());
|
||||
this.register(new LiteLLMProvider());
|
||||
}
|
||||
|
||||
register(provider: MiProvider): void {
|
||||
|
|
|
|||
|
|
@ -18,6 +18,11 @@ import type { EncryptIDClaims } from "./auth";
|
|||
import { buildModuleCapabilities, MODULE_ROUTES } from "../lib/mi-module-routes";
|
||||
import type { MiAction } from "../lib/mi-actions";
|
||||
import { getUpcomingEventsForMI } from "../modules/rcal/mod";
|
||||
import { getRecentNotesForMI } from "../modules/rnotes/mod";
|
||||
import { getRecentTasksForMI } from "../modules/rtasks/mod";
|
||||
import { runAgenticLoop } from "./mi-agent";
|
||||
import { generateImage, generateVideoViaFal } from "./mi-media";
|
||||
import { queryModuleContent } from "./mi-data-queries";
|
||||
|
||||
const mi = new Hono();
|
||||
|
||||
|
|
@ -135,6 +140,8 @@ mi.post("/ask", async (c) => {
|
|||
const timeContext = `${weekdays[now.getUTCDay()]}, ${now.toISOString().split("T")[0]} at ${now.toISOString().split("T")[1].split(".")[0]} UTC`;
|
||||
|
||||
let calendarContext = "";
|
||||
let notesContext = "";
|
||||
let tasksContext = "";
|
||||
if (space) {
|
||||
const upcoming = getUpcomingEventsForMI(space);
|
||||
if (upcoming.length > 0) {
|
||||
|
|
@ -148,6 +155,22 @@ mi.post("/ask", async (c) => {
|
|||
});
|
||||
calendarContext = `\n- Upcoming events (next 14 days):\n${lines.join("\n")}`;
|
||||
}
|
||||
|
||||
const recentNotes = getRecentNotesForMI(space, 3);
|
||||
if (recentNotes.length > 0) {
|
||||
const lines = recentNotes.map((n) =>
|
||||
`- "${n.title}" (${n.type}${n.tags.length ? `, tags: ${n.tags.join(", ")}` : ""}): ${n.contentPlain.slice(0, 100)}…`
|
||||
);
|
||||
notesContext = `\n- Recent notes:\n${lines.join("\n")}`;
|
||||
}
|
||||
|
||||
const openTasks = getRecentTasksForMI(space, 5);
|
||||
if (openTasks.length > 0) {
|
||||
const lines = openTasks.map((t) =>
|
||||
`- "${t.title}" [${t.status}]${t.priority ? ` (${t.priority})` : ""}${t.description ? `: ${t.description.slice(0, 80)}` : ""}`
|
||||
);
|
||||
tasksContext = `\n- Open tasks:\n${lines.join("\n")}`;
|
||||
}
|
||||
}
|
||||
|
||||
const systemPrompt = `You are mi (mycelial intelligence), the intelligent assistant for rSpace — a self-hosted, community-run platform.
|
||||
|
|
@ -173,7 +196,7 @@ ${moduleCapabilities}
|
|||
When the user asks to create a social media campaign, use create-content with module rsocials, contentType campaign, body.rawBrief set to the user's description, body.navigateToWizard true.
|
||||
|
||||
## Current Context
|
||||
${contextSection}${calendarContext}
|
||||
${contextSection}${calendarContext}${notesContext}${tasksContext}
|
||||
|
||||
## Guidelines
|
||||
- Be concise and helpful. Keep responses short (2-4 sentences) unless the user asks for detail.
|
||||
|
|
@ -216,6 +239,23 @@ When the user asks to create content in a specific rApp (not a canvas shape):
|
|||
For multi-step setup requests like "set up this space for a book club":
|
||||
[MI_ACTION:{"type":"scaffold","name":"Book Club Setup","steps":[...ordered actions...]}]
|
||||
|
||||
## Media Generation (server-side — MI will execute these and return the result URL)
|
||||
When the user asks you to generate an image or video, use these actions:
|
||||
[MI_ACTION:{"type":"generate-image","prompt":"a forest of glowing mushrooms at dusk","style":"illustration","ref":"$1"}]
|
||||
[MI_ACTION:{"type":"generate-video","prompt":"timelapse of mycelium growing through soil","ref":"$2"}]
|
||||
After the server generates the media, you will receive the URL in a follow-up message.
|
||||
Then create a canvas shape referencing that URL:
|
||||
[MI_ACTION:{"type":"create-shape","tagName":"folk-image-gen","props":{"src":"<the returned URL>"},"ref":"$3"}]
|
||||
Available styles: illustration, photorealistic, painting, sketch, punk-zine.
|
||||
|
||||
## Content Queries (server-side — MI will fetch and return results)
|
||||
When you need to look up the user's actual data (notes, tasks, events):
|
||||
[MI_ACTION:{"type":"query-content","module":"rnotes","queryType":"recent","limit":5}]
|
||||
[MI_ACTION:{"type":"query-content","module":"rtasks","queryType":"recent","limit":5}]
|
||||
[MI_ACTION:{"type":"query-content","module":"rcal","queryType":"recent","limit":5}]
|
||||
queryType can be: "recent", "summary", or "count".
|
||||
Results will be provided in a follow-up message for you to incorporate into your response.
|
||||
|
||||
## Batch Actions
|
||||
[MI_ACTION:{"type":"batch","actions":[...actions...],"requireConfirm":true}]
|
||||
Use requireConfirm:true for destructive batches.`;
|
||||
|
|
@ -228,8 +268,13 @@ Use requireConfirm:true for destructive batches.`;
|
|||
];
|
||||
|
||||
try {
|
||||
const gen = providerInfo.provider.stream(miMessages, providerInfo.providerModel);
|
||||
const body = miRegistry.streamToNDJSON(gen);
|
||||
const body = runAgenticLoop({
|
||||
messages: miMessages,
|
||||
provider: providerInfo.provider,
|
||||
providerModel: providerInfo.providerModel,
|
||||
space: space || "",
|
||||
maxTurns: 5,
|
||||
});
|
||||
|
||||
return new Response(body, {
|
||||
headers: {
|
||||
|
|
@ -377,8 +422,11 @@ function getRequiredRole(action: MiAction): SpaceRoleString {
|
|||
case "transform":
|
||||
case "scaffold":
|
||||
case "batch":
|
||||
case "generate-image":
|
||||
case "generate-video":
|
||||
return "member";
|
||||
case "navigate":
|
||||
case "query-content":
|
||||
return "viewer";
|
||||
default:
|
||||
return "member";
|
||||
|
|
@ -413,6 +461,30 @@ mi.post("/validate-actions", async (c) => {
|
|||
return c.json({ validated, callerRole });
|
||||
});
|
||||
|
||||
// ── POST /execute-server-action — client-side fallback for server actions ──
|
||||
|
||||
mi.post("/execute-server-action", async (c) => {
|
||||
const { action, space } = await c.req.json();
|
||||
if (!action?.type) return c.json({ error: "action required" }, 400);
|
||||
|
||||
switch (action.type) {
|
||||
case "generate-image": {
|
||||
const result = await generateImage(action.prompt, action.style);
|
||||
return c.json(result);
|
||||
}
|
||||
case "generate-video": {
|
||||
const result = await generateVideoViaFal(action.prompt, action.source_image);
|
||||
return c.json(result);
|
||||
}
|
||||
case "query-content": {
|
||||
const result = queryModuleContent(space || "", action.module, action.queryType, action.limit);
|
||||
return c.json(result);
|
||||
}
|
||||
default:
|
||||
return c.json({ ok: false, error: `Unknown server action: ${action.type}` }, 400);
|
||||
}
|
||||
});
|
||||
|
||||
// ── Fallback response (when AI is unavailable) ──
|
||||
|
||||
function generateFallbackResponse(
|
||||
|
|
|
|||
|
|
@ -445,6 +445,8 @@ export class RStackMi extends HTMLElement {
|
|||
const reader = res.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
|
||||
const serverActionResults: { type: string; url?: string; ref?: string }[] = [];
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
|
@ -453,12 +455,36 @@ export class RStackMi extends HTMLElement {
|
|||
for (const line of chunk.split("\n").filter(Boolean)) {
|
||||
try {
|
||||
const data = JSON.parse(line);
|
||||
|
||||
// Standard message content (backwards-compatible)
|
||||
if (data.message?.content) {
|
||||
this.#messages[assistantIdx].content += data.message.content;
|
||||
}
|
||||
if (data.response) {
|
||||
this.#messages[assistantIdx].content = data.response;
|
||||
}
|
||||
|
||||
// Agentic loop: turn indicator
|
||||
if (data.type === "turn" && data.turn > 0) {
|
||||
this.#messages[assistantIdx].content += `\n\n*— thinking (turn ${data.turn + 1}/${data.maxTurns}) —*\n\n`;
|
||||
}
|
||||
|
||||
// Agentic loop: server action starting
|
||||
if (data.type === "action-start") {
|
||||
this.#messages[assistantIdx].content += `\n*⏳ ${data.action?.type}…*\n`;
|
||||
}
|
||||
|
||||
// Agentic loop: server action result
|
||||
if (data.type === "action-result") {
|
||||
if (data.ok && data.url) {
|
||||
serverActionResults.push({ type: data.actionType, url: data.url, ref: data.ref });
|
||||
this.#messages[assistantIdx].content += `\n*✓ ${data.actionType}: generated*\n`;
|
||||
} else if (data.ok && data.summary) {
|
||||
this.#messages[assistantIdx].content += `\n*✓ ${data.actionType}: ${data.summary.slice(0, 80)}*\n`;
|
||||
} else if (!data.ok) {
|
||||
this.#messages[assistantIdx].content += `\n*✗ ${data.actionType}: ${data.error || "failed"}*\n`;
|
||||
}
|
||||
}
|
||||
} catch { /* skip malformed lines */ }
|
||||
}
|
||||
this.#renderMessages(messagesEl);
|
||||
|
|
@ -468,7 +494,7 @@ export class RStackMi extends HTMLElement {
|
|||
this.#messages[assistantIdx].content = "I couldn't generate a response. Please try again.";
|
||||
this.#renderMessages(messagesEl);
|
||||
} else {
|
||||
// Parse and execute MI actions
|
||||
// Parse and execute MI actions (client-side canvas actions)
|
||||
const rawText = this.#messages[assistantIdx].content;
|
||||
const { displayText, actions } = parseMiActions(rawText);
|
||||
this.#messages[assistantIdx].content = displayText;
|
||||
|
|
@ -479,6 +505,23 @@ export class RStackMi extends HTMLElement {
|
|||
this.#messages[assistantIdx].actionDetails = detailedActionSummary(actions);
|
||||
}
|
||||
|
||||
// Create canvas shapes for server-generated media
|
||||
if (serverActionResults.length) {
|
||||
const mediaActions: MiAction[] = [];
|
||||
for (const r of serverActionResults) {
|
||||
if (r.type === "generate-image" && r.url) {
|
||||
mediaActions.push({ type: "create-shape", tagName: "folk-image-gen", props: { src: r.url }, ref: r.ref });
|
||||
} else if (r.type === "generate-video" && r.url) {
|
||||
mediaActions.push({ type: "create-shape", tagName: "folk-video-gen", props: { src: r.url }, ref: r.ref });
|
||||
}
|
||||
}
|
||||
if (mediaActions.length) {
|
||||
const executor = new MiActionExecutor();
|
||||
executor.setContext(context.space || "", getAccessToken() || "");
|
||||
executor.execute(mediaActions);
|
||||
}
|
||||
}
|
||||
|
||||
// Check for tool suggestions
|
||||
const hints = suggestTools(query);
|
||||
if (hints.length) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue