From 7805a1e961e5b58f88448c46820c328d92a57e37 Mon Sep 17 00:00:00 2001 From: Jeff-Emmett Date: Thu, 23 Jan 2025 22:38:27 +0100 Subject: [PATCH] working llm util --- package.json | 3 +- src/routes/Board.tsx | 170 ++++++++---------------- src/shapes/PromptShapeUtil.tsx | 165 +++++++++++++++++++++++ src/tools/PromptShapeTool.ts | 8 ++ src/ui/SettingsDialog.tsx | 233 ++++++--------------------------- src/ui/components.tsx | 106 ++++++++++++++- src/ui/overrides.tsx | 76 ++++++++++- src/utils/llm.ts | 34 +++++ 8 files changed, 477 insertions(+), 318 deletions(-) create mode 100644 src/shapes/PromptShapeUtil.tsx create mode 100644 src/tools/PromptShapeTool.ts create mode 100644 src/utils/llm.ts diff --git a/package.json b/package.json index 20684fd..2a011c4 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,8 @@ "dev:worker": "wrangler dev --local --port 5172 --ip 0.0.0.0", "build": "tsc && vite build", "preview": "vite preview", - "deploy": "tsc && vite build && vercel deploy --prod && wrangler deploy" + "deploy": "tsc && vite build && vercel deploy --prod && wrangler deploy", + "types": "tsc --noEmit" }, "keywords": [], "author": "Jeff Emmett", diff --git a/src/routes/Board.tsx b/src/routes/Board.tsx index 2afe4e0..8798dbd 100644 --- a/src/routes/Board.tsx +++ b/src/routes/Board.tsx @@ -1,6 +1,6 @@ import { useSync } from "@tldraw/sync" -import { useMemo } from "react" -import { Tldraw, Editor, useTools, useIsToolSelected, DefaultToolbar, TldrawUiMenuItem, DefaultToolbarContent, DefaultKeyboardShortcutsDialog, DefaultKeyboardShortcutsDialogContent, defaultTools } from "tldraw" +import { useMemo, useEffect, useState } from "react" +import { Tldraw, Editor } from "tldraw" import { useParams } from "react-router-dom" import { ChatBoxTool } from "@/tools/ChatBoxTool" import { ChatBoxShape } from "@/shapes/ChatBoxShapeUtil" @@ -12,130 +12,47 @@ import { EmbedTool } from "@/tools/EmbedTool" import { MarkdownShape } from "@/shapes/MarkdownShapeUtil" import { MarkdownTool } from "@/tools/MarkdownTool" import { defaultShapeUtils, defaultBindingUtils } from "tldraw" -import { useState } from "react" import { components } from "@/ui/components" import { overrides } from "@/ui/overrides" import { unfurlBookmarkUrl } from "../utils/unfurlBookmarkUrl" import { handleInitialPageLoad } from "@/utils/handleInitialPageLoad" import { MycrozineTemplateTool } from "@/tools/MycrozineTemplateTool" import { MycrozineTemplateShape } from "@/shapes/MycrozineTemplateShapeUtil" -import { registerPropagators, ChangePropagator, TickPropagator, ClickPropagator } from "@/propagators/ScopedPropagators" +import { + registerPropagators, + ChangePropagator, + TickPropagator, + ClickPropagator, +} from "@/propagators/ScopedPropagators" import { SlideShapeTool } from "@/tools/SlideShapeTool" -import { ISlideShape, SlideShapeUtil } from "@/shapes/SlideShapeUtil" -import { SlidesPanel } from "@/slides/SlidesPanel" -import { moveToSlide } from "@/slides/useSlides" +import { SlideShapeUtil } from "@/shapes/SlideShapeUtil" +import { makeRealSettings, applySettingsMigrations } from "@/lib/settings" +import { PromptShapeTool } from "@/tools/PromptShapeTool" +import { PromptShape } from "@/shapes/PromptShapeUtil" +import { llm } from "@/utils/llm" // Default to production URL if env var isn't available export const WORKER_URL = "https://jeffemmett-canvas.jeffemmett.workers.dev" -const updatedComponents = { - ...components, - HelperButtons: SlidesPanel, - Minimap: null, - Toolbar: (props: any) => { - const tools = useTools() - const slideTool = tools['Slide'] - const isSlideSelected = slideTool ? useIsToolSelected(slideTool) : false - return ( - - {slideTool && } - - - ) - }, - KeyboardShortcutsDialog: (props: any) => { - const tools = useTools() - return ( - - - - - ) - }, -} - const customShapeUtils = [ - ChatBoxShape, - VideoChatShape, - EmbedShape, + ChatBoxShape, + VideoChatShape, + EmbedShape, SlideShapeUtil, - MycrozineTemplateShape, - MarkdownShape + MycrozineTemplateShape, + MarkdownShape, + PromptShape, ] const customTools = [ - ChatBoxTool, - VideoChatTool, - EmbedTool, + ChatBoxTool, + VideoChatTool, + EmbedTool, SlideShapeTool, - MycrozineTemplateTool, - MarkdownTool + MycrozineTemplateTool, + MarkdownTool, + PromptShapeTool, ] -const updatedOverrides = { - ...overrides, - actions(editor: Editor, actions: any) { - return { - ...actions, - 'next-slide': { - id: 'next-slide', - label: 'Next slide', - kbd: 'right', - onSelect() { - const slides = editor.getCurrentPageShapes().filter(shape => shape.type === 'Slide') - if (slides.length === 0) return - - const currentSlide = editor.getSelectedShapes().find(shape => shape.type === 'Slide') - const currentIndex = currentSlide - ? slides.findIndex(slide => slide.id === currentSlide.id) - : -1 - -console.log('Current index:', currentIndex) -console.log('Current slide:', currentSlide) - - - // Calculate next index with wraparound - const nextIndex = currentIndex === -1 - ? 0 - : currentIndex >= slides.length - 1 - ? 0 - : currentIndex + 1 - - const nextSlide = slides[nextIndex] - - editor.select(nextSlide.id) - editor.stopCameraAnimation() - moveToSlide(editor, nextSlide as ISlideShape) - }, - }, - 'previous-slide': { - id: 'previous-slide', - label: 'Previous slide', - kbd: 'left', - onSelect() { - const slides = editor.getCurrentPageShapes().filter(shape => shape.type === 'Slide') - if (slides.length === 0) return - - const currentSlide = editor.getSelectedShapes().find(shape => shape.type === 'Slide') - const currentIndex = currentSlide - ? slides.findIndex(slide => slide.id === currentSlide.id) - : -1 - - // Calculate previous index with wraparound - const previousIndex = currentIndex <= 0 - ? slides.length - 1 - : currentIndex - 1 - - const previousSlide = slides[previousIndex] - - editor.select(previousSlide.id) - editor.stopCameraAnimation() - moveToSlide(editor, previousSlide as ISlideShape) - }, - }, - } - }, -} - export function Board() { const { slug } = useParams<{ slug: string }>() const roomId = slug || "default-room" @@ -153,8 +70,18 @@ export function Board() { const store = useSync(storeConfig) const [editor, setEditor] = useState(null) - //console.log("store:", store) - //console.log("store.store:",store.store) + useEffect(() => { + const value = localStorage.getItem("makereal_settings_2") + if (value) { + const json = JSON.parse(value) + const migratedSettings = applySettingsMigrations(json) + localStorage.setItem( + "makereal_settings_2", + JSON.stringify(migratedSettings), + ) + makeRealSettings.set(migratedSettings) + } + }, []) return (
@@ -162,11 +89,11 @@ export function Board() { store={store.store} shapeUtils={customShapeUtils} tools={customTools} - components={updatedComponents} - overrides={updatedOverrides} + components={components} + overrides={overrides} cameraOptions={{ zoomSteps: [ - 0.001, // Min zoom + 0.001, // Min zoom 0.0025, 0.005, 0.01, @@ -181,15 +108,26 @@ export function Board() { 8, 16, 32, - 64 // Max zoom - ] + 64, // Max zoom + ], }} onMount={(editor) => { setEditor(editor) editor.registerExternalAssetHandler("url", unfurlBookmarkUrl) editor.setCurrentTool("hand") handleInitialPageLoad(editor) - registerPropagators(editor, [TickPropagator,ChangePropagator,ClickPropagator]) + registerPropagators(editor, [ + TickPropagator, + ChangePropagator, + ClickPropagator, + ]) + llm( + "You are a helpful assistant.", + "Hello, how are you?", + (partialResponse, done) => { + console.log({ partialResponse, done }) + }, + ) }} />
diff --git a/src/shapes/PromptShapeUtil.tsx b/src/shapes/PromptShapeUtil.tsx new file mode 100644 index 0000000..531a3cf --- /dev/null +++ b/src/shapes/PromptShapeUtil.tsx @@ -0,0 +1,165 @@ +import { + BaseBoxShapeUtil, + HTMLContainer, + TLBaseShape, + TLGeoShape, + TLShape, +} from "tldraw" +import { getEdge } from "@/propagators/tlgraph" +import { llm } from "@/utils/llm" +import { isShapeOfType } from "@/propagators/utils" + +type IPrompt = TLBaseShape< + "prompt", + { + w: number + h: number + prompt: string + value: string + agentBinding: string | null + } +> + +export class PromptShape extends BaseBoxShapeUtil { + static override type = "prompt" as const + + FIXED_HEIGHT = 50 as const + MIN_WIDTH = 150 as const + PADDING = 4 as const + + getDefaultProps(): IPrompt["props"] { + return { + w: 300, + h: 50, + prompt: "", + value: "", + agentBinding: null, + } + } + + // override onResize: TLResizeHandle = ( + // shape, + // { scaleX, initialShape }, + // ) => { + // const { x, y } = shape + // const w = initialShape.props.w * scaleX + // return { + // x, + // y, + // props: { + // ...shape.props, + // w: Math.max(Math.abs(w), this.MIN_WIDTH), + // h: this.FIXED_HEIGHT, + // }, + // } + // } + + component(shape: IPrompt) { + const arrowBindings = this.editor.getBindingsInvolvingShape( + shape.id, + "arrow", + ) + const arrows = arrowBindings + .map((binding) => this.editor.getShape(binding.fromId)) + + const inputMap = arrows.reduce((acc, arrow) => { + const edge = getEdge(arrow, this.editor); + if (edge) { + const sourceShape = this.editor.getShape(edge.from); + if (sourceShape && edge.text) { + acc[edge.text] = sourceShape; + } + } + return acc; + }, {} as Record); + + const generateText = async (prompt: string) => { + await llm('', prompt, (partial: string, done: boolean) => { + console.log("DONE??", done) + this.editor.updateShape({ + id: shape.id, + type: "prompt", + props: { value: partial, agentBinding: done ? null : 'someone' }, + }) + }) + } + + const handlePrompt = () => { + if (shape.props.agentBinding) { + return + } + let processedPrompt = shape.props.prompt; + for (const [key, sourceShape] of Object.entries(inputMap)) { + const pattern = `{${key}}`; + if (processedPrompt.includes(pattern)) { + if (isShapeOfType(sourceShape, 'geo')) { + processedPrompt = processedPrompt.replace(pattern, sourceShape.props.text); + } + } + } + console.log(processedPrompt); + generateText(processedPrompt) + }; + + return ( + + { + this.editor.updateShape({ + id: shape.id, + type: "prompt", + props: { prompt: text.target.value }, + }) + }} + /> + + + ) + } + + // [5] + indicator(shape: IPrompt) { + return + } +} \ No newline at end of file diff --git a/src/tools/PromptShapeTool.ts b/src/tools/PromptShapeTool.ts new file mode 100644 index 0000000..51ba34f --- /dev/null +++ b/src/tools/PromptShapeTool.ts @@ -0,0 +1,8 @@ +import { BaseBoxShapeTool } from 'tldraw' + +export class PromptShapeTool extends BaseBoxShapeTool { + static override id = 'prompt' + static override initial = 'idle' + override shapeType = 'prompt' + +} \ No newline at end of file diff --git a/src/ui/SettingsDialog.tsx b/src/ui/SettingsDialog.tsx index 05108a1..df1bf92 100644 --- a/src/ui/SettingsDialog.tsx +++ b/src/ui/SettingsDialog.tsx @@ -1,198 +1,47 @@ import { - TLUiDialogProps, - TldrawUiButton, - TldrawUiButtonLabel, - TldrawUiDialogBody, - TldrawUiDialogCloseButton, - TldrawUiDialogFooter, - TldrawUiDialogHeader, - TldrawUiDialogTitle, - TldrawUiIcon, - TldrawUiInput, - useReactor, - useValue, -} from 'tldraw' -import { PROVIDERS, makeRealSettings } from '../lib/settings' -import { SYSTEM_PROMPT } from '@/prompt' + TLUiDialogProps, + TldrawUiButton, + TldrawUiButtonLabel, + TldrawUiDialogBody, + TldrawUiDialogCloseButton, + TldrawUiDialogFooter, + TldrawUiDialogHeader, + TldrawUiDialogTitle, + TldrawUiInput, +} from "tldraw" +import React from "react" export function SettingsDialog({ onClose }: TLUiDialogProps) { - const settings = useValue('settings', () => makeRealSettings.get(), []) + const [apiKey, setApiKey] = React.useState(() => { + return localStorage.getItem("openai_api_key") || "" + }) - useReactor( - 'update settings local storage', - () => { - localStorage.setItem('makereal_settings_2', JSON.stringify(makeRealSettings.get())) - }, - [] - ) + const handleChange = (value: string) => { + setApiKey(value) + localStorage.setItem("openai_api_key", value) + } - return ( - <> - - Settings - - - -

- To use Make Real, enter your API key for each model provider that you wish to use. Draw - some shapes, then select the shapes and click Make Real.{' '} - - Read our guide. - -

-
-
- -
- - {settings.provider !== 'all' && ( - <> -
- -
- - - )} -
-
- {PROVIDERS.map((provider) => { - if (provider.id === 'google') return null - const value = settings.keys[provider.id as keyof typeof settings.keys] - return ( - - ) - })} - {/*
-
- -
- { - const next = { ...settings, keys: { ...settings.keys, google: value } } - localStorage.setItem('makereal_settings_2', JSON.stringify(next)) - makeRealSettings.set(next) - }} - /> -
*/} -
-
-
- - -
- { - makeRealSettings.update((s) => ({ ...s, prompts: { ...s.prompts, system: value } })) - }} - /> -
- - - { - onClose() - }} - > - Save - - - - ) + return ( + <> + + API Keys + + + +
+ + +
+ + + + Close + + + + ) } - -function ApiKeyInput({ - provider, - value, - warning, -}: { - provider: (typeof PROVIDERS)[number] - value: string - warning: boolean -}) { - const isValid = value.length === 0 || provider.validate(value) - - return ( -
-
- - - - -
- { - makeRealSettings.update((s) => ({ ...s, keys: { ...s.keys, [provider.id]: value } })) - }} - /> -
- ) -} \ No newline at end of file diff --git a/src/ui/components.tsx b/src/ui/components.tsx index ed302a7..f48e51f 100644 --- a/src/ui/components.tsx +++ b/src/ui/components.tsx @@ -1,10 +1,112 @@ import { CustomMainMenu } from "./CustomMainMenu" import { CustomToolbar } from "./CustomToolbar" import { CustomContextMenu } from "./CustomContextMenu" -import { TLComponents } from "tldraw" +import { + DefaultKeyboardShortcutsDialog, + DefaultKeyboardShortcutsDialogContent, + DefaultToolbar, + DefaultToolbarContent, + TLComponents, + TldrawUiMenuItem, + useDialogs, + useIsToolSelected, + useTools, +} from "tldraw" +import { SettingsDialog } from "./SettingsDialog" +import { useEffect } from "react" +import { SlidesPanel } from "@/slides/SlidesPanel" +import { useState } from "react" export const components: TLComponents = { - Toolbar: CustomToolbar, + // Toolbar: CustomToolbar, MainMenu: CustomMainMenu, ContextMenu: CustomContextMenu, + HelperButtons: SlidesPanel, + Toolbar: (props: any) => { + const tools = useTools() + const slideTool = tools["Slide"] + const isSlideSelected = slideTool ? useIsToolSelected(slideTool) : false + const { addDialog, removeDialog } = useDialogs() + const [hasApiKey, setHasApiKey] = useState(false) + + useEffect(() => { + const key = localStorage.getItem("openai_api_key") + setHasApiKey(!!key) + }, []) + + return ( +
+
+ ( + + ) +
+ + {slideTool && ( + + )} + + +
+ ) + }, + KeyboardShortcutsDialog: (props: any) => { + const tools = useTools() + return ( + + + + + ) + }, } diff --git a/src/ui/overrides.tsx b/src/ui/overrides.tsx index 64b93b7..ffbdd1e 100644 --- a/src/ui/overrides.tsx +++ b/src/ui/overrides.tsx @@ -9,6 +9,8 @@ import { import { saveToPdf } from "../utils/pdfUtils" import { searchText } from "../utils/searchUtils" import { EmbedShape, IEmbedShape } from "@/shapes/EmbedShapeUtil" +import { moveToSlide } from "@/slides/useSlides" +import { ISlideShape } from "@/shapes/SlideShapeUtil" export const overrides: TLUiOverrides = { tools(editor, tools) { @@ -44,19 +46,19 @@ export const overrides: TLUiOverrides = { //TODO: Fix double click to zoom on selector tool later... onDoubleClick: (info: any) => { const shape = editor.getShapeAtPoint(info.point) - if (shape?.type === 'Embed') { + if (shape?.type === "Embed") { // Let the Embed shape handle its own double-click behavior const util = editor.getShapeUtil(shape) as EmbedShape util?.onDoubleClick?.(shape as IEmbedShape) return } - + // Handle all pointer types (mouse, touch, pen) const point = info.point || (info.touches && info.touches[0]) || info - + // Zoom in at the clicked/touched point editor.zoomIn(point, { animation: { duration: 200 } }) - + // Stop event propagation and prevent default handling info.stopPropagation?.() return false @@ -97,10 +99,10 @@ export const overrides: TLUiOverrides = { type: "Slide", readonlyOk: true, onSelect: () => { - console.log('SlideShape tool selected from menu') - console.log('Current tool before:', editor.getCurrentToolId()) + console.log("SlideShape tool selected from menu") + console.log("Current tool before:", editor.getCurrentToolId()) editor.setCurrentTool("Slide") - console.log('Current tool after:', editor.getCurrentToolId()) + console.log("Current tool after:", editor.getCurrentToolId()) }, }, Markdown: { @@ -370,6 +372,66 @@ export const overrides: TLUiOverrides = { readonlyOk: true, onSelect: () => searchText(editor), }, + "next-slide": { + id: "next-slide", + label: "Next slide", + kbd: "right", + onSelect() { + const slides = editor + .getCurrentPageShapes() + .filter((shape) => shape.type === "Slide") + if (slides.length === 0) return + + const currentSlide = editor + .getSelectedShapes() + .find((shape) => shape.type === "Slide") + const currentIndex = currentSlide + ? slides.findIndex((slide) => slide.id === currentSlide.id) + : -1 + + // Calculate next index with wraparound + const nextIndex = + currentIndex === -1 + ? 0 + : currentIndex >= slides.length - 1 + ? 0 + : currentIndex + 1 + + const nextSlide = slides[nextIndex] + + editor.select(nextSlide.id) + editor.stopCameraAnimation() + moveToSlide(editor, nextSlide as ISlideShape) + }, + }, + "previous-slide": { + id: "previous-slide", + label: "Previous slide", + kbd: "left", + onSelect() { + const slides = editor + .getCurrentPageShapes() + .filter((shape) => shape.type === "Slide") + if (slides.length === 0) return + + const currentSlide = editor + .getSelectedShapes() + .find((shape) => shape.type === "Slide") + const currentIndex = currentSlide + ? slides.findIndex((slide) => slide.id === currentSlide.id) + : -1 + + // Calculate previous index with wraparound + const previousIndex = + currentIndex <= 0 ? slides.length - 1 : currentIndex - 1 + + const previousSlide = slides[previousIndex] + + editor.select(previousSlide.id) + editor.stopCameraAnimation() + moveToSlide(editor, previousSlide as ISlideShape) + }, + }, } }, } diff --git a/src/utils/llm.ts b/src/utils/llm.ts new file mode 100644 index 0000000..2c09c31 --- /dev/null +++ b/src/utils/llm.ts @@ -0,0 +1,34 @@ +import OpenAI from "openai"; + +const apiKey = localStorage.getItem("openai_api_key") || "" +const openai = new OpenAI({ + apiKey, + dangerouslyAllowBrowser: true, +}); + +export async function llm( + systemPrompt: string, + userPrompt: string, + onToken: (partialResponse: string, done: boolean) => void, +) { + if (!apiKey) { + throw new Error("No API key found") + } + console.log("System Prompt:", systemPrompt); + console.log("User Prompt:", userPrompt); + let partial = ""; + const stream = await openai.chat.completions.create({ + model: "gpt-4o", + messages: [ + { role: "system", content: 'You are a helpful assistant.' }, + { role: "user", content: userPrompt }, + ], + stream: true, + }); + for await (const chunk of stream) { + partial += chunk.choices[0]?.delta?.content || ""; + onToken(partial, false); + } + console.log("Generated:", partial); + onToken(partial, true); +} \ No newline at end of file