diff --git a/src/app/page.tsx b/src/app/page.tsx index ddff6dd..64a60ed 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -1,21 +1,23 @@ 'use client' import { LiveImageShape, LiveImageShapeUtil } from '@/components/LiveImageShapeUtil' +import * as fal from '@fal-ai/serverless-client' import { Editor, Tldraw, useEditor } from '@tldraw/tldraw' import { useEffect } from 'react' import { LiveImageTool, MakeLiveButton } from '../components/LiveImageTool' -// fal.config({ -// requestMiddleware: fal.withProxy({ -// targetUrl: '/api/fal/proxy', -// }), -// }) +fal.config({ + requestMiddleware: fal.withProxy({ + targetUrl: '/api/fal/proxy', + }), +}) const shapeUtils = [LiveImageShapeUtil] const tools = [LiveImageTool] export default function Home() { const onEditorMount = (editor: Editor) => { + // We need the editor to think that the live image shape is a frame // @ts-expect-error: patch editor.isShapeOfType = function (arg, type) { const shape = typeof arg === 'string' ? this.getShape(arg)! : arg @@ -26,24 +28,18 @@ export default function Home() { } // If there isn't a live image shape, create one - const liveImage = editor.getCurrentPageShapes().find((shape) => { - return shape.type === 'live-image' - }) - - if (liveImage) { - return + if (!editor.getCurrentPageShapes().some((shape) => shape.type === 'live-image')) { + editor.createShape({ + type: 'live-image', + x: 120, + y: 180, + props: { + w: 512, + h: 512, + name: 'a city skyline', + }, + }) } - - editor.createShape({ - type: 'live-image', - x: 120, - y: 180, - props: { - w: 512, - h: 512, - name: 'a city skyline', - }, - }) } return ( diff --git a/src/components/LiveImageShapeUtil.tsx b/src/components/LiveImageShapeUtil.tsx index 76b24d6..28af9a2 100644 --- a/src/components/LiveImageShapeUtil.tsx +++ b/src/components/LiveImageShapeUtil.tsx @@ -19,7 +19,7 @@ import { useIsDarkMode, } from '@tldraw/tldraw' -import { useFal } from '@/hooks/useFal' +import { useLiveImage } from '@/hooks/useLiveImage' import { FrameHeading } from './FrameHeading' // See https://www.fal.ai/models/latent-consistency-sd @@ -109,9 +109,6 @@ export class LiveImageShapeUtil extends ShapeUtil { const parent = this.editor.getShape(_shape.parentId) const isInGroup = parent && this.editor.isShapeOfType(parent, 'group') - // If frame is in a group, keep the shape - // moved out in that group - if (isInGroup) { this.editor.reparentShapes(shapes, parent.id) } else { @@ -156,7 +153,7 @@ export class LiveImageShapeUtil extends ShapeUtil { override component(shape: LiveImageShape) { const editor = useEditor() - useFal(shape.id, { + useLiveImage(shape.id, { debounceTime: 0, appId: '110602490-lcm-plexed-sd15-i2i', }) diff --git a/src/hooks/useFal.ts b/src/hooks/useLiveImage.ts similarity index 85% rename from src/hooks/useFal.ts rename to src/hooks/useLiveImage.ts index ab573a7..68693a5 100644 --- a/src/hooks/useFal.ts +++ b/src/hooks/useLiveImage.ts @@ -8,12 +8,13 @@ import { debounce, getHashForObject, getSvgAsImage, + rng, throttle, useEditor, } from '@tldraw/tldraw' import { useEffect, useRef } from 'react' -export function useFal( +export function useLiveImage( shapeId: TLShapeId, opts: { debounceTime?: number @@ -21,7 +22,7 @@ export function useFal( appId: string } ) { - const { appId, throttleTime = 500, debounceTime = 0 } = opts + const { appId, throttleTime = 100, debounceTime = 0 } = opts const editor = useEditor() const startedIteration = useRef(0) const finishedIteration = useRef(0) @@ -69,13 +70,11 @@ export function useFal( const { send: sendCurrentData, close } = fal.realtime.connect(appId, { connectionKey: 'fal-realtime-example', clientOnly: false, - throttleInterval: 1000, + throttleInterval: throttleTime, onError: (error) => { - console.log(`Received error!`) console.error(error) }, onResult: (result) => { - console.log(`Received result!`) if (result.images && result.images[0]) { updateImage(result.images[0].url) } @@ -85,12 +84,13 @@ export function useFal( async function updateDrawing() { const iteration = startedIteration.current++ - const shapes = Array.from(editor.getShapeAndDescendantIds([shapeId])).map((id) => - editor.getShape(id) - ) as TLShape[] + const shapes = Array.from(editor.getShapeAndDescendantIds([shapeId])) + .filter((id) => id !== shapeId) + .map((id) => editor.getShape(id)) as TLShape[] const hash = getHashForObject(shapes) if (hash === prevHash.current) return + prevHash.current = hash const shape = editor.getShape(shapeId)! @@ -99,29 +99,26 @@ export function useFal( padding: 0, darkMode: editor.user.getIsDarkMode(), }) + // We might be stale + if (iteration <= finishedIteration.current) return if (!svg) { console.error('No SVG') updateImage('') return } - // We might be stale - if (iteration <= finishedIteration.current) return - const image = await getSvgAsImage(svg, editor.environment.isSafari, { type: 'png', quality: 1, scale: 1, }) + // We might be stale + if (iteration <= finishedIteration.current) return if (!image) { console.error('No image') updateImage('') return } - - // We might be stale - if (iteration <= finishedIteration.current) return - const prompt = shape.props.name ? shape.props.name + ' hd award-winning impressive' : 'A random image that is safe for work and not surprising—something boring like a city or shoe watercolor' @@ -129,18 +126,17 @@ export function useFal( // We might be stale if (iteration <= finishedIteration.current) return + const random = rng() + try { - console.log('Sending data...') - sendCurrentData( - JSON.stringify({ - prompt, - image_url: imageDataUri, - sync_mode: true, - strength: 0.7, - seed: 11252023, // TODO make this configurable in the UI - enable_safety_checks: false, - }) - ) + sendCurrentData({ + prompt, + image_url: imageDataUri, + sync_mode: true, + strength: 0.7, + seed: Math.abs(random() * 10000), // TODO make this configurable in the UI + enable_safety_checks: false, + }) finishedIteration.current = iteration } catch (e) { console.error(e) @@ -156,7 +152,11 @@ export function useFal( editor.on('update-drawings' as any, onDrawingChange) return () => { - // close() + try { + close() + } catch (e) { + // noop + } editor.off('update-drawings' as any, onDrawingChange) } }, [editor, shapeId, throttleTime, debounceTime, appId])