working

src/app/page.tsx

@@ -1,21 +1,23 @@
 'use client'
 
 import { LiveImageShape, LiveImageShapeUtil } from '@/components/LiveImageShapeUtil'
+import * as fal from '@fal-ai/serverless-client'
 import { Editor, Tldraw, useEditor } from '@tldraw/tldraw'
 import { useEffect } from 'react'
 import { LiveImageTool, MakeLiveButton } from '../components/LiveImageTool'
 
-// fal.config({
+fal.config({
-// 	requestMiddleware: fal.withProxy({
+	requestMiddleware: fal.withProxy({
-// 		targetUrl: '/api/fal/proxy',
+		targetUrl: '/api/fal/proxy',
-// 	}),
+	}),
-// })
+})
 
 const shapeUtils = [LiveImageShapeUtil]
 const tools = [LiveImageTool]
 
 export default function Home() {
 	const onEditorMount = (editor: Editor) => {
+		// We need the editor to think that the live image shape is a frame
 		// @ts-expect-error: patch
 		editor.isShapeOfType = function (arg, type) {
 			const shape = typeof arg === 'string' ? this.getShape(arg)! : arg
@@ -26,24 +28,18 @@ export default function Home() {
 		}
 
 		// If there isn't a live image shape, create one
-		const liveImage = editor.getCurrentPageShapes().find((shape) => {
+		if (!editor.getCurrentPageShapes().some((shape) => shape.type === 'live-image')) {
-			return shape.type === 'live-image'
+			editor.createShape<LiveImageShape>({
-		})
+				type: 'live-image',
-
+				x: 120,
-		if (liveImage) {
+				y: 180,
-			return
+				props: {
+					w: 512,
+					h: 512,
+					name: 'a city skyline',
+				},
+			})
 		}
-
-		editor.createShape<LiveImageShape>({
-			type: 'live-image',
-			x: 120,
-			y: 180,
-			props: {
-				w: 512,
-				h: 512,
-				name: 'a city skyline',
-			},
-		})
 	}
 
 	return (

src/components/LiveImageShapeUtil.tsx

@@ -19,7 +19,7 @@ import {
 	useIsDarkMode,
 } from '@tldraw/tldraw'
 
-import { useFal } from '@/hooks/useFal'
+import { useLiveImage } from '@/hooks/useLiveImage'
 import { FrameHeading } from './FrameHeading'
 
 // See https://www.fal.ai/models/latent-consistency-sd
@@ -109,9 +109,6 @@ export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {
 		const parent = this.editor.getShape(_shape.parentId)
 		const isInGroup = parent && this.editor.isShapeOfType<TLGroupShape>(parent, 'group')
 
-		// If frame is in a group, keep the shape
-		// moved out in that group
-
 		if (isInGroup) {
 			this.editor.reparentShapes(shapes, parent.id)
 		} else {
@@ -156,7 +153,7 @@ export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {
 	override component(shape: LiveImageShape) {
 		const editor = useEditor()
 
-		useFal(shape.id, {
+		useLiveImage(shape.id, {
 			debounceTime: 0,
 			appId: '110602490-lcm-plexed-sd15-i2i',
 		})

src/hooks/useLiveImage.ts

@@ -8,12 +8,13 @@ import {
 	debounce,
 	getHashForObject,
 	getSvgAsImage,
+	rng,
 	throttle,
 	useEditor,
 } from '@tldraw/tldraw'
 import { useEffect, useRef } from 'react'
 
-export function useFal(
+export function useLiveImage(
 	shapeId: TLShapeId,
 	opts: {
 		debounceTime?: number
@@ -21,7 +22,7 @@ export function useFal(
 		appId: string
 	}
 ) {
-	const { appId, throttleTime = 500, debounceTime = 0 } = opts
+	const { appId, throttleTime = 100, debounceTime = 0 } = opts
 	const editor = useEditor()
 	const startedIteration = useRef<number>(0)
 	const finishedIteration = useRef<number>(0)
@@ -69,13 +70,11 @@ export function useFal(
 		const { send: sendCurrentData, close } = fal.realtime.connect(appId, {
 			connectionKey: 'fal-realtime-example',
 			clientOnly: false,
-			throttleInterval: 1000,
+			throttleInterval: throttleTime,
 			onError: (error) => {
-				console.log(`Received error!`)
 				console.error(error)
 			},
 			onResult: (result) => {
-				console.log(`Received result!`)
 				if (result.images && result.images[0]) {
 					updateImage(result.images[0].url)
 				}
@@ -85,12 +84,13 @@ export function useFal(
 		async function updateDrawing() {
 			const iteration = startedIteration.current++
 
-			const shapes = Array.from(editor.getShapeAndDescendantIds([shapeId])).map((id) =>
+			const shapes = Array.from(editor.getShapeAndDescendantIds([shapeId]))
-				editor.getShape(id)
+				.filter((id) => id !== shapeId)
-			) as TLShape[]
+				.map((id) => editor.getShape(id)) as TLShape[]
 
 			const hash = getHashForObject(shapes)
 			if (hash === prevHash.current) return
+			prevHash.current = hash
 
 			const shape = editor.getShape<LiveImageShape>(shapeId)!
 
@@ -99,29 +99,26 @@ export function useFal(
 				padding: 0,
 				darkMode: editor.user.getIsDarkMode(),
 			})
+			// We might be stale
+			if (iteration <= finishedIteration.current) return
 			if (!svg) {
 				console.error('No SVG')
 				updateImage('')
 				return
 			}
 
-			// We might be stale
-			if (iteration <= finishedIteration.current) return
-
 			const image = await getSvgAsImage(svg, editor.environment.isSafari, {
 				type: 'png',
 				quality: 1,
 				scale: 1,
 			})
+			// We might be stale
+			if (iteration <= finishedIteration.current) return
 			if (!image) {
 				console.error('No image')
 				updateImage('')
 				return
 			}
-
-			// We might be stale
-			if (iteration <= finishedIteration.current) return
-
 			const prompt = shape.props.name
 				? shape.props.name + ' hd award-winning impressive'
 				: 'A random image that is safe for work and not surprising—something boring like a city or shoe watercolor'
@@ -129,18 +126,17 @@ export function useFal(
 			// We might be stale
 			if (iteration <= finishedIteration.current) return
 
+			const random = rng()
+
 			try {
-				console.log('Sending data...')
+				sendCurrentData({
-				sendCurrentData(
+					prompt,
-					JSON.stringify({
+					image_url: imageDataUri,
-						prompt,
+					sync_mode: true,
-						image_url: imageDataUri,
+					strength: 0.7,
-						sync_mode: true,
+					seed: Math.abs(random() * 10000), // TODO make this configurable in the UI
-						strength: 0.7,
+					enable_safety_checks: false,
-						seed: 11252023, // TODO make this configurable in the UI
+				})
-						enable_safety_checks: false,
-					})
-				)
 				finishedIteration.current = iteration
 			} catch (e) {
 				console.error(e)
@@ -156,7 +152,11 @@ export function useFal(
 		editor.on('update-drawings' as any, onDrawingChange)
 
 		return () => {
-			// close()
+			try {
+				close()
+			} catch (e) {
+				// noop
+			}
 			editor.off('update-drawings' as any, onDrawingChange)
 		}
 	}, [editor, shapeId, throttleTime, debounceTime, appId])