From c5aa92af3bdc3f76b041706c7ed41234262eb3b1 Mon Sep 17 00:00:00 2001 From: alex Date: Tue, 28 Nov 2023 18:11:58 +0000 Subject: [PATCH] main: fix issue with multiple shapes --- package-lock.json | 22 +- package.json | 4 +- src/app/page.tsx | 38 ++-- src/components/LiveImageShapeUtil.tsx | 7 +- src/components/LockupLink.tsx | 26 +-- src/hooks/useLiveImage.ts | 188 ----------------- src/hooks/useLiveImage.tsx | 287 ++++++++++++++++++++++++++ 7 files changed, 347 insertions(+), 225 deletions(-) delete mode 100644 src/hooks/useLiveImage.ts create mode 100644 src/hooks/useLiveImage.tsx diff --git a/package-lock.json b/package-lock.json index cbc920c..76257cc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,12 +14,14 @@ "@tldraw/tldraw": "^2.0.0-canary.ba4091c59418", "next": "14.0.3", "react": "^18", - "react-dom": "^18" + "react-dom": "^18", + "uuid": "^9.0.1" }, "devDependencies": { "@types/node": "^20", "@types/react": "^18", "@types/react-dom": "^18", + "@types/uuid": "^9.0.7", "autoprefixer": "^10.0.1", "eslint": "^8", "eslint-config-next": "14.0.3", @@ -1582,6 +1584,12 @@ "integrity": "sha512-Vlktnchmkylvc9SnwwwozTv04L/e1NykF5vgoQ0XTmI8DD+wxfjQuHuvHS3p0r2jz2x2ghPs2h1FVeDirIteWA==", "devOptional": true }, + "node_modules/@types/uuid": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-9.0.7.tgz", + "integrity": "sha512-WUtIVRUZ9i5dYXefDEAI7sh9/O7jGvHg7Df/5O/gtH3Yabe5odI3UWopVR1qbPXQtvOxWu3mM4XxlYeZtMWF4g==", + "dev": true + }, "node_modules/@typescript-eslint/parser": { "version": "6.11.0", "resolved": "https://registry.npmjs.org/@typescript-eslint/parser/-/parser-6.11.0.tgz", @@ -5605,6 +5613,18 @@ "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", "dev": true }, + "node_modules/uuid": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.1.tgz", + "integrity": "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA==", + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/watchpack": { "version": "2.4.0", "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz", diff --git a/package.json b/package.json index 1c989e1..60f3831 100644 --- a/package.json +++ b/package.json @@ -16,12 +16,14 @@ "@tldraw/tldraw": "^2.0.0-canary.ba4091c59418", "next": "14.0.3", "react": "^18", - "react-dom": "^18" + "react-dom": "^18", + "uuid": "^9.0.1" }, "devDependencies": { "@types/node": "^20", "@types/react": "^18", "@types/react-dom": "^18", + "@types/uuid": "^9.0.7", "autoprefixer": "^10.0.1", "eslint": "^8", "eslint-config-next": "14.0.3", diff --git a/src/app/page.tsx b/src/app/page.tsx index 8318ac3..007448e 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -3,6 +3,7 @@ import { LiveImageShape, LiveImageShapeUtil } from '@/components/LiveImageShapeUtil' import { LockupLink } from '@/components/LockupLink' +import { LiveImageProvider } from '@/hooks/useLiveImage' import * as fal from '@fal-ai/serverless-client' import { AssetRecordType, @@ -85,22 +86,24 @@ export default function Home() { } return ( -
-
- } - overrides={overrides} - > - - - - -
-
+ +
+
+ } + overrides={overrides} + > + + + + +
+
+ ) } @@ -146,7 +149,8 @@ const LiveImageAsset = track(function LiveImageAsset({ shape }: { shape: LiveIma const assetId = AssetRecordType.createId(shape.id.split(':')[1]) const asset = editor.getAsset(assetId) return ( - asset && ( + asset && + asset.props.src( {shape.props.name} { override component(shape: LiveImageShape) { const editor = useEditor() - useLiveImage(shape.id, { - debounceTime: 0, - appId: '110602490-lcm-sd15-i2i', - }) + useLiveImage(shape.id) const bounds = this.editor.getShapeGeometry(shape).bounds const assetId = AssetRecordType.createId(shape.id.split(':')[1]) @@ -181,7 +178,7 @@ export class LiveImageShapeUtil extends ShapeUtil { width={bounds.width} height={bounds.height} /> - {!shape.props.overlayResult && asset && ( + {!shape.props.overlayResult && asset && asset.props.src && ( {shape.props.name} ) diff --git a/src/hooks/useLiveImage.ts b/src/hooks/useLiveImage.ts deleted file mode 100644 index d9a5ea6..0000000 --- a/src/hooks/useLiveImage.ts +++ /dev/null @@ -1,188 +0,0 @@ -import { LiveImageShape } from '@/components/LiveImageShapeUtil' -import { blobToDataUri } from '@/utils/blob' -import * as fal from '@fal-ai/serverless-client' -import { - AssetRecordType, - Editor, - TLShape, - TLShapeId, - debounce, - getHashForObject, - getSvgAsImage, - rng, - throttle, - useEditor, -} from '@tldraw/tldraw' -import { useEffect, useRef } from 'react' - -export function useLiveImage( - shapeId: TLShapeId, - opts: { - debounceTime?: number - throttleTime?: number - appId: string - } -) { - const { appId, throttleTime = 100, debounceTime = 0 } = opts - const editor = useEditor() - const startedIteration = useRef(0) - const finishedIteration = useRef(0) - - const prevHash = useRef(null) - const prevPrompt = useRef('') - - useEffect(() => { - function updateImage(url: string | null) { - const shape = editor.getShape(shapeId)! - const id = AssetRecordType.createId(shape.id.split(':')[1]) - - const asset = editor.getAsset(id) - - if (!asset) { - editor.createAssets([ - AssetRecordType.create({ - id, - type: 'image', - props: { - name: shape.props.name, - w: shape.props.w, - h: shape.props.h, - src: url, - isAnimated: false, - mimeType: 'image/jpeg', - }, - }), - ]) - } else { - editor.updateAssets([ - { - ...asset, - type: 'image', - props: { - ...asset.props, - w: shape.props.w, - h: shape.props.h, - src: url, - }, - }, - ]) - } - } - - const { send: sendCurrentData, close } = fal.realtime.connect(appId, { - connectionKey: 'fal-realtime-example', - clientOnly: false, - throttleInterval: throttleTime, - onError: (error) => { - console.error(error) - }, - onResult: (result) => { - if (result.images && result.images[0]) { - updateImage(result.images[0].url) - } - }, - }) - - async function updateDrawing() { - const iteration = startedIteration.current++ - - const shapes = getShapesTouching(shapeId, editor) - - const shape = editor.getShape(shapeId)! - const hash = getHashForObject([...shapes]) - if (hash === prevHash.current && shape.props.name === prevPrompt.current) return - prevHash.current = hash - prevPrompt.current = shape.props.name - - const svg = await editor.getSvg([...shapes], { - background: true, - padding: 0, - darkMode: editor.user.getIsDarkMode(), - bounds: editor.getShapePageBounds(shapeId)!, - }) - // We might be stale - if (iteration <= finishedIteration.current) return - if (!svg) { - console.error('No SVG') - updateImage('') - return - } - - const image = await getSvgAsImage(svg, editor.environment.isSafari, { - type: 'png', - quality: 1, - scale: 512 / shape.props.w, - }) - - // We might be stale - if (iteration <= finishedIteration.current) return - if (!image) { - console.error('No image') - updateImage('') - return - } - const prompt = shape.props.name - ? shape.props.name + ' hd award-winning impressive' - : 'A random image that is safe for work and not surprising—something boring like a city or shoe watercolor' - const imageDataUri = await blobToDataUri(image) - // downloadDataURLAsFile(imageDataUri, 'test.png') - // We might be stale - if (iteration <= finishedIteration.current) return - - const random = rng(shapeId) - - try { - sendCurrentData({ - prompt, - image_url: imageDataUri, - sync_mode: true, - strength: 0.65, - seed: Math.abs(random() * 10000), // TODO make this configurable in the UI - enable_safety_checks: false, - }) - finishedIteration.current = iteration - } catch (e) { - console.error(e) - } - } - - const onDrawingChange = debounceTime - ? debounce(updateDrawing, debounceTime) - : throttleTime - ? throttle(updateDrawing, throttleTime) - : debounce(updateDrawing, 16) - - editor.on('update-drawings' as any, onDrawingChange) - - return () => { - try { - close() - } catch (e) { - // noop - } - editor.off('update-drawings' as any, onDrawingChange) - } - }, [editor, shapeId, throttleTime, debounceTime, appId]) -} - -function getShapesTouching(shapeId: TLShapeId, editor: Editor) { - const shapeIdsOnPage = editor.getCurrentPageShapeIds() - const shapesTouching: TLShape[] = [] - const targetBounds = editor.getShapePageBounds(shapeId) - if (!targetBounds) return shapesTouching - for (const id of [...shapeIdsOnPage]) { - if (id === shapeId) continue - const bounds = editor.getShapePageBounds(id)! - if (bounds.collides(targetBounds)) { - shapesTouching.push(editor.getShape(id)!) - } - } - return shapesTouching -} - -function downloadDataURLAsFile(dataUrl: string, filename: string) { - const link = document.createElement('a') - link.href = dataUrl - link.download = filename - link.click() -} diff --git a/src/hooks/useLiveImage.tsx b/src/hooks/useLiveImage.tsx new file mode 100644 index 0000000..6d73045 --- /dev/null +++ b/src/hooks/useLiveImage.tsx @@ -0,0 +1,287 @@ +import { LiveImageShape } from '@/components/LiveImageShapeUtil' +import { Queue } from '@/utils/Queue' +import { blobToDataUri } from '@/utils/blob' +import * as fal from '@fal-ai/serverless-client' +import { + AssetRecordType, + Editor, + TLShape, + TLShapeId, + getHashForObject, + getSvgAsImage, + rng, + useEditor, +} from '@tldraw/tldraw' +import { createContext, useContext, useEffect, useState } from 'react' +import { v4 as uuid } from 'uuid' + +type LiveImageResult = { url: string } +type LiveImageRequest = { + prompt: string + image_url: string + sync_mode: boolean + strength: number + seed: number + enable_safety_checks: boolean +} +type LiveImageContextType = null | ((req: LiveImageRequest) => Promise) +const LiveImageContext = createContext(null) + +const svgQueue = new Queue() + +export function LiveImageProvider({ + children, + appId, + throttleTime = 0, + timeoutTime = 5000, +}: { + children: React.ReactNode + appId: string + throttleTime?: number + timeoutTime?: number +}) { + const [count, setCount] = useState(0) + const [fetchImage, setFetchImage] = useState<{ current: LiveImageContextType }>({ current: null }) + + useEffect(() => { + const requestsById = new Map< + string, + { + resolve: (result: LiveImageResult) => void + reject: (err: unknown) => void + timer: ReturnType + } + >() + + const { send, close } = fal.realtime.connect(appId, { + connectionKey: 'fal-realtime-example', + clientOnly: false, + throttleInterval: throttleTime, + onError: (error) => { + console.error(error) + // force re-connect + setCount((count) => count + 1) + }, + onResult: (result) => { + console.log(result) + if (result.images && result.images[0]) { + const id = result.request_id + const request = requestsById.get(id) + if (request) { + request.resolve(result.images[0]) + } + } + }, + }) + + setFetchImage({ + current: (req) => { + return new Promise((resolve, reject) => { + const id = uuid() + const timer = setTimeout(() => { + requestsById.delete(id) + reject(new Error('Timeout')) + }, timeoutTime) + requestsById.set(id, { + resolve: (res) => { + resolve(res) + clearTimeout(timer) + }, + reject: (err) => { + reject(err) + clearTimeout(timer) + }, + timer, + }) + console.log('send', id, req) + send({ ...req, request_id: id }) + }) + }, + }) + + return () => { + for (const request of requestsById.values()) { + request.reject(new Error('Connection closed')) + } + try { + close() + } catch (e) { + // noop + } + } + }, [appId, count, throttleTime, timeoutTime]) + + return ( + {children} + ) +} + +export function useLiveImage(shapeId: TLShapeId) { + const editor = useEditor() + const fetchImage = useContext(LiveImageContext) + if (!fetchImage) throw new Error('Missing LiveImageProvider') + + useEffect(() => { + console.log('do effect') + let prevHash = '' + let prevPrompt = '' + let prevSvg = '' + let state: 'idle' | 'requested-latest' | 'requested-stale' = 'idle' + + async function updateDrawing() { + if (state === 'requested-stale') return + if (state === 'requested-latest') { + state = 'requested-stale' + return + } + + const shapes = getShapesTouching(shapeId, editor) + const frame = editor.getShape(shapeId)! + + const hash = getHashForObject([...shapes]) + if (hash === prevHash && frame.props.name === prevPrompt) return + + if (state !== 'idle') throw new Error('State should be idle') + state = 'requested-latest' + + prevHash = hash + prevPrompt = frame.props.name + + try { + const svg = await editor.getSvg([...shapes], { + background: true, + padding: 0, + darkMode: editor.user.getIsDarkMode(), + bounds: editor.getShapePageBounds(shapeId)!, + }) + + if (!svg) { + console.error('No SVG') + updateImage(editor, frame.id, '') + return + } + + const image = await getSvgAsImage(svg, editor.environment.isSafari, { + type: 'png', + quality: 1, + scale: 512 / frame.props.w, + }) + + if (!image) { + console.error('No image') + updateImage(editor, frame.id, '') + return + } + + const prompt = frame.props.name + ? frame.props.name + ' hd award-winning impressive' + : 'A random image that is safe for work and not surprising—something boring like a city or shoe watercolor' + const imageDataUri = await blobToDataUri(image) + + if (imageDataUri === prevSvg) { + console.log('Same image') + return + } else { + console.log({ imageDataUri, prevSvg }) + } + prevSvg = imageDataUri + + const random = rng(shapeId) + + const result = await fetchImage!({ + prompt, + image_url: imageDataUri, + sync_mode: true, + strength: 0.65, + seed: Math.abs(random() * 10000), // TODO make this configurable in the UI + enable_safety_checks: false, + }) + updateImage(editor, frame.id, result.url) + } catch (e) { + console.error(e) + } finally { + if (state === 'requested-latest') { + state = 'idle' + return + } + if (state === 'requested-stale') { + state = 'idle' + updateDrawing() + } + } + } + + let frame: number | null = null + function requestUpdate() { + if (frame) return + frame = requestAnimationFrame(() => { + frame = null + updateDrawing() + }) + } + + editor.on('update-drawings' as any, requestUpdate) + return () => { + editor.off('update-drawings' as any, requestUpdate) + } + }, [editor, fetchImage, shapeId]) +} + +function updateImage(editor: Editor, shapeId: TLShapeId, url: string | null) { + const shape = editor.getShape(shapeId)! + const id = AssetRecordType.createId(shape.id.split(':')[1]) + + const asset = editor.getAsset(id) + + if (!asset) { + editor.createAssets([ + AssetRecordType.create({ + id, + type: 'image', + props: { + name: shape.props.name, + w: shape.props.w, + h: shape.props.h, + src: url, + isAnimated: false, + mimeType: 'image/jpeg', + }, + }), + ]) + } else { + editor.updateAssets([ + { + ...asset, + type: 'image', + props: { + ...asset.props, + w: shape.props.w, + h: shape.props.h, + src: url, + }, + }, + ]) + } +} + +function getShapesTouching(shapeId: TLShapeId, editor: Editor) { + const shapeIdsOnPage = editor.getCurrentPageShapeIds() + const shapesTouching: TLShape[] = [] + const targetBounds = editor.getShapePageBounds(shapeId) + if (!targetBounds) return shapesTouching + for (const id of [...shapeIdsOnPage]) { + if (id === shapeId) continue + const bounds = editor.getShapePageBounds(id)! + if (bounds.collides(targetBounds)) { + shapesTouching.push(editor.getShape(id)!) + } + } + return shapesTouching +} + +function downloadDataURLAsFile(dataUrl: string, filename: string) { + const link = document.createElement('a') + link.href = dataUrl + link.download = filename + link.click() +}