diff --git a/src/app/page.tsx b/src/app/page.tsx index d11cb60..a2dfb74 100644 --- a/src/app/page.tsx +++ b/src/app/page.tsx @@ -2,8 +2,9 @@ import { LiveImageShapeUtil } from "@/components/live-image"; import * as fal from "@fal-ai/serverless-client"; -import { Editor, Tldraw } from "@tldraw/tldraw"; +import { Editor, FrameShapeTool, Tldraw, useEditor } from "@tldraw/tldraw"; import { useCallback } from "react"; +import { LiveImageTool, MakeLiveButton } from "../components/LiveImageTool"; fal.config({ requestMiddleware: fal.withProxy({ @@ -12,6 +13,7 @@ fal.config({ }); const shapeUtils = [LiveImageShapeUtil]; +const tools = [LiveImageTool]; export default function Home() { const onEditorMount = (editor: Editor) => { @@ -28,7 +30,11 @@ export default function Home() { type: "live-image", x: 120, y: 180, - isLocked: true, + props: { + w: 512, + h: 512, + name: "a city skyline", + }, }); }; @@ -39,6 +45,8 @@ export default function Home() { persistenceKey="tldraw-fal" onMount={onEditorMount} shapeUtils={shapeUtils} + tools={tools} + shareZone={} /> diff --git a/src/components/LiveImageTool.tsx b/src/components/LiveImageTool.tsx new file mode 100644 index 0000000..6f9147e --- /dev/null +++ b/src/components/LiveImageTool.tsx @@ -0,0 +1,27 @@ +import { FrameShapeTool, useEditor } from "@tldraw/tldraw"; +import { useCallback } from "react"; + +export class LiveImageTool extends FrameShapeTool { + static override id = "live-image"; + static override initial = "idle"; + override shapeType = "live-image"; +} + +export function MakeLiveButton() { + const editor = useEditor(); + const makeLive = useCallback(() => { + editor.setCurrentTool("live-image"); + }, [editor]); + + return ( + + + Make Live + + + ); +} diff --git a/src/components/live-image.tsx b/src/components/live-image.tsx index 5725e52..1a45e5d 100644 --- a/src/components/live-image.tsx +++ b/src/components/live-image.tsx @@ -1,10 +1,11 @@ +/* eslint-disable @next/next/no-img-element */ +/* eslint-disable react-hooks/rules-of-hooks */ import { - Box2d, + FrameShapeUtil, getSvgAsImage, - Rectangle2d, - ShapeUtil, - TLBaseShape, + HTMLContainer, TLEventMapHandler, + TLFrameShape, TLShape, useEditor, } from "@tldraw/tldraw"; @@ -13,7 +14,6 @@ import { blobToDataUri } from "@/utils/blob"; import { debounce } from "@/utils/debounce"; import * as fal from "@fal-ai/serverless-client"; import { useCallback, useEffect, useRef, useState } from "react"; -import { FalLogo } from "./fal-logo"; // See https://www.fal.ai/models/latent-consistency-sd @@ -24,6 +24,10 @@ type Input = { image_url: string; sync_mode: boolean; seed: number; + strength?: number; + guidance_scale?: number; + num_inference_steps?: number; + enable_safety_checks?: boolean; }; type Output = { @@ -36,153 +40,134 @@ type Output = { num_inference_steps: number; }; -// TODO make this an input on the canvas -const PROMPT = "a city skyline"; +export class LiveImageShapeUtil extends FrameShapeUtil { + static override type = "live-image" as any; -export function LiveImage() { - const editor = useEditor(); - const [image, setImage] = useState(null); - - // Used to prevent multiple requests from being sent at once for the same image - // There's probably a better way to do this using TLDraw's state - const imageDigest = useRef(null); - - // eslint-disable-next-line react-hooks/exhaustive-deps - const onDrawingChange = useCallback( - debounce(async () => { - // TODO get actual drawing bounds - // const bounds = new Box2d(120, 180, 512, 512); - - const shapes = editor.getCurrentPageShapes().filter((shape) => { - if (shape.type === "live-image") { - return false; - } - return true; - // const pageBounds = editor.getShapeMaskedPageBounds(shape); - // if (!pageBounds) { - // return false; - // } - // return bounds.includes(pageBounds); - }); - - // Check if should submit request - const shapesDigest = JSON.stringify(shapes); - if (shapesDigest === imageDigest.current) { - return; - } - imageDigest.current = shapesDigest; - - const svg = await editor.getSvg(shapes, { background: true }); - if (!svg) { - return; - } - const image = await getSvgAsImage(svg, editor.environment.isSafari, { - type: "png", - quality: 0.5, - scale: 1, - }); - if (!image) { - return; - } - - const imageDataUri = await blobToDataUri(image); - const result = await fal.run(LatentConsistency, { - input: { - image_url: imageDataUri, - prompt: PROMPT, - sync_mode: true, - seed: 42, // TODO make this configurable in the UI - }, - // Disable auto-upload so we can submit the data uri of the image as is - autoUpload: false, - }); - if (result && result.images.length > 0) { - setImage(result.images[0].url); - } - }, 16), - [] - ); - - useEffect(() => { - const onChange: TLEventMapHandler<"change"> = (event) => { - if (event.source !== "user") { - return; - } - if ( - Object.keys(event.changes.added).length || - Object.keys(event.changes.removed).length || - Object.keys(event.changes.updated).length - ) { - onDrawingChange(); - } - }; - editor.addListener("change", onChange); - return () => { - editor.removeListener("change", onChange); - }; - }, []); - - return ( - - - - /imagine - - - - - {/* eslint-disable-next-line @next/next/no-img-element */} - {image && } - - - - powered by - - - - - - - ); -} - -type LiveImageShape = TLBaseShape<"live-image", { w: number; h: number }>; - -export class LiveImageShapeUtil extends ShapeUtil { - static override type = "live-image" as const; - - override canResize = () => false; - - getDefaultProps(): LiveImageShape["props"] { + override getDefaultProps(): { w: number; h: number; name: string } { return { - w: 1060, - h: 560, + w: 512, + h: 512, + name: "a city skyline", }; } - getGeometry(shape: LiveImageShape) { - return new Rectangle2d({ - width: shape.props.w, - height: shape.props.h, - isFilled: true, - }); - } + override component(shape: TLFrameShape) { + const editor = useEditor(); + const component = super.component(shape); + const [image, setImage] = useState(null); - component(shape: LiveImageShape) { - return ; - } + const imageDigest = useRef(null); + const startedIteration = useRef(0); + const finishedIteration = useRef(0); + + // eslint-disable-next-line react-hooks/exhaustive-deps + const onDrawingChange = useCallback( + debounce(async () => { + // TODO get actual drawing bounds + // const bounds = new Box2d(120, 180, 512, 512); + + const iteration = startedIteration.current++; + + const shapes = Array.from(editor.getShapeAndDescendantIds([shape.id])) + .filter((id) => id !== shape.id) + .map((id) => editor.getShape(id)) as TLShape[]; + + // Check if should submit request + const shapesDigest = JSON.stringify(shapes); + if (shapesDigest === imageDigest.current) { + return; + } + imageDigest.current = shapesDigest; + + const svg = await editor.getSvg(shapes, { background: true }); + if (iteration <= finishedIteration.current) return; + + if (!svg) { + return; + } + const image = await getSvgAsImage(svg, editor.environment.isSafari, { + type: "png", + quality: 1, + scale: 1, + }); + + if (iteration <= finishedIteration.current) return; + + if (!image) { + return; + } + + const prompt = + editor.getShape(shape.id)?.props.name ?? ""; + const imageDataUri = await blobToDataUri(image); + if (iteration <= finishedIteration.current) return; + + const result = await fal.run(LatentConsistency, { + input: { + image_url: imageDataUri, + prompt, + sync_mode: true, + strength: 0.6, + seed: 42, // TODO make this configurable in the UI + enable_safety_checks: false, + }, + // Disable auto-upload so we can submit the data uri of the image as is + autoUpload: true, + }); + if (iteration <= finishedIteration.current) return; + + finishedIteration.current = iteration; + if (result && result.images.length > 0) { + setImage(result.images[0].url); + } + }, 32), + [] + ); + + useEffect(() => { + const onChange: TLEventMapHandler<"change"> = (event) => { + if (event.source !== "user") { + return; + } + if ( + Object.keys(event.changes.added).length || + Object.keys(event.changes.removed).length || + Object.keys(event.changes.updated).length + ) { + onDrawingChange(); + } + }; + editor.addListener("change", onChange); + return () => { + editor.removeListener("change", onChange); + }; + }, [editor, onDrawingChange]); - indicator(shape: LiveImageShape) { return ( - + + + {component} + + {image && ( + + )} + + ); } }