more tests

src/app/page.tsx

@@ -2,8 +2,9 @@
 
 import { LiveImageShapeUtil } from "@/components/live-image";
 import * as fal from "@fal-ai/serverless-client";
-import { Editor, Tldraw } from "@tldraw/tldraw";
+import { Editor, FrameShapeTool, Tldraw, useEditor } from "@tldraw/tldraw";
 import { useCallback } from "react";
+import { LiveImageTool, MakeLiveButton } from "../components/LiveImageTool";
 
 fal.config({
   requestMiddleware: fal.withProxy({
@@ -12,6 +13,7 @@ fal.config({
 });
 
 const shapeUtils = [LiveImageShapeUtil];
+const tools = [LiveImageTool];
 
 export default function Home() {
   const onEditorMount = (editor: Editor) => {
@@ -28,7 +30,11 @@ export default function Home() {
       type: "live-image",
       x: 120,
       y: 180,
-      isLocked: true,
+      props: {
+        w: 512,
+        h: 512,
+        name: "a city skyline",
+      },
     });
   };
 
@@ -39,6 +45,8 @@ export default function Home() {
           persistenceKey="tldraw-fal"
           onMount={onEditorMount}
           shapeUtils={shapeUtils}
+          tools={tools}
+          shareZone={<MakeLiveButton />}
         />
       </div>
     </main>

src/components/LiveImageTool.tsx

@@ -0,0 +1,27 @@
+import { FrameShapeTool, useEditor } from "@tldraw/tldraw";
+import { useCallback } from "react";
+
+export class LiveImageTool extends FrameShapeTool {
+  static override id = "live-image";
+  static override initial = "idle";
+  override shapeType = "live-image";
+}
+
+export function MakeLiveButton() {
+  const editor = useEditor();
+  const makeLive = useCallback(() => {
+    editor.setCurrentTool("live-image");
+  }, [editor]);
+
+  return (
+    <button
+      onClick={makeLive}
+      className="p-2"
+      style={{ cursor: "pointer", zIndex: 100000, pointerEvents: "all" }}
+    >
+      <div className="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">
+        Make Live
+      </div>
+    </button>
+  );
+}

src/components/live-image.tsx

@@ -1,10 +1,11 @@
+/* eslint-disable @next/next/no-img-element */
+/* eslint-disable react-hooks/rules-of-hooks */
 import {
-  Box2d,
+  FrameShapeUtil,
   getSvgAsImage,
-  Rectangle2d,
+  HTMLContainer,
-  ShapeUtil,
-  TLBaseShape,
   TLEventMapHandler,
+  TLFrameShape,
   TLShape,
   useEditor,
 } from "@tldraw/tldraw";
@@ -13,7 +14,6 @@ import { blobToDataUri } from "@/utils/blob";
 import { debounce } from "@/utils/debounce";
 import * as fal from "@fal-ai/serverless-client";
 import { useCallback, useEffect, useRef, useState } from "react";
-import { FalLogo } from "./fal-logo";
 
 // See https://www.fal.ai/models/latent-consistency-sd
 
@@ -24,6 +24,10 @@ type Input = {
   image_url: string;
   sync_mode: boolean;
   seed: number;
+  strength?: number;
+  guidance_scale?: number;
+  num_inference_steps?: number;
+  enable_safety_checks?: boolean;
 };
 
 type Output = {
@@ -36,153 +40,134 @@ type Output = {
   num_inference_steps: number;
 };
 
-// TODO make this an input on the canvas
+export class LiveImageShapeUtil extends FrameShapeUtil {
-const PROMPT = "a city skyline";
+  static override type = "live-image" as any;
 
-export function LiveImage() {
+  override getDefaultProps(): { w: number; h: number; name: string } {
-  const editor = useEditor();
-  const [image, setImage] = useState<string | null>(null);
-
-  // Used to prevent multiple requests from being sent at once for the same image
-  // There's probably a better way to do this using TLDraw's state
-  const imageDigest = useRef<string | null>(null);
-
-  // eslint-disable-next-line react-hooks/exhaustive-deps
-  const onDrawingChange = useCallback(
-    debounce(async () => {
-      // TODO get actual drawing bounds
-      // const bounds = new Box2d(120, 180, 512, 512);
-
-      const shapes = editor.getCurrentPageShapes().filter((shape) => {
-        if (shape.type === "live-image") {
-          return false;
-        }
-        return true;
-        // const pageBounds = editor.getShapeMaskedPageBounds(shape);
-        // if (!pageBounds) {
-        //   return false;
-        // }
-        // return bounds.includes(pageBounds);
-      });
-
-      // Check if should submit request
-      const shapesDigest = JSON.stringify(shapes);
-      if (shapesDigest === imageDigest.current) {
-        return;
-      }
-      imageDigest.current = shapesDigest;
-
-      const svg = await editor.getSvg(shapes, { background: true });
-      if (!svg) {
-        return;
-      }
-      const image = await getSvgAsImage(svg, editor.environment.isSafari, {
-        type: "png",
-        quality: 0.5,
-        scale: 1,
-      });
-      if (!image) {
-        return;
-      }
-
-      const imageDataUri = await blobToDataUri(image);
-      const result = await fal.run<Input, Output>(LatentConsistency, {
-        input: {
-          image_url: imageDataUri,
-          prompt: PROMPT,
-          sync_mode: true,
-          seed: 42, // TODO make this configurable in the UI
-        },
-        // Disable auto-upload so we can submit the data uri of the image as is
-        autoUpload: false,
-      });
-      if (result && result.images.length > 0) {
-        setImage(result.images[0].url);
-      }
-    }, 16),
-    []
-  );
-
-  useEffect(() => {
-    const onChange: TLEventMapHandler<"change"> = (event) => {
-      if (event.source !== "user") {
-        return;
-      }
-      if (
-        Object.keys(event.changes.added).length ||
-        Object.keys(event.changes.removed).length ||
-        Object.keys(event.changes.updated).length
-      ) {
-        onDrawingChange();
-      }
-    };
-    editor.addListener("change", onChange);
-    return () => {
-      editor.removeListener("change", onChange);
-    };
-  }, []);
-
-  return (
-    <div className="flex flex-row w-[1060px] h-[560px] absolute bg-indigo-200 border border-indigo-500 rounded space-x-4 p-4 pb-8">
-      <div className="flex-1 h-[512px] bg-white border border-indigo-500">
-        <div className="flex flex-row items-center px-4 py-2 space-x-2">
-          <span className="font-mono text-indigo-900/50">/imagine</span>
-          <input
-            className="border-0 bg-transparent flex-1 text-base text-indigo-900"
-            placeholder="something cool..."
-            value={PROMPT}
-          />
-        </div>
-      </div>
-      <div className="flex-1 h-[512px] bg-white border border-indigo-500">
-        {/* eslint-disable-next-line @next/next/no-img-element */}
-        {image && <img src={image} alt="" width={512} height={512} />}
-      </div>
-      <span className="absolute bottom-1.5 right-4">
-        <a
-          href="https://fal.ai/models/latent-consistency"
-          target="_blank"
-          className="flex flex-row space-x-1"
-        >
-          <span className="text-xs text-indigo-900/50">powered by</span>
-          <span className="w-[36px] opacity-50">
-            <FalLogo />
-          </span>
-        </a>
-      </span>
-    </div>
-  );
-}
-
-type LiveImageShape = TLBaseShape<"live-image", { w: number; h: number }>;
-
-export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {
-  static override type = "live-image" as const;
-
-  override canResize = () => false;
-
-  getDefaultProps(): LiveImageShape["props"] {
     return {
-      w: 1060,
+      w: 512,
-      h: 560,
+      h: 512,
+      name: "a city skyline",
     };
   }
 
-  getGeometry(shape: LiveImageShape) {
+  override component(shape: TLFrameShape) {
-    return new Rectangle2d({
+    const editor = useEditor();
-      width: shape.props.w,
+    const component = super.component(shape);
-      height: shape.props.h,
+    const [image, setImage] = useState<string | null>(null);
-      isFilled: true,
-    });
-  }
 
-  component(shape: LiveImageShape) {
+    const imageDigest = useRef<string | null>(null);
-    return <LiveImage />;
+    const startedIteration = useRef<number>(0);
-  }
+    const finishedIteration = useRef<number>(0);
+
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+    const onDrawingChange = useCallback(
+      debounce(async () => {
+        // TODO get actual drawing bounds
+        // const bounds = new Box2d(120, 180, 512, 512);
+
+        const iteration = startedIteration.current++;
+
+        const shapes = Array.from(editor.getShapeAndDescendantIds([shape.id]))
+          .filter((id) => id !== shape.id)
+          .map((id) => editor.getShape(id)) as TLShape[];
+
+        // Check if should submit request
+        const shapesDigest = JSON.stringify(shapes);
+        if (shapesDigest === imageDigest.current) {
+          return;
+        }
+        imageDigest.current = shapesDigest;
+
+        const svg = await editor.getSvg(shapes, { background: true });
+        if (iteration <= finishedIteration.current) return;
+
+        if (!svg) {
+          return;
+        }
+        const image = await getSvgAsImage(svg, editor.environment.isSafari, {
+          type: "png",
+          quality: 1,
+          scale: 1,
+        });
+
+        if (iteration <= finishedIteration.current) return;
+
+        if (!image) {
+          return;
+        }
+
+        const prompt =
+          editor.getShape<TLFrameShape>(shape.id)?.props.name ?? "";
+        const imageDataUri = await blobToDataUri(image);
+        if (iteration <= finishedIteration.current) return;
+
+        const result = await fal.run<Input, Output>(LatentConsistency, {
+          input: {
+            image_url: imageDataUri,
+            prompt,
+            sync_mode: true,
+            strength: 0.6,
+            seed: 42, // TODO make this configurable in the UI
+            enable_safety_checks: false,
+          },
+          // Disable auto-upload so we can submit the data uri of the image as is
+          autoUpload: true,
+        });
+        if (iteration <= finishedIteration.current) return;
+
+        finishedIteration.current = iteration;
+        if (result && result.images.length > 0) {
+          setImage(result.images[0].url);
+        }
+      }, 32),
+      []
+    );
+
+    useEffect(() => {
+      const onChange: TLEventMapHandler<"change"> = (event) => {
+        if (event.source !== "user") {
+          return;
+        }
+        if (
+          Object.keys(event.changes.added).length ||
+          Object.keys(event.changes.removed).length ||
+          Object.keys(event.changes.updated).length
+        ) {
+          onDrawingChange();
+        }
+      };
+      editor.addListener("change", onChange);
+      return () => {
+        editor.removeListener("change", onChange);
+      };
+    }, [editor, onDrawingChange]);
 
-  indicator(shape: LiveImageShape) {
     return (
-      <rect width={shape.props.w} height={shape.props.h} radius={4}></rect>
+      <HTMLContainer>
+        <div
+          style={{
+            display: "flex",
+          }}
+        >
+          {component}
+
+          {image && (
+            <img
+              src={image}
+              alt=""
+              width={shape.props.w}
+              height={shape.props.h}
+              style={{
+                position: "relative",
+                left: shape.props.w,
+                width: shape.props.w,
+                height: shape.props.h,
+              }}
+            />
+          )}
+        </div>
+      </HTMLContainer>
     );
   }
 }