feat: tldraw + fal (#1)

* chore: minimal fal + tldraw * feat: tldraw + fal * fix: default prompt
2023-11-19 21:07:39 -08:00 · 2023-11-19 21:07:39 -08:00 · 7b1dd6314b
parent 6f290a0139
commit 7b1dd6314b
13 changed files with 1868 additions and 143 deletions
--- a/next.config.js
+++ b/next.config.js
@ -1,4 +1,4 @@
 /** @type {import('next').NextConfig} */
-const nextConfig = {}
+const nextConfig = {};

-module.exports = nextConfig
+module.exports = nextConfig;
--- a/package-lock.json
+++ b/package-lock.json
--- a/package.json
+++ b/package.json
@ -6,22 +6,27 @@
    "dev": "next dev",
    "build": "next build",
    "start": "next start",
-    "lint": "next lint"
+    "lint": "next lint",
+    "format": "prettier --write ."
  },
  "dependencies": {
+    "@fal-ai/serverless-client": "^0.5.3",
+    "@fal-ai/serverless-proxy": "^0.5.0",
+    "@tldraw/tldraw": "^2.0.0-canary.ba4091c59418",
+    "next": "14.0.3",
    "react": "^18",
-    "react-dom": "^18",
-    "next": "14.0.3"
+    "react-dom": "^18"
  },
  "devDependencies": {
-    "typescript": "^5",
    "@types/node": "^20",
    "@types/react": "^18",
    "@types/react-dom": "^18",
    "autoprefixer": "^10.0.1",
-    "postcss": "^8",
-    "tailwindcss": "^3.3.0",
    "eslint": "^8",
-    "eslint-config-next": "14.0.3"
+    "eslint-config-next": "14.0.3",
+    "postcss": "^8",
+    "prettier": "^3.1.0",
+    "tailwindcss": "^3.3.0",
+    "typescript": "^5"
  }
 }
--- a/postcss.config.js
+++ b/postcss.config.js
@ -3,4 +3,4 @@ module.exports = {
    tailwindcss: {},
    autoprefixer: {},
  },
-}
+};
--- a/src/app/api/fal/proxy/route.ts
+++ b/src/app/api/fal/proxy/route.ts
@ -0,0 +1,3 @@
+import { route } from "@fal-ai/serverless-proxy/nextjs";
+
+export const { GET, POST } = route;
--- a/src/app/globals.css
+++ b/src/app/globals.css
@ -1,3 +1,5 @@
+@import url("@tldraw/tldraw/tldraw.css");
+
@tailwind base;
@tailwind components;
@tailwind utilities;
--- a/src/app/layout.tsx
+++ b/src/app/layout.tsx
@ -1,22 +1,22 @@
-import type { Metadata } from 'next'
-import { Inter } from 'next/font/google'
-import './globals.css'
+import type { Metadata } from "next";
+import { Inter } from "next/font/google";
+import "./globals.css";

-const inter = Inter({ subsets: ['latin'] })
+const inter = Inter({ subsets: ["latin"] });

 export const metadata: Metadata = {
-  title: 'Create Next App',
-  description: 'Generated by create next app',
-}
+  title: "Create Next App",
+  description: "Generated by create next app",
+};

 export default function RootLayout({
  children,
 }: {
-  children: React.ReactNode
+  children: React.ReactNode;
 }) {
  return (
    <html lang="en">
      <body className={inter.className}>{children}</body>
    </html>
-  )
+  );
 }
--- a/src/app/page.tsx
+++ b/src/app/page.tsx
@ -1,113 +1,29 @@
-import Image from 'next/image'
+"use client";
+
+import { LiveImageShapeUtil } from "@/components/live-image";
+import * as fal from "@fal-ai/serverless-client";
+import { Editor, Tldraw } from "@tldraw/tldraw";
+
+fal.config({
+  requestMiddleware: fal.withProxy({
+    targetUrl: "/api/fal/proxy",
+  }),
+});

 export default function Home() {
+  const onEditorMount = (editor: Editor) => {
+    editor.createShape({
+      type: "live-image",
+      x: 120,
+      y: 180,
+    });
+  };
+
  return (
-    <main className="flex min-h-screen flex-col items-center justify-between p-24">
-      <div className="z-10 max-w-5xl w-full items-center justify-between font-mono text-sm lg:flex">
-        <p className="fixed left-0 top-0 flex w-full justify-center border-b border-gray-300 bg-gradient-to-b from-zinc-200 pb-6 pt-8 backdrop-blur-2xl dark:border-neutral-800 dark:bg-zinc-800/30 dark:from-inherit lg:static lg:w-auto  lg:rounded-xl lg:border lg:bg-gray-200 lg:p-4 lg:dark:bg-zinc-800/30">
-          Get started by editing&nbsp;
-          <code className="font-mono font-bold">src/app/page.tsx</code>
-        </p>
-        <div className="fixed bottom-0 left-0 flex h-48 w-full items-end justify-center bg-gradient-to-t from-white via-white dark:from-black dark:via-black lg:static lg:h-auto lg:w-auto lg:bg-none">
-          <a
-            className="pointer-events-none flex place-items-center gap-2 p-8 lg:pointer-events-auto lg:p-0"
-            href="https://vercel.com?utm_source=create-next-app&utm_medium=appdir-template&utm_campaign=create-next-app"
-            target="_blank"
-            rel="noopener noreferrer"
-          >
-            By{' '}
-            <Image
-              src="/vercel.svg"
-              alt="Vercel Logo"
-              className="dark:invert"
-              width={100}
-              height={24}
-              priority
-            />
-          </a>
-        </div>
-      </div>
-
-      <div className="relative flex place-items-center before:absolute before:h-[300px] before:w-[480px] before:-translate-x-1/2 before:rounded-full before:bg-gradient-radial before:from-white before:to-transparent before:blur-2xl before:content-[''] after:absolute after:-z-20 after:h-[180px] after:w-[240px] after:translate-x-1/3 after:bg-gradient-conic after:from-sky-200 after:via-blue-200 after:blur-2xl after:content-[''] before:dark:bg-gradient-to-br before:dark:from-transparent before:dark:to-blue-700 before:dark:opacity-10 after:dark:from-sky-900 after:dark:via-[#0141ff] after:dark:opacity-40 before:lg:h-[360px] z-[-1]">
-        <Image
-          className="relative dark:drop-shadow-[0_0_0.3rem_#ffffff70] dark:invert"
-          src="/next.svg"
-          alt="Next.js Logo"
-          width={180}
-          height={37}
-          priority
-        />
-      </div>
-
-      <div className="mb-32 grid text-center lg:max-w-5xl lg:w-full lg:mb-0 lg:grid-cols-4 lg:text-left">
-        <a
-          href="https://nextjs.org/docs?utm_source=create-next-app&utm_medium=appdir-template&utm_campaign=create-next-app"
-          className="group rounded-lg border border-transparent px-5 py-4 transition-colors hover:border-gray-300 hover:bg-gray-100 hover:dark:border-neutral-700 hover:dark:bg-neutral-800/30"
-          target="_blank"
-          rel="noopener noreferrer"
-        >
-          <h2 className={`mb-3 text-2xl font-semibold`}>
-            Docs{' '}
-            <span className="inline-block transition-transform group-hover:translate-x-1 motion-reduce:transform-none">
-              -&gt;
-            </span>
-          </h2>
-          <p className={`m-0 max-w-[30ch] text-sm opacity-50`}>
-            Find in-depth information about Next.js features and API.
-          </p>
-        </a>
-
-        <a
-          href="https://nextjs.org/learn?utm_source=create-next-app&utm_medium=appdir-template-tw&utm_campaign=create-next-app"
-          className="group rounded-lg border border-transparent px-5 py-4 transition-colors hover:border-gray-300 hover:bg-gray-100 hover:dark:border-neutral-700 hover:dark:bg-neutral-800/30"
-          target="_blank"
-          rel="noopener noreferrer"
-        >
-          <h2 className={`mb-3 text-2xl font-semibold`}>
-            Learn{' '}
-            <span className="inline-block transition-transform group-hover:translate-x-1 motion-reduce:transform-none">
-              -&gt;
-            </span>
-          </h2>
-          <p className={`m-0 max-w-[30ch] text-sm opacity-50`}>
-            Learn about Next.js in an interactive course with&nbsp;quizzes!
-          </p>
-        </a>
-
-        <a
-          href="https://vercel.com/templates?framework=next.js&utm_source=create-next-app&utm_medium=appdir-template&utm_campaign=create-next-app"
-          className="group rounded-lg border border-transparent px-5 py-4 transition-colors hover:border-gray-300 hover:bg-gray-100 hover:dark:border-neutral-700 hover:dark:bg-neutral-800/30"
-          target="_blank"
-          rel="noopener noreferrer"
-        >
-          <h2 className={`mb-3 text-2xl font-semibold`}>
-            Templates{' '}
-            <span className="inline-block transition-transform group-hover:translate-x-1 motion-reduce:transform-none">
-              -&gt;
-            </span>
-          </h2>
-          <p className={`m-0 max-w-[30ch] text-sm opacity-50`}>
-            Explore starter templates for Next.js.
-          </p>
-        </a>
-
-        <a
-          href="https://vercel.com/new?utm_source=create-next-app&utm_medium=appdir-template&utm_campaign=create-next-app"
-          className="group rounded-lg border border-transparent px-5 py-4 transition-colors hover:border-gray-300 hover:bg-gray-100 hover:dark:border-neutral-700 hover:dark:bg-neutral-800/30"
-          target="_blank"
-          rel="noopener noreferrer"
-        >
-          <h2 className={`mb-3 text-2xl font-semibold`}>
-            Deploy{' '}
-            <span className="inline-block transition-transform group-hover:translate-x-1 motion-reduce:transform-none">
-              -&gt;
-            </span>
-          </h2>
-          <p className={`m-0 max-w-[30ch] text-sm opacity-50`}>
-            Instantly deploy your Next.js site to a shareable URL with Vercel.
-          </p>
-        </a>
+    <main className="flex min-h-screen flex-col items-center justify-between">
+      <div className="fixed inset-0">
+        <Tldraw onMount={onEditorMount} shapeUtils={[LiveImageShapeUtil]} />
      </div>
    </main>
-  )
+  );
 }
--- a/src/components/fal-logo.tsx
+++ b/src/components/fal-logo.tsx
@ -0,0 +1,202 @@
+export function FalLogo() {
+  return (
+    <svg
+      width="100%"
+      height="100%"
+      viewBox="0 0 89 32"
+      fill="none"
+      xmlns="http://www.w3.org/2000/svg"
+    >
+      <path
+        d="M52.308 3.07812H57.8465V4.92428H56.0003V6.77043H54.1541V10.4627H57.8465V12.3089H54.1541V25.232H52.308V27.0781H46.7695V25.232H48.6157V12.3089H46.7695V10.4627H48.6157V6.77043H50.4618V4.92428H52.308V3.07812Z"
+        fill="currentColor"
+      ></path>
+      <path
+        d="M79.3849 23.3858H81.2311V25.232H83.0772V27.0781H88.6157V25.232H86.7695V23.3858H84.9234V4.92428H79.3849V23.3858Z"
+        fill="currentColor"
+      ></path>
+      <path
+        d="M57.8465 14.155H59.6926V12.3089H61.5388V10.4627H70.7695V12.3089H74.4618V23.3858H76.308V25.232H78.1541V27.0781H72.6157V25.232H70.7695V23.3858H68.9234V14.155H67.0772V12.3089H65.2311V14.155H63.3849V23.3858H65.2311V25.232H67.0772V27.0781H61.5388V25.232H59.6926V23.3858H57.8465V14.155Z"
+        fill="currentColor"
+      ></path>
+      <path
+        d="M67.0772 25.232V23.3858H68.9234V25.232H67.0772Z"
+        fill="currentColor"
+      ></path>
+      <rect
+        opacity="0.22"
+        x="7.38477"
+        y="29.5391"
+        width="2.46154"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.85"
+        x="2.46094"
+        y="19.6914"
+        width="12.3077"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        x="4.92383"
+        y="17.2305"
+        width="9.84615"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.4"
+        x="7.38477"
+        y="27.0781"
+        width="4.92308"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.7"
+        y="22.1562"
+        width="14.7692"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.5"
+        x="7.38477"
+        y="24.6133"
+        width="7.38462"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.22"
+        x="7.38477"
+        y="12.3086"
+        width="2.46154"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.85"
+        x="2.46094"
+        y="2.46094"
+        width="12.3077"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect x="4.92383" width="9.84615" height="2.46154" fill="#5F4CD9"></rect>
+      <rect
+        opacity="0.4"
+        x="7.38477"
+        y="9.84375"
+        width="4.92308"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.7"
+        y="4.92188"
+        width="14.7692"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.5"
+        x="7.38477"
+        y="7.38281"
+        width="7.38462"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.22"
+        x="24.6152"
+        y="29.5391"
+        width="2.46154"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.85"
+        x="19.6914"
+        y="19.6914"
+        width="12.3077"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        x="22.1543"
+        y="17.2305"
+        width="9.84615"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.4"
+        x="24.6152"
+        y="27.0781"
+        width="4.92308"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.7"
+        x="17.2305"
+        y="22.1562"
+        width="14.7692"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.5"
+        x="24.6152"
+        y="24.6133"
+        width="7.38462"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.22"
+        x="24.6152"
+        y="12.3086"
+        width="2.46154"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.85"
+        x="19.6914"
+        y="2.46094"
+        width="12.3077"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect x="22.1543" width="9.84615" height="2.46154" fill="#5F4CD9"></rect>
+      <rect
+        opacity="0.4"
+        x="24.6152"
+        y="9.84375"
+        width="4.92308"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.7"
+        x="17.2305"
+        y="4.92188"
+        width="14.7692"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+      <rect
+        opacity="0.5"
+        x="24.6152"
+        y="7.38281"
+        width="7.38462"
+        height="2.46154"
+        fill="#5F4CD9"
+      ></rect>
+    </svg>
+  );
+}
--- a/src/components/live-image.tsx
+++ b/src/components/live-image.tsx
@ -0,0 +1,182 @@
+import {
+  Box2d,
+  getSvgAsImage,
+  Rectangle2d,
+  ShapeUtil,
+  TLBaseShape,
+  TLEventMapHandler,
+  useEditor,
+} from "@tldraw/tldraw";
+
+import { blobToDataUri } from "@/utils/blob";
+import { debounce } from "@/utils/debounce";
+import * as fal from "@fal-ai/serverless-client";
+import { useCallback, useEffect, useRef, useState } from "react";
+import { FalLogo } from "./fal-logo";
+
+// See https://www.fal.ai/models/latent-consistency-sd
+
+const LatentConsistency = "110602490-lcm-sd15-i2i";
+
+type Input = {
+  prompt: string;
+  image_url: string;
+  sync_mode: boolean;
+  seed: number;
+};
+
+type Output = {
+  images: Array<{
+    url: string;
+    width: number;
+    height: number;
+  }>;
+  seed: number;
+  num_inference_steps: number;
+};
+
+// TODO make this an input on the canvas
+const PROMPT = "a sunset at a tropical beach with palm trees";
+
+export function LiveImage() {
+  const editor = useEditor();
+  const [image, setImage] = useState<string | null>(null);
+
+  // Used to prevent multiple requests from being sent at once for the same image
+  // There's probably a better way to do this using TLDraw's state
+  const imageDigest = useRef<string | null>(null);
+
+  // eslint-disable-next-line react-hooks/exhaustive-deps
+  const onDrawingChange = useCallback(
+    debounce(async () => {
+      // TODO get actual drawing bounds
+      const bounds = new Box2d(120, 180, 512, 512);
+
+      const shapes = editor.getCurrentPageShapes().filter((shape) => {
+        if (shape.type === "live-image") {
+          return false;
+        }
+        const pageBounds = editor.getShapeMaskedPageBounds(shape);
+        if (!pageBounds) {
+          return false;
+        }
+        return bounds.includes(pageBounds);
+      });
+
+      // Check if should submit request
+      const shapesDigest = JSON.stringify(shapes);
+      if (shapesDigest === imageDigest.current) {
+        return;
+      }
+      imageDigest.current = shapesDigest;
+
+      const svg = await editor.getSvg(shapes, { bounds, background: true });
+      if (!svg) {
+        return;
+      }
+      const image = await getSvgAsImage(svg, editor.environment.isSafari, {
+        type: "png",
+        quality: 0.5,
+        scale: 1,
+      });
+      if (!image) {
+        return;
+      }
+
+      const imageDataUri = await blobToDataUri(image);
+      const result = await fal.run<Input, Output>(LatentConsistency, {
+        input: {
+          image_url: imageDataUri,
+          prompt: PROMPT,
+          sync_mode: true,
+          seed: 42, // TODO make this configurable in the UI
+        },
+      });
+      if (result && result.images.length > 0) {
+        setImage(result.images[0].url);
+      }
+    }, 16),
+    [],
+  );
+
+  useEffect(() => {
+    const onChange: TLEventMapHandler<"change"> = (event) => {
+      if (event.source !== "user") {
+        return;
+      }
+      if (
+        Object.keys(event.changes.added).length ||
+        Object.keys(event.changes.removed).length ||
+        Object.keys(event.changes.updated).length
+      ) {
+        onDrawingChange();
+      }
+    };
+    editor.addListener("change", onChange);
+    return () => {
+      editor.removeListener("change", onChange);
+    };
+  }, []);
+
+  return (
+    <div className="flex flex-row w-[1060px] h-[560px] absolute bg-indigo-200 border border-indigo-500 rounded space-x-4 p-4 pb-8">
+      <div className="flex-1 h-[512px] bg-white border border-indigo-500">
+        <div className="flex flex-row items-center px-4 py-2 space-x-2">
+          <span className="font-mono text-indigo-900/50">/imagine</span>
+          <input
+            className="border-0 bg-transparent flex-1 text-base text-indigo-900"
+            placeholder="something cool..."
+            value={PROMPT}
+          />
+        </div>
+      </div>
+      <div className="flex-1 h-[512px] bg-white border border-indigo-500">
+        {/* eslint-disable-next-line @next/next/no-img-element */}
+        {image && <img src={image} alt="" width={512} height={512} />}
+      </div>
+      <span className="absolute bottom-1.5 right-4">
+        <a
+          href="https://fal.ai/models/latent-consistency"
+          target="_blank"
+          className="flex flex-row space-x-1"
+        >
+          <span className="text-xs text-indigo-900/50">powered by</span>
+          <span className="w-[36px] opacity-50">
+            <FalLogo />
+          </span>
+        </a>
+      </span>
+    </div>
+  );
+}
+
+type LiveImageShape = TLBaseShape<"live-image", { w: number; h: number }>;
+
+export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {
+  static override type = "live-image" as const;
+
+  getDefaultProps(): LiveImageShape["props"] {
+    return {
+      w: 1060,
+      h: 560,
+    };
+  }
+
+  getGeometry(shape: LiveImageShape) {
+    return new Rectangle2d({
+      width: shape.props.w,
+      height: shape.props.h,
+      isFilled: true,
+    });
+  }
+
+  component(shape: LiveImageShape) {
+    return <LiveImage />;
+  }
+
+  indicator(shape: LiveImageShape) {
+    return (
+      <rect width={shape.props.w} height={shape.props.h} radius={4}></rect>
+    );
+  }
+}
--- a/src/utils/blob.ts
+++ b/src/utils/blob.ts
@ -0,0 +1,14 @@
+export async function blobToDataUri(blob: Blob): Promise<string> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onloadend = () => {
+      if (typeof reader.result === "string") {
+        resolve(reader.result);
+      } else {
+        reject(new Error("Failed to convert Blob to base64 data URI"));
+      }
+    };
+    reader.onerror = reject;
+    reader.readAsDataURL(blob);
+  });
+}
--- a/src/utils/debounce.ts
+++ b/src/utils/debounce.ts
@ -0,0 +1,19 @@
+export function debounce<T extends (...args: any[]) => any>(
+  func: T,
+  wait: number,
+): (...funcArgs: Parameters<T>) => void {
+  let timeout: NodeJS.Timeout | null;
+
+  return (...args: Parameters<T>): void => {
+    const later = () => {
+      timeout = null;
+      func(...args);
+    };
+
+    if (timeout) {
+      clearTimeout(timeout);
+    }
+
+    timeout = setTimeout(later, wait);
+  };
+}
--- a/tailwind.config.ts
+++ b/tailwind.config.ts
@ -1,20 +1,20 @@
-import type { Config } from 'tailwindcss'
+import type { Config } from "tailwindcss";

 const config: Config = {
  content: [
-    './src/pages/**/*.{js,ts,jsx,tsx,mdx}',
-    './src/components/**/*.{js,ts,jsx,tsx,mdx}',
-    './src/app/**/*.{js,ts,jsx,tsx,mdx}',
+    "./src/pages/**/*.{js,ts,jsx,tsx,mdx}",
+    "./src/components/**/*.{js,ts,jsx,tsx,mdx}",
+    "./src/app/**/*.{js,ts,jsx,tsx,mdx}",
  ],
  theme: {
    extend: {
      backgroundImage: {
-        'gradient-radial': 'radial-gradient(var(--tw-gradient-stops))',
-        'gradient-conic':
-          'conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))',
+        "gradient-radial": "radial-gradient(var(--tw-gradient-stops))",
+        "gradient-conic":
+          "conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))",
      },
    },
  },
  plugins: [],
-}
-export default config
+};
+export default config;