Merge pull request #8 from tldraw/realtime-2

Realtime 2
2023-11-25 21:55:23 +00:00 · 2023-11-25 21:55:23 +00:00 · 4668ba8bf7
parent 63ccc75ac6 1545f6c09e
commit 4668ba8bf7
7 changed files with 126 additions and 175 deletions
--- a/.prettierrc
+++ b/.prettierrc
@ -0,0 +1,11 @@
+{
+  "trailingComma": "es5",
+  "singleQuote": true,
+  "semi": false,
+  "printWidth": 100,
+  "tabWidth": 2,
+  "useTabs": true,
+  "plugins": [
+    "prettier-plugin-organize-imports"
+  ]
+}
--- a/package-lock.json
+++ b/package-lock.json
@ -7,8 +7,9 @@
    "": {
      "name": "tldraw-fal",
      "version": "0.1.0",
+      "license": "MIT",
      "dependencies": {
-        "@fal-ai/serverless-client": "^0.5.4",
+        "@fal-ai/serverless-client": "^0.6.0-alpha.4",
        "@fal-ai/serverless-proxy": "^0.5.0",
        "@tldraw/tldraw": "^2.0.0-canary.ba4091c59418",
        "next": "14.0.3",
@ -24,6 +25,7 @@
        "eslint-config-next": "14.0.3",
        "postcss": "^8",
        "prettier": "^3.1.0",
+        "prettier-plugin-organize-imports": "^3.2.4",
        "tailwindcss": "^3.3.0",
        "typescript": "^5"
      }
@ -117,9 +119,9 @@
      }
    },
    "node_modules/@fal-ai/serverless-client": {
-      "version": "0.5.4",
-      "resolved": "https://registry.npmjs.org/@fal-ai/serverless-client/-/serverless-client-0.5.4.tgz",
-      "integrity": "sha512-8eTA+lBXtGzYqDTjIHm7vnViu5AlPZYR1D9BQbsDxro/K53W9sMhZtc7QSboE1MMcbs4hW2B+f+Jkkx39hJCPw=="
+      "version": "0.6.0-alpha.4",
+      "resolved": "https://registry.npmjs.org/@fal-ai/serverless-client/-/serverless-client-0.6.0-alpha.4.tgz",
+      "integrity": "sha512-T9fqiMU1LohzzqsNZjY4zq8ZMxJcZo8eYvSXaz2i6nmb9sQxuQ4hCB/YTeZJ2Ong4P+ZOMkHPMD2kW4pNgf5gw=="
    },
    "node_modules/@fal-ai/serverless-proxy": {
      "version": "0.5.0",
@ -4642,6 +4644,26 @@
        "url": "https://github.com/prettier/prettier?sponsor=1"
      }
    },
+    "node_modules/prettier-plugin-organize-imports": {
+      "version": "3.2.4",
+      "resolved": "https://registry.npmjs.org/prettier-plugin-organize-imports/-/prettier-plugin-organize-imports-3.2.4.tgz",
+      "integrity": "sha512-6m8WBhIp0dfwu0SkgfOxJqh+HpdyfqSSLfKKRZSFbDuEQXDDndb8fTpRWkUrX/uBenkex3MgnVk0J3b3Y5byog==",
+      "dev": true,
+      "peerDependencies": {
+        "@volar/vue-language-plugin-pug": "^1.0.4",
+        "@volar/vue-typescript": "^1.0.4",
+        "prettier": ">=2.0",
+        "typescript": ">=2.9"
+      },
+      "peerDependenciesMeta": {
+        "@volar/vue-language-plugin-pug": {
+          "optional": true
+        },
+        "@volar/vue-typescript": {
+          "optional": true
+        }
+      }
+    },
    "node_modules/prop-types": {
      "version": "15.8.1",
      "resolved": "https://registry.npmjs.org/prop-types/-/prop-types-15.8.1.tgz",
--- a/package.json
+++ b/package.json
@ -2,6 +2,7 @@
  "name": "tldraw-fal",
  "version": "0.1.0",
  "private": true,
+  "license": "MIT",
  "scripts": {
    "dev": "next dev",
    "build": "next build",
@ -10,7 +11,7 @@
    "format": "prettier --write ."
  },
  "dependencies": {
-    "@fal-ai/serverless-client": "^0.5.4",
+    "@fal-ai/serverless-client": "^0.6.0-alpha.4",
    "@fal-ai/serverless-proxy": "^0.5.0",
    "@tldraw/tldraw": "^2.0.0-canary.ba4091c59418",
    "next": "14.0.3",
@ -26,6 +27,7 @@
    "eslint-config-next": "14.0.3",
    "postcss": "^8",
    "prettier": "^3.1.0",
+    "prettier-plugin-organize-imports": "^3.2.4",
    "tailwindcss": "^3.3.0",
    "typescript": "^5"
  }
--- a/src/app/page.tsx
+++ b/src/app/page.tsx
@ -1,18 +1,9 @@
 'use client'

-import {
-	LiveImageShape,
-	LiveImageShapeUtil,
-} from '@/components/LiveImageShapeUtil'
+import { LiveImageShape, LiveImageShapeUtil } from '@/components/LiveImageShapeUtil'
 import * as fal from '@fal-ai/serverless-client'
-import {
-	AssetRecordType,
-	Editor,
-	FrameShapeTool,
-	Tldraw,
-	useEditor,
-} from '@tldraw/tldraw'
-import { useCallback, useEffect } from 'react'
+import { Editor, Tldraw, useEditor } from '@tldraw/tldraw'
+import { useEffect } from 'react'
 import { LiveImageTool, MakeLiveButton } from '../components/LiveImageTool'

 fal.config({
@ -26,6 +17,7 @@ const tools = [LiveImageTool]

 export default function Home() {
 	const onEditorMount = (editor: Editor) => {
+		// We need the editor to think that the live image shape is a frame
 		// @ts-expect-error: patch
 		editor.isShapeOfType = function (arg, type) {
 			const shape = typeof arg === 'string' ? this.getShape(arg)! : arg
@ -36,24 +28,18 @@ export default function Home() {
 		}

 		// If there isn't a live image shape, create one
-		const liveImage = editor.getCurrentPageShapes().find((shape) => {
-			return shape.type === 'live-image'
-		})
-
-		if (liveImage) {
-			return
+		if (!editor.getCurrentPageShapes().some((shape) => shape.type === 'live-image')) {
+			editor.createShape<LiveImageShape>({
+				type: 'live-image',
+				x: 120,
+				y: 180,
+				props: {
+					w: 512,
+					h: 512,
+					name: 'a city skyline',
+				},
+			})
 		}
-
-		editor.createShape<LiveImageShape>({
-			type: 'live-image',
-			x: 120,
-			y: 180,
-			props: {
-				w: 512,
-				h: 512,
-				name: 'a city skyline',
-			},
-		})
 	}

 	return (
--- a/src/components/FrameHeading.tsx
+++ b/src/components/FrameHeading.tsx
@ -8,9 +8,9 @@ import {
 	useIsEditing,
 	useValue,
 } from '@tldraw/editor'
+import { preventDefault, stopEventPropagation } from '@tldraw/tldraw'
 import { useCallback, useEffect, useRef } from 'react'
 import { FrameLabelInput } from './FrameLabelInput'
-import { preventDefault, stopEventPropagation } from '@tldraw/tldraw'

 export function FrameHeading({
 	id,
@ -41,8 +41,6 @@ export function FrameHeading({

 			const event = getPointerInfo(e)

-			console.log('hello')
-
 			// If we're editing the frame label, we shouldn't hijack the pointer event
 			if (editor.getEditingShapeId() === id) return

@ -77,9 +75,9 @@ export function FrameHeading({
 	// rotate right 45 deg
 	const offsetRotation = pageRotation + Math.PI / 4
 	const scaledRotation = (offsetRotation * (2 / Math.PI) + 4) % 4
-	const labelSide: SelectionEdge = (
-		['top', 'left', 'bottom', 'right'] as const
-	)[Math.floor(scaledRotation)]
+	const labelSide: SelectionEdge = (['top', 'left', 'bottom', 'right'] as const)[
+		Math.floor(scaledRotation)
+	]

 	let labelTranslate: string
 	switch (labelSide) {
@ -87,9 +85,7 @@ export function FrameHeading({
 			labelTranslate = ``
 			break
 		case 'right':
-			labelTranslate = `translate(${toDomPrecision(
-				width
-			)}px, 0px) rotate(90deg)`
+			labelTranslate = `translate(${toDomPrecision(width)}px, 0px) rotate(90deg)`
 			break
 		case 'bottom':
 			labelTranslate = `translate(${toDomPrecision(width)}px, ${toDomPrecision(
@ -97,9 +93,7 @@ export function FrameHeading({
 			)}px) rotate(180deg)`
 			break
 		case 'left':
-			labelTranslate = `translate(0px, ${toDomPrecision(
-				height
-			)}px) rotate(270deg)`
+			labelTranslate = `translate(0px, ${toDomPrecision(height)}px) rotate(270deg)`
 			break
 	}

@ -109,9 +103,7 @@ export function FrameHeading({
 			style={{
 				overflow: isEditing ? 'visible' : 'hidden',
 				maxWidth: `calc(var(--tl-zoom) * ${
-					labelSide === 'top' || labelSide === 'bottom'
-						? Math.ceil(width)
-						: Math.ceil(height)
+					labelSide === 'top' || labelSide === 'bottom' ? Math.ceil(width) : Math.ceil(height)
 				}px + var(--space-5))`,
 				bottom: '100%',
 				transform: `${labelTranslate} scale(var(--tl-scale)) translateX(calc(-1 * var(--space-3))`,
@ -119,12 +111,7 @@ export function FrameHeading({
 			onPointerDown={handlePointerDown}
 		>
 			<div className="tl-frame-heading-hit-area">
-				<FrameLabelInput
-					ref={rInput}
-					id={id}
-					name={name}
-					isEditing={isEditing}
-				/>
+				<FrameLabelInput ref={rInput} id={id} name={name} isEditing={isEditing} />
 			</div>
 		</div>
 	)
--- a/src/components/LiveImageShapeUtil.tsx
+++ b/src/components/LiveImageShapeUtil.tsx
@ -2,20 +2,12 @@
 /* eslint-disable react-hooks/rules-of-hooks */
 import {
 	AssetRecordType,
-	canonicalizeRotation,
-	FrameShapeUtil,
 	Geometry2d,
 	getDefaultColorTheme,
-	getHashForObject,
-	getSvgAsImage,
-	HTMLContainer,
-	IdOf,
 	Rectangle2d,
 	resizeBox,
-	SelectionEdge,
 	ShapeUtil,
 	SVGContainer,
-	TLAsset,
 	TLBaseShape,
 	TLGroupShape,
 	TLOnResizeEndHandler,
@ -27,15 +19,8 @@ import {
 	useIsDarkMode,
 } from '@tldraw/tldraw'

-import { blobToDataUri } from '@/utils/blob'
-import { debounce } from '@/utils/debounce'
-import * as fal from '@fal-ai/serverless-client'
-import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
-import result from 'postcss/lib/result'
+import { useLiveImage } from '@/hooks/useLiveImage'
 import { FrameHeading } from './FrameHeading'
-import image from 'next/image'
-import { connect } from 'http2'
-import { useFal } from '@/hooks/useFal'

 // See https://www.fal.ai/models/latent-consistency-sd

@ -94,10 +79,7 @@ export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {

 	canUnmount = () => false

-	override canReceiveNewChildrenOfType = (
-		shape: TLShape,
-		_type: TLShape['type']
-	) => {
+	override canReceiveNewChildrenOfType = (shape: TLShape, _type: TLShape['type']) => {
 		return !shape.isLocked
 	}

@ -105,10 +87,7 @@ export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {
 		return true
 	}

-	override canDropShapes = (
-		shape: LiveImageShape,
-		_shapes: TLShape[]
-	): boolean => {
+	override canDropShapes = (shape: LiveImageShape, _shapes: TLShape[]): boolean => {
 		return !shape.isLocked
 	}

@ -126,16 +105,9 @@ export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {
 		return { shouldHint: false }
 	}

-	override onDragShapesOut = (
-		_shape: LiveImageShape,
-		shapes: TLShape[]
-	): void => {
+	override onDragShapesOut = (_shape: LiveImageShape, shapes: TLShape[]): void => {
 		const parent = this.editor.getShape(_shape.parentId)
-		const isInGroup =
-			parent && this.editor.isShapeOfType<TLGroupShape>(parent, 'group')
-
-		// If frame is in a group, keep the shape
-		// moved out in that group
+		const isInGroup = parent && this.editor.isShapeOfType<TLGroupShape>(parent, 'group')

 		if (isInGroup) {
 			this.editor.reparentShapes(shapes, parent.id)
@ -158,10 +130,7 @@ export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {
 		}

 		if (shapesToReparent.length > 0) {
-			this.editor.reparentShapes(
-				shapesToReparent,
-				this.editor.getCurrentPageId()
-			)
+			this.editor.reparentShapes(shapesToReparent, this.editor.getCurrentPageId())
 		}
 	}

@ -184,9 +153,9 @@ export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {
 	override component(shape: LiveImageShape) {
 		const editor = useEditor()

-		useFal(shape.id, {
+		useLiveImage(shape.id, {
 			debounceTime: 0,
-			url: 'wss://110602490-lcm-sd15-i2i.gateway.alpha.fal.ai/ws',
+			appId: '110602490-lcm-plexed-sd15-i2i',
 		})

 		const bounds = this.editor.getShapeGeometry(shape).bounds
--- a/src/hooks/useLiveImage.ts
+++ b/src/hooks/useLiveImage.ts
@ -1,5 +1,6 @@
 import { LiveImageShape } from '@/components/LiveImageShapeUtil'
 import { blobToDataUri } from '@/utils/blob'
+import * as fal from '@fal-ai/serverless-client'
 import {
 	AssetRecordType,
 	TLShape,
@ -7,31 +8,29 @@ import {
 	debounce,
 	getHashForObject,
 	getSvgAsImage,
+	rng,
 	throttle,
 	useEditor,
 } from '@tldraw/tldraw'
-import { useRef, useEffect } from 'react'
+import { useEffect, useRef } from 'react'

-export function useFal(
+export function useLiveImage(
 	shapeId: TLShapeId,
 	opts: {
 		debounceTime?: number
 		throttleTime?: number
-		url: string
+		appId: string
 	}
 ) {
-	const { url, throttleTime = 500, debounceTime = 0 } = opts
+	const { appId, throttleTime = 100, debounceTime = 0 } = opts
 	const editor = useEditor()
 	const startedIteration = useRef<number>(0)
 	const finishedIteration = useRef<number>(0)

 	const prevHash = useRef<string | null>(null)
+	const prevPrompt = useRef<string>('')

 	useEffect(() => {
-		let socket: WebSocket | null = null
-
-		let isReconnecting = false
-
 		function updateImage(url: string | null) {
 			const shape = editor.getShape<LiveImageShape>(shapeId)!
 			const id = AssetRecordType.createId(shape.id.split(':')[1])
@ -69,110 +68,80 @@ export function useFal(
 			}
 		}

-		async function connect() {
-			{
-				socket = new WebSocket(url)
-				socket.onopen = () => {
-					// console.log("WebSocket Open");
+		const { send: sendCurrentData, close } = fal.realtime.connect(appId, {
+			connectionKey: 'fal-realtime-example',
+			clientOnly: false,
+			throttleInterval: throttleTime,
+			onError: (error) => {
+				console.error(error)
+			},
+			onResult: (result) => {
+				if (result.images && result.images[0]) {
+					updateImage(result.images[0].url)
 				}
-
-				socket.onclose = () => {
-					// console.log("WebSocket Close");
-				}
-
-				socket.onerror = (error) => {
-					// console.error("WebSocket Error:", error);
-				}
-
-				socket.onmessage = (message) => {
-					try {
-						const data = JSON.parse(message.data)
-						// console.log("WebSocket Message:", data);
-						if (data.images && data.images.length > 0) {
-							updateImage(data.images[0].url ?? '')
-						}
-					} catch (e) {
-						console.error('Error parsing the WebSocket response:', e)
-					}
-				}
-			}
-		}
-
-		async function sendCurrentData(message: string) {
-			if (!isReconnecting && socket?.readyState !== WebSocket.OPEN) {
-				isReconnecting = true
-				connect()
-			}
-
-			if (isReconnecting && socket?.readyState !== WebSocket.OPEN) {
-				await new Promise<void>((resolve) => {
-					const checkConnection = setInterval(() => {
-						if (socket?.readyState === WebSocket.OPEN) {
-							clearInterval(checkConnection)
-							resolve()
-						}
-					}, 100)
-				})
-				isReconnecting = false
-			}
-			socket?.send(message)
-		}
+			},
+		})

 		async function updateDrawing() {
 			const iteration = startedIteration.current++

-			const shapes = Array.from(editor.getShapeAndDescendantIds([shapeId])).map(
-				(id) => editor.getShape(id)
-			) as TLShape[]
-
-			const hash = getHashForObject(shapes)
-			if (hash === prevHash.current) return
+			const shapes = Array.from(editor.getShapeAndDescendantIds([shapeId]))
+				.filter((id) => id !== shapeId)
+				.map((id) => editor.getShape(id)) as TLShape[]

 			const shape = editor.getShape<LiveImageShape>(shapeId)!
+			const hash = getHashForObject(shapes)
+			if (hash === prevHash.current && shape.props.name === prevPrompt.current) return
+			prevHash.current = hash
+			prevPrompt.current = shape.props.name

 			const svg = await editor.getSvg([shape], {
 				background: true,
 				padding: 0,
 				darkMode: editor.user.getIsDarkMode(),
 			})
+			// We might be stale
+			if (iteration <= finishedIteration.current) return
 			if (!svg) {
+				console.error('No SVG')
 				updateImage('')
 				return
 			}

-			// We might be stale
-			if (iteration <= finishedIteration.current) return
-
 			const image = await getSvgAsImage(svg, editor.environment.isSafari, {
 				type: 'png',
 				quality: 1,
 				scale: 1,
 			})
+			// We might be stale
+			if (iteration <= finishedIteration.current) return
 			if (!image) {
+				console.error('No image')
 				updateImage('')
 				return
 			}
-
-			// We might be stale
-			if (iteration <= finishedIteration.current) return
-
 			const prompt = shape.props.name
 				? shape.props.name + ' hd award-winning impressive'
 				: 'A random image that is safe for work and not surprising—something boring like a city or shoe watercolor'
 			const imageDataUri = await blobToDataUri(image)
-			const request = {
-				image_url: imageDataUri,
-				prompt,
-				sync_mode: true,
-				strength: 0.7,
-				seed: 42, // TODO make this configurable in the UI
-				enable_safety_checks: false,
-			}
-
 			// We might be stale
 			if (iteration <= finishedIteration.current) return
-			sendCurrentData(JSON.stringify(request))
-			finishedIteration.current = iteration
+
+			const random = rng()
+
+			try {
+				sendCurrentData({
+					prompt,
+					image_url: imageDataUri,
+					sync_mode: true,
+					strength: 0.7,
+					seed: Math.abs(random() * 10000), // TODO make this configurable in the UI
+					enable_safety_checks: false,
+				})
+				finishedIteration.current = iteration
+			} catch (e) {
+				console.error(e)
+			}
 		}

 		const onDrawingChange = debounceTime
@ -184,7 +153,12 @@ export function useFal(
 		editor.on('update-drawings' as any, onDrawingChange)

 		return () => {
+			try {
+				close()
+			} catch (e) {
+				// noop
+			}
 			editor.off('update-drawings' as any, onDrawingChange)
 		}
-	}, [editor, shapeId, throttleTime, debounceTime, url])
+	}, [editor, shapeId, throttleTime, debounceTime, appId])
 }