alex/overlay-mode: overlay mode

src/app/page.tsx

@@ -1,17 +1,21 @@
+/* eslint-disable @next/next/no-img-element */
 'use client'
 
 import { LiveImageShape, LiveImageShapeUtil } from '@/components/LiveImageShapeUtil'
 import { LockupLink } from '@/components/LockupLink'
 import * as fal from '@fal-ai/serverless-client'
 import {
+	AssetRecordType,
 	DefaultSizeStyle,
 	Editor,
 	TLUiOverrides,
 	Tldraw,
 	toolbarItem,
+	track,
 	useEditor,
 } from '@tldraw/tldraw'
-import { useEffect } from 'react'
+import { useEffect, useMemo, useState } from 'react'
+import { createPortal } from 'react-dom'
 import { LiveImageTool, MakeLiveButton } from '../components/LiveImageTool'
 
 fal.config({
@@ -93,6 +97,7 @@ export default function Home() {
 				>
 					<SneakySideEffects />
 					<LockupLink />
+					<LiveImageAssets />
 				</Tldraw>
 			</div>
 		</main>
@@ -106,7 +111,71 @@ function SneakySideEffects() {
 		editor.sideEffects.registerAfterChangeHandler('shape', () => {
 			editor.emit('update-drawings' as any)
 		})
+		editor.sideEffects.registerAfterCreateHandler('shape', () => {
+			editor.emit('update-drawings' as any)
+		})
+		editor.sideEffects.registerAfterDeleteHandler('shape', () => {
+			editor.emit('update-drawings' as any)
+		})
 	}, [editor])
 
 	return null
 }
+
+const LiveImageAssets = track(function LiveImageAssets() {
+	const editor = useEditor()
+
+	return (
+		<Inject selector=".tl-overlays .tl-html-layer">
+			{editor
+				.getCurrentPageShapes()
+				.filter((shape): shape is LiveImageShape => shape.type === 'live-image')
+				.map((shape) => (
+					<LiveImageAsset key={shape.id} shape={shape} />
+				))}
+		</Inject>
+	)
+})
+
+const LiveImageAsset = track(function LiveImageAsset({ shape }: { shape: LiveImageShape }) {
+	const editor = useEditor()
+
+	if (!shape.props.overlayResult) return null
+
+	const transform = editor.getShapePageTransform(shape).toCssString()
+	const assetId = AssetRecordType.createId(shape.id.split(':')[1])
+	const asset = editor.getAsset(assetId)
+	return (
+		asset && (
+			<img
+				src={asset.props.src!}
+				alt={shape.props.name}
+				width={shape.props.w}
+				height={shape.props.h}
+				style={{
+					position: 'absolute',
+					top: 0,
+					left: 0,
+					width: shape.props.w,
+					height: shape.props.h,
+					maxWidth: 'none',
+					transform,
+					transformOrigin: 'top left',
+					opacity: shape.opacity,
+				}}
+			/>
+		)
+	)
+})
+
+function Inject({ children, selector }: { children: React.ReactNode; selector: string }) {
+	const [parent, setParent] = useState<Element | null>(null)
+	const target = useMemo(() => parent?.querySelector(selector) ?? null, [parent, selector])
+
+	return (
+		<>
+			<div ref={(el) => setParent(el?.parentElement ?? null)} style={{ display: 'none' }} />
+			{target && createPortal(children, target)}
+		</>
+	)
+}

src/components/LiveImageShapeUtil.tsx

@@ -2,6 +2,7 @@
 /* eslint-disable react-hooks/rules-of-hooks */
 import {
 	AssetRecordType,
+	Button,
 	Geometry2d,
 	getDefaultColorTheme,
 	Rectangle2d,
@@ -51,13 +52,14 @@ export type LiveImageShape = TLBaseShape<
 		w: number
 		h: number
 		name: string
+		overlayResult?: boolean
 	}
 >
 
 export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {
 	static type = 'live-image' as any
 
-	override canBind = () => true
+	override canBind = () => false
 	override canUnmount = () => false
 	override canEdit = () => true
 	override isAspectRatioLocked = () => true
@@ -107,7 +109,6 @@ export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {
 	override onDragShapesOut = (_shape: LiveImageShape, shapes: TLShape[]): void => {
 		const parent = this.editor.getShape(_shape.parentId)
 		const isInGroup = parent && this.editor.isShapeOfType<TLGroupShape>(parent, 'group')
-
 		if (isInGroup) {
 			this.editor.reparentShapes(shapes, parent.id)
 		} else {
@@ -161,7 +162,6 @@ export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {
 		const assetId = AssetRecordType.createId(shape.id.split(':')[1])
 		const asset = editor.getAsset(assetId)
 
-		// eslint-disable-next-line react-hooks/rules-of-hooks
 		const theme = getDefaultColorTheme({ isDarkMode: useIsDarkMode() })
 
 		return (
@@ -181,7 +181,7 @@ export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {
 					width={bounds.width}
 					height={bounds.height}
 				/>
-				{asset && (
+				{!shape.props.overlayResult && asset && (
 					<img
 						src={asset.props.src!}
 						alt={shape.props.name}
@@ -195,6 +195,30 @@ export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {
 						}}
 					/>
 				)}
+				<Button
+					type="icon"
+					icon={shape.props.overlayResult ? 'chevron-right' : 'chevron-left'}
+					style={{
+						position: 'absolute',
+						top: -4,
+						left: shape.props.overlayResult ? shape.props.w : shape.props.w * 2,
+						pointerEvents: 'auto',
+						transform: 'scale(var(--tl-scale))',
+						transformOrigin: '0 4px',
+					}}
+					onPointerDown={(e) => {
+						e.stopPropagation()
+						console.log('pointerdowm', e)
+					}}
+					onClick={(e) => {
+						console.log('click', e)
+						editor.updateShape<LiveImageShape>({
+							id: shape.id,
+							type: 'live-image',
+							props: { overlayResult: !shape.props.overlayResult },
+						})
+					}}
+				/>
 			</>
 		)
 	}

src/hooks/useLiveImage.ts

@@ -3,6 +3,7 @@ import { blobToDataUri } from '@/utils/blob'
 import * as fal from '@fal-ai/serverless-client'
 import {
 	AssetRecordType,
+	Editor,
 	TLShape,
 	TLShapeId,
 	debounce,
@@ -22,7 +23,7 @@ export function useLiveImage(
 		appId: string
 	}
 ) {
-	const { appId, throttleTime = 100, debounceTime = 0 } = opts
+	const { appId, throttleTime = 30, debounceTime = 0 } = opts
 	const editor = useEditor()
 	const startedIteration = useRef<number>(0)
 	const finishedIteration = useRef<number>(0)
@@ -76,6 +77,7 @@ export function useLiveImage(
 				console.error(error)
 			},
 			onResult: (result) => {
+				console.log(result)
 				if (result.images && result.images[0]) {
 					updateImage(result.images[0].url)
 				}
@@ -85,20 +87,19 @@ export function useLiveImage(
 		async function updateDrawing() {
 			const iteration = startedIteration.current++
 
-			const shapes = Array.from(editor.getShapeAndDescendantIds([shapeId]))
+			const shapes = getShapesTouching(shapeId, editor)
-				.filter((id) => id !== shapeId)
-				.map((id) => editor.getShape(id)) as TLShape[]
 
 			const shape = editor.getShape<LiveImageShape>(shapeId)!
-			const hash = getHashForObject(shapes)
+			const hash = getHashForObject([...shapes])
 			if (hash === prevHash.current && shape.props.name === prevPrompt.current) return
 			prevHash.current = hash
 			prevPrompt.current = shape.props.name
 
-			const svg = await editor.getSvg([shape], {
+			const svg = await editor.getSvg([...shapes], {
 				background: true,
 				padding: 0,
 				darkMode: editor.user.getIsDarkMode(),
+				bounds: editor.getShapePageBounds(shapeId)!,
 			})
 			// We might be stale
 			if (iteration <= finishedIteration.current) return
@@ -113,6 +114,7 @@ export function useLiveImage(
 				quality: 1,
 				scale: 512 / shape.props.w,
 			})
+
 			// We might be stale
 			if (iteration <= finishedIteration.current) return
 			if (!image) {
@@ -124,6 +126,7 @@ export function useLiveImage(
 				? shape.props.name + ' hd award-winning impressive'
 				: 'A random image that is safe for work and not surprising—something boring like a city or shoe watercolor'
 			const imageDataUri = await blobToDataUri(image)
+			// downloadDataURLAsFile(imageDataUri, 'test.png')
 			// We might be stale
 			if (iteration <= finishedIteration.current) return
 
@@ -134,7 +137,7 @@ export function useLiveImage(
 					prompt,
 					image_url: imageDataUri,
 					sync_mode: true,
-					strength: 0.65,
+					strength: 0.5,
 					seed: Math.abs(random() * 10000), // TODO make this configurable in the UI
 					enable_safety_checks: false,
 				})
@@ -162,3 +165,25 @@ export function useLiveImage(
 		}
 	}, [editor, shapeId, throttleTime, debounceTime, appId])
 }
+
+function getShapesTouching(shapeId: TLShapeId, editor: Editor) {
+	const shapeIdsOnPage = editor.getCurrentPageShapeIds()
+	const shapesTouching: TLShape[] = []
+	const targetBounds = editor.getShapePageBounds(shapeId)
+	if (!targetBounds) return shapesTouching
+	for (const id of [...shapeIdsOnPage]) {
+		if (id === shapeId) continue
+		const bounds = editor.getShapePageBounds(id)!
+		if (bounds.collides(targetBounds)) {
+			shapesTouching.push(editor.getShape(id)!)
+		}
+	}
+	return shapesTouching
+}
+
+function downloadDataURLAsFile(dataUrl: string, filename: string) {
+	const link = document.createElement('a')
+	link.href = dataUrl
+	link.download = filename
+	link.click()
+}

tsconfig.json

@@ -1,27 +1,28 @@
 {
-  "compilerOptions": {
+	"compilerOptions": {
-    "target": "es5",
+		"target": "es5",
-    "lib": ["dom", "dom.iterable", "esnext"],
+		"lib": ["dom", "dom.iterable", "esnext"],
-    "allowJs": true,
+		"allowJs": true,
-    "skipLibCheck": true,
+		"skipLibCheck": true,
-    "strict": true,
+		"strict": true,
-    "noEmit": true,
+		"noEmit": true,
-    "esModuleInterop": true,
+		"esModuleInterop": true,
-    "module": "esnext",
+		"module": "esnext",
-    "moduleResolution": "bundler",
+		"moduleResolution": "bundler",
-    "resolveJsonModule": true,
+		"resolveJsonModule": true,
-    "isolatedModules": true,
+		"isolatedModules": true,
-    "jsx": "preserve",
+		"jsx": "preserve",
-    "incremental": true,
+		"incremental": true,
-    "plugins": [
+		"downlevelIteration": true,
-      {
+		"plugins": [
-        "name": "next"
+			{
-      }
+				"name": "next"
-    ],
+			}
-    "paths": {
+		],
-      "@/*": ["./src/*"]
+		"paths": {
-    }
+			"@/*": ["./src/*"]
-  },
+		}
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+	},
-  "exclude": ["node_modules"]
+	"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+	"exclude": ["node_modules"]
 }