more tests
This commit is contained in:
Lu[ke] Wilson
parent
6be0379c49
commit
7e6fcbe1dc
|
|
@ -2,8 +2,9 @@
|
|||
|
||||
import { LiveImageShapeUtil } from "@/components/live-image";
|
||||
import * as fal from "@fal-ai/serverless-client";
|
||||
import { Editor, Tldraw } from "@tldraw/tldraw";
|
||||
import { Editor, FrameShapeTool, Tldraw, useEditor } from "@tldraw/tldraw";
|
||||
import { useCallback } from "react";
|
||||
import { LiveImageTool, MakeLiveButton } from "../components/LiveImageTool";
|
||||
|
||||
fal.config({
|
||||
requestMiddleware: fal.withProxy({
|
||||
|
|
@ -12,6 +13,7 @@ fal.config({
|
|||
});
|
||||
|
||||
const shapeUtils = [LiveImageShapeUtil];
|
||||
const tools = [LiveImageTool];
|
||||
|
||||
export default function Home() {
|
||||
const onEditorMount = (editor: Editor) => {
|
||||
|
|
@ -28,7 +30,11 @@ export default function Home() {
|
|||
type: "live-image",
|
||||
x: 120,
|
||||
y: 180,
|
||||
isLocked: true,
|
||||
props: {
|
||||
w: 512,
|
||||
h: 512,
|
||||
name: "a city skyline",
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
|
|
@ -39,6 +45,8 @@ export default function Home() {
|
|||
persistenceKey="tldraw-fal"
|
||||
onMount={onEditorMount}
|
||||
shapeUtils={shapeUtils}
|
||||
tools={tools}
|
||||
shareZone={<MakeLiveButton />}
|
||||
/>
|
||||
</div>
|
||||
</main>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,27 @@
|
|||
import { FrameShapeTool, useEditor } from "@tldraw/tldraw";
|
||||
import { useCallback } from "react";
|
||||
|
||||
export class LiveImageTool extends FrameShapeTool {
|
||||
static override id = "live-image";
|
||||
static override initial = "idle";
|
||||
override shapeType = "live-image";
|
||||
}
|
||||
|
||||
export function MakeLiveButton() {
|
||||
const editor = useEditor();
|
||||
const makeLive = useCallback(() => {
|
||||
editor.setCurrentTool("live-image");
|
||||
}, [editor]);
|
||||
|
||||
return (
|
||||
<button
|
||||
onClick={makeLive}
|
||||
className="p-2"
|
||||
style={{ cursor: "pointer", zIndex: 100000, pointerEvents: "all" }}
|
||||
>
|
||||
<div className="bg-blue-500 hover:bg-blue-700 text-white font-bold py-2 px-4 rounded">
|
||||
Make Live
|
||||
</div>
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,10 +1,11 @@
|
|||
/* eslint-disable @next/next/no-img-element */
|
||||
/* eslint-disable react-hooks/rules-of-hooks */
|
||||
import {
|
||||
Box2d,
|
||||
FrameShapeUtil,
|
||||
getSvgAsImage,
|
||||
Rectangle2d,
|
||||
ShapeUtil,
|
||||
TLBaseShape,
|
||||
HTMLContainer,
|
||||
TLEventMapHandler,
|
||||
TLFrameShape,
|
||||
TLShape,
|
||||
useEditor,
|
||||
} from "@tldraw/tldraw";
|
||||
|
|
@ -13,7 +14,6 @@ import { blobToDataUri } from "@/utils/blob";
|
|||
import { debounce } from "@/utils/debounce";
|
||||
import * as fal from "@fal-ai/serverless-client";
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { FalLogo } from "./fal-logo";
|
||||
|
||||
// See https://www.fal.ai/models/latent-consistency-sd
|
||||
|
||||
|
|
@ -24,6 +24,10 @@ type Input = {
|
|||
image_url: string;
|
||||
sync_mode: boolean;
|
||||
seed: number;
|
||||
strength?: number;
|
||||
guidance_scale?: number;
|
||||
num_inference_steps?: number;
|
||||
enable_safety_checks?: boolean;
|
||||
};
|
||||
|
||||
type Output = {
|
||||
|
|
@ -36,153 +40,134 @@ type Output = {
|
|||
num_inference_steps: number;
|
||||
};
|
||||
|
||||
// TODO make this an input on the canvas
|
||||
const PROMPT = "a city skyline";
|
||||
export class LiveImageShapeUtil extends FrameShapeUtil {
|
||||
static override type = "live-image" as any;
|
||||
|
||||
export function LiveImage() {
|
||||
const editor = useEditor();
|
||||
const [image, setImage] = useState<string | null>(null);
|
||||
|
||||
// Used to prevent multiple requests from being sent at once for the same image
|
||||
// There's probably a better way to do this using TLDraw's state
|
||||
const imageDigest = useRef<string | null>(null);
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
const onDrawingChange = useCallback(
|
||||
debounce(async () => {
|
||||
// TODO get actual drawing bounds
|
||||
// const bounds = new Box2d(120, 180, 512, 512);
|
||||
|
||||
const shapes = editor.getCurrentPageShapes().filter((shape) => {
|
||||
if (shape.type === "live-image") {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
// const pageBounds = editor.getShapeMaskedPageBounds(shape);
|
||||
// if (!pageBounds) {
|
||||
// return false;
|
||||
// }
|
||||
// return bounds.includes(pageBounds);
|
||||
});
|
||||
|
||||
// Check if should submit request
|
||||
const shapesDigest = JSON.stringify(shapes);
|
||||
if (shapesDigest === imageDigest.current) {
|
||||
return;
|
||||
}
|
||||
imageDigest.current = shapesDigest;
|
||||
|
||||
const svg = await editor.getSvg(shapes, { background: true });
|
||||
if (!svg) {
|
||||
return;
|
||||
}
|
||||
const image = await getSvgAsImage(svg, editor.environment.isSafari, {
|
||||
type: "png",
|
||||
quality: 0.5,
|
||||
scale: 1,
|
||||
});
|
||||
if (!image) {
|
||||
return;
|
||||
}
|
||||
|
||||
const imageDataUri = await blobToDataUri(image);
|
||||
const result = await fal.run<Input, Output>(LatentConsistency, {
|
||||
input: {
|
||||
image_url: imageDataUri,
|
||||
prompt: PROMPT,
|
||||
sync_mode: true,
|
||||
seed: 42, // TODO make this configurable in the UI
|
||||
},
|
||||
// Disable auto-upload so we can submit the data uri of the image as is
|
||||
autoUpload: false,
|
||||
});
|
||||
if (result && result.images.length > 0) {
|
||||
setImage(result.images[0].url);
|
||||
}
|
||||
}, 16),
|
||||
[]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const onChange: TLEventMapHandler<"change"> = (event) => {
|
||||
if (event.source !== "user") {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
Object.keys(event.changes.added).length ||
|
||||
Object.keys(event.changes.removed).length ||
|
||||
Object.keys(event.changes.updated).length
|
||||
) {
|
||||
onDrawingChange();
|
||||
}
|
||||
};
|
||||
editor.addListener("change", onChange);
|
||||
return () => {
|
||||
editor.removeListener("change", onChange);
|
||||
};
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<div className="flex flex-row w-[1060px] h-[560px] absolute bg-indigo-200 border border-indigo-500 rounded space-x-4 p-4 pb-8">
|
||||
<div className="flex-1 h-[512px] bg-white border border-indigo-500">
|
||||
<div className="flex flex-row items-center px-4 py-2 space-x-2">
|
||||
<span className="font-mono text-indigo-900/50">/imagine</span>
|
||||
<input
|
||||
className="border-0 bg-transparent flex-1 text-base text-indigo-900"
|
||||
placeholder="something cool..."
|
||||
value={PROMPT}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex-1 h-[512px] bg-white border border-indigo-500">
|
||||
{/* eslint-disable-next-line @next/next/no-img-element */}
|
||||
{image && <img src={image} alt="" width={512} height={512} />}
|
||||
</div>
|
||||
<span className="absolute bottom-1.5 right-4">
|
||||
<a
|
||||
href="https://fal.ai/models/latent-consistency"
|
||||
target="_blank"
|
||||
className="flex flex-row space-x-1"
|
||||
>
|
||||
<span className="text-xs text-indigo-900/50">powered by</span>
|
||||
<span className="w-[36px] opacity-50">
|
||||
<FalLogo />
|
||||
</span>
|
||||
</a>
|
||||
</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
type LiveImageShape = TLBaseShape<"live-image", { w: number; h: number }>;
|
||||
|
||||
export class LiveImageShapeUtil extends ShapeUtil<LiveImageShape> {
|
||||
static override type = "live-image" as const;
|
||||
|
||||
override canResize = () => false;
|
||||
|
||||
getDefaultProps(): LiveImageShape["props"] {
|
||||
override getDefaultProps(): { w: number; h: number; name: string } {
|
||||
return {
|
||||
w: 1060,
|
||||
h: 560,
|
||||
w: 512,
|
||||
h: 512,
|
||||
name: "a city skyline",
|
||||
};
|
||||
}
|
||||
|
||||
getGeometry(shape: LiveImageShape) {
|
||||
return new Rectangle2d({
|
||||
width: shape.props.w,
|
||||
height: shape.props.h,
|
||||
isFilled: true,
|
||||
});
|
||||
}
|
||||
override component(shape: TLFrameShape) {
|
||||
const editor = useEditor();
|
||||
const component = super.component(shape);
|
||||
const [image, setImage] = useState<string | null>(null);
|
||||
|
||||
component(shape: LiveImageShape) {
|
||||
return <LiveImage />;
|
||||
}
|
||||
const imageDigest = useRef<string | null>(null);
|
||||
const startedIteration = useRef<number>(0);
|
||||
const finishedIteration = useRef<number>(0);
|
||||
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
const onDrawingChange = useCallback(
|
||||
debounce(async () => {
|
||||
// TODO get actual drawing bounds
|
||||
// const bounds = new Box2d(120, 180, 512, 512);
|
||||
|
||||
const iteration = startedIteration.current++;
|
||||
|
||||
const shapes = Array.from(editor.getShapeAndDescendantIds([shape.id]))
|
||||
.filter((id) => id !== shape.id)
|
||||
.map((id) => editor.getShape(id)) as TLShape[];
|
||||
|
||||
// Check if should submit request
|
||||
const shapesDigest = JSON.stringify(shapes);
|
||||
if (shapesDigest === imageDigest.current) {
|
||||
return;
|
||||
}
|
||||
imageDigest.current = shapesDigest;
|
||||
|
||||
const svg = await editor.getSvg(shapes, { background: true });
|
||||
if (iteration <= finishedIteration.current) return;
|
||||
|
||||
if (!svg) {
|
||||
return;
|
||||
}
|
||||
const image = await getSvgAsImage(svg, editor.environment.isSafari, {
|
||||
type: "png",
|
||||
quality: 1,
|
||||
scale: 1,
|
||||
});
|
||||
|
||||
if (iteration <= finishedIteration.current) return;
|
||||
|
||||
if (!image) {
|
||||
return;
|
||||
}
|
||||
|
||||
const prompt =
|
||||
editor.getShape<TLFrameShape>(shape.id)?.props.name ?? "";
|
||||
const imageDataUri = await blobToDataUri(image);
|
||||
if (iteration <= finishedIteration.current) return;
|
||||
|
||||
const result = await fal.run<Input, Output>(LatentConsistency, {
|
||||
input: {
|
||||
image_url: imageDataUri,
|
||||
prompt,
|
||||
sync_mode: true,
|
||||
strength: 0.6,
|
||||
seed: 42, // TODO make this configurable in the UI
|
||||
enable_safety_checks: false,
|
||||
},
|
||||
// Disable auto-upload so we can submit the data uri of the image as is
|
||||
autoUpload: true,
|
||||
});
|
||||
if (iteration <= finishedIteration.current) return;
|
||||
|
||||
finishedIteration.current = iteration;
|
||||
if (result && result.images.length > 0) {
|
||||
setImage(result.images[0].url);
|
||||
}
|
||||
}, 32),
|
||||
[]
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const onChange: TLEventMapHandler<"change"> = (event) => {
|
||||
if (event.source !== "user") {
|
||||
return;
|
||||
}
|
||||
if (
|
||||
Object.keys(event.changes.added).length ||
|
||||
Object.keys(event.changes.removed).length ||
|
||||
Object.keys(event.changes.updated).length
|
||||
) {
|
||||
onDrawingChange();
|
||||
}
|
||||
};
|
||||
editor.addListener("change", onChange);
|
||||
return () => {
|
||||
editor.removeListener("change", onChange);
|
||||
};
|
||||
}, [editor, onDrawingChange]);
|
||||
|
||||
indicator(shape: LiveImageShape) {
|
||||
return (
|
||||
<rect width={shape.props.w} height={shape.props.h} radius={4}></rect>
|
||||
<HTMLContainer>
|
||||
<div
|
||||
style={{
|
||||
display: "flex",
|
||||
}}
|
||||
>
|
||||
{component}
|
||||
|
||||
{image && (
|
||||
<img
|
||||
src={image}
|
||||
alt=""
|
||||
width={shape.props.w}
|
||||
height={shape.props.h}
|
||||
style={{
|
||||
position: "relative",
|
||||
left: shape.props.w,
|
||||
width: shape.props.w,
|
||||
height: shape.props.h,
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
</HTMLContainer>
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue