Add gesture recognition and collaborative sync to folk-drawfast
Implements $1 Unistroke Recognizer for detecting circles, rectangles, triangles, lines, arrows, and checkmarks from freehand strokes. Detected gestures are converted to clean geometric shapes with a confidence badge. Fixes applyData() to restore strokes, prompt text, and generated images from Automerge sync data, enabling collaborative drawing across clients. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
f58445c35e
commit
edabad18e4
|
|
@ -309,6 +309,28 @@ const styles = css`
|
|||
.export-btn:hover {
|
||||
opacity: 0.9;
|
||||
}
|
||||
|
||||
.gesture-badge {
|
||||
position: absolute;
|
||||
top: 8px;
|
||||
left: 50%;
|
||||
transform: translateX(-50%);
|
||||
background: rgba(249, 115, 22, 0.9);
|
||||
color: white;
|
||||
padding: 4px 12px;
|
||||
border-radius: 12px;
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
z-index: 3;
|
||||
pointer-events: none;
|
||||
animation: badge-fade 1.5s ease-out forwards;
|
||||
}
|
||||
|
||||
@keyframes badge-fade {
|
||||
0% { opacity: 1; transform: translateX(-50%) translateY(0); }
|
||||
70% { opacity: 1; }
|
||||
100% { opacity: 0; transform: translateX(-50%) translateY(-10px); }
|
||||
}
|
||||
`;
|
||||
|
||||
const COLORS = ["#0f172a", "#ef4444", "#f97316", "#eab308", "#22c55e", "#3b82f6", "#8b5cf6", "#ec4899", "#ffffff"];
|
||||
|
|
@ -320,6 +342,213 @@ interface Stroke {
|
|||
tool: string;
|
||||
}
|
||||
|
||||
// --- $1 Unistroke Recognizer (lightweight, self-contained) ---
|
||||
// Based on Wobbrock et al. 2007, adapted from canvas-website gesture templates
|
||||
|
||||
interface Point2D { x: number; y: number; }
|
||||
interface RecognizeResult { name: string; score: number; }
|
||||
|
||||
const NUM_POINTS = 64;
|
||||
const SQUARE_SIZE = 250;
|
||||
const HALF_DIAGONAL = 0.5 * Math.sqrt(SQUARE_SIZE * SQUARE_SIZE + SQUARE_SIZE * SQUARE_SIZE);
|
||||
const ANGLE_RANGE = Math.PI * 2;
|
||||
const ANGLE_PRECISION = Math.PI / 90; // 2 degrees
|
||||
const PHI = 0.5 * (-1 + Math.sqrt(5)); // golden ratio
|
||||
|
||||
function resample(pts: Point2D[], n: number): Point2D[] {
|
||||
const totalLen = pathLength(pts);
|
||||
const interval = totalLen / (n - 1);
|
||||
const newPts: Point2D[] = [pts[0]];
|
||||
let D = 0;
|
||||
for (let i = 1; i < pts.length; i++) {
|
||||
const d = distance(pts[i - 1], pts[i]);
|
||||
if (D + d >= interval) {
|
||||
const t = (interval - D) / d;
|
||||
const qx = pts[i - 1].x + t * (pts[i].x - pts[i - 1].x);
|
||||
const qy = pts[i - 1].y + t * (pts[i].y - pts[i - 1].y);
|
||||
const q: Point2D = { x: qx, y: qy };
|
||||
newPts.push(q);
|
||||
pts.splice(i, 0, q);
|
||||
D = 0;
|
||||
} else {
|
||||
D += d;
|
||||
}
|
||||
}
|
||||
while (newPts.length < n) newPts.push(pts[pts.length - 1]);
|
||||
return newPts;
|
||||
}
|
||||
|
||||
function rotateToZero(pts: Point2D[]): Point2D[] {
|
||||
const c = centroid(pts);
|
||||
const angle = Math.atan2(c.y - pts[0].y, c.x - pts[0].x);
|
||||
return rotateBy(pts, -angle);
|
||||
}
|
||||
|
||||
function scaleToSquare(pts: Point2D[]): Point2D[] {
|
||||
let minX = Infinity, maxX = -Infinity, minY = Infinity, maxY = -Infinity;
|
||||
for (const p of pts) {
|
||||
minX = Math.min(minX, p.x); maxX = Math.max(maxX, p.x);
|
||||
minY = Math.min(minY, p.y); maxY = Math.max(maxY, p.y);
|
||||
}
|
||||
const w = maxX - minX || 1;
|
||||
const h = maxY - minY || 1;
|
||||
return pts.map(p => ({ x: p.x * (SQUARE_SIZE / w), y: p.y * (SQUARE_SIZE / h) }));
|
||||
}
|
||||
|
||||
function translateToOrigin(pts: Point2D[]): Point2D[] {
|
||||
const c = centroid(pts);
|
||||
return pts.map(p => ({ x: p.x - c.x, y: p.y - c.y }));
|
||||
}
|
||||
|
||||
function recognize(pts: Point2D[], templates: { name: string; points: Point2D[] }[]): RecognizeResult {
|
||||
let best = Infinity;
|
||||
let bestName = "none";
|
||||
for (const t of templates) {
|
||||
const d = distanceAtBestAngle(pts, t.points, -ANGLE_RANGE, ANGLE_RANGE, ANGLE_PRECISION);
|
||||
if (d < best) {
|
||||
best = d;
|
||||
bestName = t.name;
|
||||
}
|
||||
}
|
||||
const score = 1 - best / HALF_DIAGONAL;
|
||||
return { name: bestName, score };
|
||||
}
|
||||
|
||||
function distanceAtBestAngle(pts: Point2D[], template: Point2D[], a: number, b: number, threshold: number): number {
|
||||
let x1 = PHI * a + (1 - PHI) * b;
|
||||
let f1 = distanceAtAngle(pts, template, x1);
|
||||
let x2 = (1 - PHI) * a + PHI * b;
|
||||
let f2 = distanceAtAngle(pts, template, x2);
|
||||
while (Math.abs(b - a) > threshold) {
|
||||
if (f1 < f2) {
|
||||
b = x2; x2 = x1; f2 = f1;
|
||||
x1 = PHI * a + (1 - PHI) * b;
|
||||
f1 = distanceAtAngle(pts, template, x1);
|
||||
} else {
|
||||
a = x1; x1 = x2; f1 = f2;
|
||||
x2 = (1 - PHI) * a + PHI * b;
|
||||
f2 = distanceAtAngle(pts, template, x2);
|
||||
}
|
||||
}
|
||||
return Math.min(f1, f2);
|
||||
}
|
||||
|
||||
function distanceAtAngle(pts: Point2D[], template: Point2D[], angle: number): number {
|
||||
const rotated = rotateBy(pts, angle);
|
||||
return pathDistance(rotated, template);
|
||||
}
|
||||
|
||||
function centroid(pts: Point2D[]): Point2D {
|
||||
let x = 0, y = 0;
|
||||
for (const p of pts) { x += p.x; y += p.y; }
|
||||
return { x: x / pts.length, y: y / pts.length };
|
||||
}
|
||||
|
||||
function rotateBy(pts: Point2D[], angle: number): Point2D[] {
|
||||
const c = centroid(pts);
|
||||
const cos = Math.cos(angle), sin = Math.sin(angle);
|
||||
return pts.map(p => ({
|
||||
x: (p.x - c.x) * cos - (p.y - c.y) * sin + c.x,
|
||||
y: (p.x - c.x) * sin + (p.y - c.y) * cos + c.y,
|
||||
}));
|
||||
}
|
||||
|
||||
function pathDistance(a: Point2D[], b: Point2D[]): number {
|
||||
let d = 0;
|
||||
const n = Math.min(a.length, b.length);
|
||||
for (let i = 0; i < n; i++) d += distance(a[i], b[i]);
|
||||
return d / n;
|
||||
}
|
||||
|
||||
function pathLength(pts: Point2D[]): number {
|
||||
let d = 0;
|
||||
for (let i = 1; i < pts.length; i++) d += distance(pts[i - 1], pts[i]);
|
||||
return d;
|
||||
}
|
||||
|
||||
function distance(a: Point2D, b: Point2D): number {
|
||||
const dx = b.x - a.x, dy = b.y - a.y;
|
||||
return Math.sqrt(dx * dx + dy * dy);
|
||||
}
|
||||
|
||||
function processTemplate(pts: Point2D[]): Point2D[] {
|
||||
return translateToOrigin(scaleToSquare(rotateToZero(resample(pts, NUM_POINTS))));
|
||||
}
|
||||
|
||||
// Generate templates procedurally (more compact than storing point arrays)
|
||||
function makeCircleTemplate(): Point2D[] {
|
||||
const pts: Point2D[] = [];
|
||||
for (let i = 0; i <= 32; i++) {
|
||||
const a = (i / 32) * Math.PI * 2;
|
||||
pts.push({ x: 100 + 80 * Math.cos(a), y: 100 + 80 * Math.sin(a) });
|
||||
}
|
||||
return processTemplate(pts);
|
||||
}
|
||||
|
||||
function makeRectangleTemplate(): Point2D[] {
|
||||
const pts: Point2D[] = [];
|
||||
// Draw rectangle starting top-left, clockwise
|
||||
const steps = 8;
|
||||
for (let i = 0; i <= steps; i++) pts.push({ x: 20 + (160 * i / steps), y: 20 }); // top
|
||||
for (let i = 0; i <= steps; i++) pts.push({ x: 180, y: 20 + (160 * i / steps) }); // right
|
||||
for (let i = 0; i <= steps; i++) pts.push({ x: 180 - (160 * i / steps), y: 180 }); // bottom
|
||||
for (let i = 0; i <= steps; i++) pts.push({ x: 20, y: 180 - (160 * i / steps) }); // left
|
||||
return processTemplate(pts);
|
||||
}
|
||||
|
||||
function makeLineTemplate(): Point2D[] {
|
||||
const pts: Point2D[] = [];
|
||||
for (let i = 0; i <= 16; i++) pts.push({ x: 20 + (160 * i / 16), y: 100 });
|
||||
return processTemplate(pts);
|
||||
}
|
||||
|
||||
function makeArrowTemplate(): Point2D[] {
|
||||
// Horizontal line with arrowhead at the end
|
||||
const pts: Point2D[] = [];
|
||||
for (let i = 0; i <= 12; i++) pts.push({ x: 20 + (140 * i / 12), y: 100 }); // shaft
|
||||
for (let i = 0; i <= 4; i++) pts.push({ x: 160 - (40 * i / 4), y: 100 - (40 * i / 4) }); // upper head
|
||||
pts.push({ x: 160, y: 100 }); // back to tip
|
||||
for (let i = 0; i <= 4; i++) pts.push({ x: 160 - (40 * i / 4), y: 100 + (40 * i / 4) }); // lower head
|
||||
return processTemplate(pts);
|
||||
}
|
||||
|
||||
function makeTriangleTemplate(): Point2D[] {
|
||||
const pts: Point2D[] = [];
|
||||
const steps = 8;
|
||||
// Top to bottom-right
|
||||
for (let i = 0; i <= steps; i++) pts.push({ x: 100 + (80 * i / steps), y: 20 + (160 * i / steps) });
|
||||
// Bottom-right to bottom-left
|
||||
for (let i = 0; i <= steps; i++) pts.push({ x: 180 - (160 * i / steps), y: 180 });
|
||||
// Bottom-left to top
|
||||
for (let i = 0; i <= steps; i++) pts.push({ x: 20 + (80 * i / steps), y: 180 - (160 * i / steps) });
|
||||
return processTemplate(pts);
|
||||
}
|
||||
|
||||
function makeCheckTemplate(): Point2D[] {
|
||||
const pts: Point2D[] = [];
|
||||
for (let i = 0; i <= 6; i++) pts.push({ x: 20 + (40 * i / 6), y: 100 + (60 * i / 6) });
|
||||
for (let i = 0; i <= 8; i++) pts.push({ x: 60 + (120 * i / 8), y: 160 - (140 * i / 8) });
|
||||
return processTemplate(pts);
|
||||
}
|
||||
|
||||
const GESTURE_TEMPLATES = [
|
||||
{ name: "circle", points: makeCircleTemplate() },
|
||||
{ name: "rectangle", points: makeRectangleTemplate() },
|
||||
{ name: "line", points: makeLineTemplate() },
|
||||
{ name: "arrow", points: makeArrowTemplate() },
|
||||
{ name: "triangle", points: makeTriangleTemplate() },
|
||||
{ name: "check", points: makeCheckTemplate() },
|
||||
];
|
||||
|
||||
function recognizeGesture(rawPoints: Point2D[]): RecognizeResult | null {
|
||||
if (rawPoints.length < 8) return null; // too few points
|
||||
const processed = processTemplate(rawPoints);
|
||||
const result = recognize(processed, GESTURE_TEMPLATES);
|
||||
if (result.score < 0.7) return null; // low confidence
|
||||
return result;
|
||||
}
|
||||
// --- End $1 Unistroke Recognizer ---
|
||||
|
||||
declare global {
|
||||
interface HTMLElementTagNameMap {
|
||||
"folk-drawfast": FolkDrawfast;
|
||||
|
|
@ -364,6 +593,8 @@ export class FolkDrawfast extends FolkShape {
|
|||
#promptInput: HTMLInputElement | null = null;
|
||||
#generateBtn: HTMLButtonElement | null = null;
|
||||
#resultArea: HTMLElement | null = null;
|
||||
#canvasArea: HTMLElement | null = null;
|
||||
#gestureEnabled = true;
|
||||
|
||||
get strokes() {
|
||||
return this.#strokes;
|
||||
|
|
@ -443,6 +674,7 @@ export class FolkDrawfast extends FolkShape {
|
|||
const sizeSlider = wrapper.querySelector(".size-slider") as HTMLInputElement;
|
||||
const sizeLabel = wrapper.querySelector(".size-label") as HTMLElement;
|
||||
const canvasArea = wrapper.querySelector(".canvas-area") as HTMLElement;
|
||||
this.#canvasArea = canvasArea;
|
||||
const autoCheckbox = wrapper.querySelector(".auto-checkbox") as HTMLInputElement;
|
||||
const providerSelect = wrapper.querySelector(".provider-select") as HTMLSelectElement;
|
||||
const strengthSlider = wrapper.querySelector(".strength-slider") as HTMLInputElement;
|
||||
|
|
@ -533,10 +765,29 @@ export class FolkDrawfast extends FolkShape {
|
|||
if (!this.#isDrawing) return;
|
||||
this.#isDrawing = false;
|
||||
if (this.#currentStroke && this.#currentStroke.points.length > 0) {
|
||||
// Try gesture recognition before adding stroke
|
||||
let gestureResult: RecognizeResult | null = null;
|
||||
if (this.#gestureEnabled && this.#currentStroke.tool === "pen") {
|
||||
const rawPts = this.#currentStroke.points.map(p => ({ x: p.x, y: p.y }));
|
||||
gestureResult = recognizeGesture(rawPts);
|
||||
}
|
||||
|
||||
if (gestureResult) {
|
||||
// Replace freehand stroke with clean geometric shape
|
||||
const cleanStroke = this.#makeCleanShape(gestureResult.name, this.#currentStroke);
|
||||
this.#strokes.push(cleanStroke);
|
||||
this.#redraw();
|
||||
this.#showGestureBadge(gestureResult.name, gestureResult.score);
|
||||
this.dispatchEvent(new CustomEvent("stroke-complete", {
|
||||
detail: { stroke: cleanStroke, gesture: gestureResult.name },
|
||||
}));
|
||||
} else {
|
||||
this.#strokes.push(this.#currentStroke);
|
||||
this.dispatchEvent(new CustomEvent("stroke-complete", {
|
||||
detail: { stroke: this.#currentStroke },
|
||||
}));
|
||||
}
|
||||
|
||||
// Auto-generate on stroke complete if enabled
|
||||
if (this.#autoGenerate && this.#promptInput?.value.trim()) {
|
||||
this.#scheduleAutoGenerate();
|
||||
|
|
@ -774,6 +1025,95 @@ export class FolkDrawfast extends FolkShape {
|
|||
ctx.globalCompositeOperation = "source-over";
|
||||
}
|
||||
|
||||
#makeCleanShape(gesture: string, original: Stroke): Stroke {
|
||||
const pts = original.points;
|
||||
const minX = Math.min(...pts.map(p => p.x));
|
||||
const maxX = Math.max(...pts.map(p => p.x));
|
||||
const minY = Math.min(...pts.map(p => p.y));
|
||||
const maxY = Math.max(...pts.map(p => p.y));
|
||||
const cx = (minX + maxX) / 2;
|
||||
const cy = (minY + maxY) / 2;
|
||||
const w = maxX - minX || 1;
|
||||
const h = maxY - minY || 1;
|
||||
const pressure = 0.5;
|
||||
|
||||
const toStroke = (points: { x: number; y: number }[]): Stroke => ({
|
||||
points: points.map(p => ({ ...p, pressure })),
|
||||
color: original.color,
|
||||
size: original.size,
|
||||
tool: "pen",
|
||||
});
|
||||
|
||||
switch (gesture) {
|
||||
case "circle": {
|
||||
const rx = w / 2, ry = h / 2;
|
||||
const circPts: { x: number; y: number }[] = [];
|
||||
for (let i = 0; i <= 48; i++) {
|
||||
const a = (i / 48) * Math.PI * 2;
|
||||
circPts.push({ x: cx + rx * Math.cos(a), y: cy + ry * Math.sin(a) });
|
||||
}
|
||||
return toStroke(circPts);
|
||||
}
|
||||
case "rectangle": {
|
||||
return toStroke([
|
||||
{ x: minX, y: minY }, { x: maxX, y: minY },
|
||||
{ x: maxX, y: maxY }, { x: minX, y: maxY },
|
||||
{ x: minX, y: minY },
|
||||
]);
|
||||
}
|
||||
case "triangle": {
|
||||
return toStroke([
|
||||
{ x: cx, y: minY }, { x: maxX, y: maxY },
|
||||
{ x: minX, y: maxY }, { x: cx, y: minY },
|
||||
]);
|
||||
}
|
||||
case "line": {
|
||||
// Use first and last point for direction
|
||||
const first = pts[0], last = pts[pts.length - 1];
|
||||
return toStroke([
|
||||
{ x: first.x, y: first.y },
|
||||
{ x: last.x, y: last.y },
|
||||
]);
|
||||
}
|
||||
case "arrow": {
|
||||
const first = pts[0], last = pts[pts.length - 1];
|
||||
const dx = last.x - first.x, dy = last.y - first.y;
|
||||
const len = Math.sqrt(dx * dx + dy * dy) || 1;
|
||||
const ux = dx / len, uy = dy / len;
|
||||
const headLen = Math.min(20, len * 0.3);
|
||||
// Arrow shaft + two head lines
|
||||
return toStroke([
|
||||
{ x: first.x, y: first.y }, { x: last.x, y: last.y },
|
||||
{ x: last.x - headLen * (ux + uy * 0.5), y: last.y - headLen * (uy - ux * 0.5) },
|
||||
{ x: last.x, y: last.y },
|
||||
{ x: last.x - headLen * (ux - uy * 0.5), y: last.y - headLen * (uy + ux * 0.5) },
|
||||
]);
|
||||
}
|
||||
case "check": {
|
||||
// V-shape: find the lowest point as the vertex
|
||||
const lowestIdx = pts.reduce((best, p, i) => p.y > pts[best].y ? i : best, 0);
|
||||
return toStroke([
|
||||
{ x: pts[0].x, y: pts[0].y },
|
||||
{ x: pts[lowestIdx].x, y: pts[lowestIdx].y },
|
||||
{ x: pts[pts.length - 1].x, y: pts[pts.length - 1].y },
|
||||
]);
|
||||
}
|
||||
default:
|
||||
return original;
|
||||
}
|
||||
}
|
||||
|
||||
#showGestureBadge(name: string, score: number) {
|
||||
if (!this.#canvasArea) return;
|
||||
// Remove existing badge
|
||||
this.#canvasArea.querySelector(".gesture-badge")?.remove();
|
||||
const badge = document.createElement("div");
|
||||
badge.className = "gesture-badge";
|
||||
badge.textContent = `${name} (${Math.round(score * 100)}%)`;
|
||||
this.#canvasArea.appendChild(badge);
|
||||
setTimeout(() => badge.remove(), 1500);
|
||||
}
|
||||
|
||||
#exportPNG() {
|
||||
if (!this.#canvas) return;
|
||||
const dataUrl = this.#canvas.toDataURL("image/png");
|
||||
|
|
@ -808,12 +1148,31 @@ export class FolkDrawfast extends FolkShape {
|
|||
size: s.size,
|
||||
tool: s.tool,
|
||||
})),
|
||||
prompt: this.#promptInput?.value || "",
|
||||
lastResultUrl: this.#lastResultUrl,
|
||||
};
|
||||
}
|
||||
|
||||
override applyData(data: Record<string, any>): void {
|
||||
super.applyData(data);
|
||||
|
||||
// Restore strokes from sync data
|
||||
if (data.strokes && Array.isArray(data.strokes)) {
|
||||
this.#strokes = data.strokes.map((s: any) => ({
|
||||
points: Array.isArray(s.points) ? s.points : [],
|
||||
color: s.color || "#0f172a",
|
||||
size: s.size || 4,
|
||||
tool: s.tool || "pen",
|
||||
}));
|
||||
this.#redraw();
|
||||
}
|
||||
|
||||
// Restore prompt text
|
||||
if (data.prompt !== undefined && this.#promptInput) {
|
||||
this.#promptInput.value = data.prompt || "";
|
||||
}
|
||||
|
||||
// Restore last generated image
|
||||
if (data.lastResultUrl && this.#resultArea) {
|
||||
this.#lastResultUrl = data.lastResultUrl;
|
||||
this.#renderResult(data.lastResultUrl, "");
|
||||
|
|
|
|||
Loading…
Reference in New Issue