From edabad18e428e3b648df81e2ea147b03e51c7253 Mon Sep 17 00:00:00 2001 From: Jeff Emmett Date: Fri, 10 Apr 2026 17:03:38 -0400 Subject: [PATCH] Add gesture recognition and collaborative sync to folk-drawfast Implements $1 Unistroke Recognizer for detecting circles, rectangles, triangles, lines, arrows, and checkmarks from freehand strokes. Detected gestures are converted to clean geometric shapes with a confidence badge. Fixes applyData() to restore strokes, prompt text, and generated images from Automerge sync data, enabling collaborative drawing across clients. Co-Authored-By: Claude Opus 4.6 --- lib/folk-drawfast.ts | 367 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 363 insertions(+), 4 deletions(-) diff --git a/lib/folk-drawfast.ts b/lib/folk-drawfast.ts index 6170879b..6d7f0c49 100644 --- a/lib/folk-drawfast.ts +++ b/lib/folk-drawfast.ts @@ -309,6 +309,28 @@ const styles = css` .export-btn:hover { opacity: 0.9; } + + .gesture-badge { + position: absolute; + top: 8px; + left: 50%; + transform: translateX(-50%); + background: rgba(249, 115, 22, 0.9); + color: white; + padding: 4px 12px; + border-radius: 12px; + font-size: 11px; + font-weight: 600; + z-index: 3; + pointer-events: none; + animation: badge-fade 1.5s ease-out forwards; + } + + @keyframes badge-fade { + 0% { opacity: 1; transform: translateX(-50%) translateY(0); } + 70% { opacity: 1; } + 100% { opacity: 0; transform: translateX(-50%) translateY(-10px); } + } `; const COLORS = ["#0f172a", "#ef4444", "#f97316", "#eab308", "#22c55e", "#3b82f6", "#8b5cf6", "#ec4899", "#ffffff"]; @@ -320,6 +342,213 @@ interface Stroke { tool: string; } +// --- $1 Unistroke Recognizer (lightweight, self-contained) --- +// Based on Wobbrock et al. 2007, adapted from canvas-website gesture templates + +interface Point2D { x: number; y: number; } +interface RecognizeResult { name: string; score: number; } + +const NUM_POINTS = 64; +const SQUARE_SIZE = 250; +const HALF_DIAGONAL = 0.5 * Math.sqrt(SQUARE_SIZE * SQUARE_SIZE + SQUARE_SIZE * SQUARE_SIZE); +const ANGLE_RANGE = Math.PI * 2; +const ANGLE_PRECISION = Math.PI / 90; // 2 degrees +const PHI = 0.5 * (-1 + Math.sqrt(5)); // golden ratio + +function resample(pts: Point2D[], n: number): Point2D[] { + const totalLen = pathLength(pts); + const interval = totalLen / (n - 1); + const newPts: Point2D[] = [pts[0]]; + let D = 0; + for (let i = 1; i < pts.length; i++) { + const d = distance(pts[i - 1], pts[i]); + if (D + d >= interval) { + const t = (interval - D) / d; + const qx = pts[i - 1].x + t * (pts[i].x - pts[i - 1].x); + const qy = pts[i - 1].y + t * (pts[i].y - pts[i - 1].y); + const q: Point2D = { x: qx, y: qy }; + newPts.push(q); + pts.splice(i, 0, q); + D = 0; + } else { + D += d; + } + } + while (newPts.length < n) newPts.push(pts[pts.length - 1]); + return newPts; +} + +function rotateToZero(pts: Point2D[]): Point2D[] { + const c = centroid(pts); + const angle = Math.atan2(c.y - pts[0].y, c.x - pts[0].x); + return rotateBy(pts, -angle); +} + +function scaleToSquare(pts: Point2D[]): Point2D[] { + let minX = Infinity, maxX = -Infinity, minY = Infinity, maxY = -Infinity; + for (const p of pts) { + minX = Math.min(minX, p.x); maxX = Math.max(maxX, p.x); + minY = Math.min(minY, p.y); maxY = Math.max(maxY, p.y); + } + const w = maxX - minX || 1; + const h = maxY - minY || 1; + return pts.map(p => ({ x: p.x * (SQUARE_SIZE / w), y: p.y * (SQUARE_SIZE / h) })); +} + +function translateToOrigin(pts: Point2D[]): Point2D[] { + const c = centroid(pts); + return pts.map(p => ({ x: p.x - c.x, y: p.y - c.y })); +} + +function recognize(pts: Point2D[], templates: { name: string; points: Point2D[] }[]): RecognizeResult { + let best = Infinity; + let bestName = "none"; + for (const t of templates) { + const d = distanceAtBestAngle(pts, t.points, -ANGLE_RANGE, ANGLE_RANGE, ANGLE_PRECISION); + if (d < best) { + best = d; + bestName = t.name; + } + } + const score = 1 - best / HALF_DIAGONAL; + return { name: bestName, score }; +} + +function distanceAtBestAngle(pts: Point2D[], template: Point2D[], a: number, b: number, threshold: number): number { + let x1 = PHI * a + (1 - PHI) * b; + let f1 = distanceAtAngle(pts, template, x1); + let x2 = (1 - PHI) * a + PHI * b; + let f2 = distanceAtAngle(pts, template, x2); + while (Math.abs(b - a) > threshold) { + if (f1 < f2) { + b = x2; x2 = x1; f2 = f1; + x1 = PHI * a + (1 - PHI) * b; + f1 = distanceAtAngle(pts, template, x1); + } else { + a = x1; x1 = x2; f1 = f2; + x2 = (1 - PHI) * a + PHI * b; + f2 = distanceAtAngle(pts, template, x2); + } + } + return Math.min(f1, f2); +} + +function distanceAtAngle(pts: Point2D[], template: Point2D[], angle: number): number { + const rotated = rotateBy(pts, angle); + return pathDistance(rotated, template); +} + +function centroid(pts: Point2D[]): Point2D { + let x = 0, y = 0; + for (const p of pts) { x += p.x; y += p.y; } + return { x: x / pts.length, y: y / pts.length }; +} + +function rotateBy(pts: Point2D[], angle: number): Point2D[] { + const c = centroid(pts); + const cos = Math.cos(angle), sin = Math.sin(angle); + return pts.map(p => ({ + x: (p.x - c.x) * cos - (p.y - c.y) * sin + c.x, + y: (p.x - c.x) * sin + (p.y - c.y) * cos + c.y, + })); +} + +function pathDistance(a: Point2D[], b: Point2D[]): number { + let d = 0; + const n = Math.min(a.length, b.length); + for (let i = 0; i < n; i++) d += distance(a[i], b[i]); + return d / n; +} + +function pathLength(pts: Point2D[]): number { + let d = 0; + for (let i = 1; i < pts.length; i++) d += distance(pts[i - 1], pts[i]); + return d; +} + +function distance(a: Point2D, b: Point2D): number { + const dx = b.x - a.x, dy = b.y - a.y; + return Math.sqrt(dx * dx + dy * dy); +} + +function processTemplate(pts: Point2D[]): Point2D[] { + return translateToOrigin(scaleToSquare(rotateToZero(resample(pts, NUM_POINTS)))); +} + +// Generate templates procedurally (more compact than storing point arrays) +function makeCircleTemplate(): Point2D[] { + const pts: Point2D[] = []; + for (let i = 0; i <= 32; i++) { + const a = (i / 32) * Math.PI * 2; + pts.push({ x: 100 + 80 * Math.cos(a), y: 100 + 80 * Math.sin(a) }); + } + return processTemplate(pts); +} + +function makeRectangleTemplate(): Point2D[] { + const pts: Point2D[] = []; + // Draw rectangle starting top-left, clockwise + const steps = 8; + for (let i = 0; i <= steps; i++) pts.push({ x: 20 + (160 * i / steps), y: 20 }); // top + for (let i = 0; i <= steps; i++) pts.push({ x: 180, y: 20 + (160 * i / steps) }); // right + for (let i = 0; i <= steps; i++) pts.push({ x: 180 - (160 * i / steps), y: 180 }); // bottom + for (let i = 0; i <= steps; i++) pts.push({ x: 20, y: 180 - (160 * i / steps) }); // left + return processTemplate(pts); +} + +function makeLineTemplate(): Point2D[] { + const pts: Point2D[] = []; + for (let i = 0; i <= 16; i++) pts.push({ x: 20 + (160 * i / 16), y: 100 }); + return processTemplate(pts); +} + +function makeArrowTemplate(): Point2D[] { + // Horizontal line with arrowhead at the end + const pts: Point2D[] = []; + for (let i = 0; i <= 12; i++) pts.push({ x: 20 + (140 * i / 12), y: 100 }); // shaft + for (let i = 0; i <= 4; i++) pts.push({ x: 160 - (40 * i / 4), y: 100 - (40 * i / 4) }); // upper head + pts.push({ x: 160, y: 100 }); // back to tip + for (let i = 0; i <= 4; i++) pts.push({ x: 160 - (40 * i / 4), y: 100 + (40 * i / 4) }); // lower head + return processTemplate(pts); +} + +function makeTriangleTemplate(): Point2D[] { + const pts: Point2D[] = []; + const steps = 8; + // Top to bottom-right + for (let i = 0; i <= steps; i++) pts.push({ x: 100 + (80 * i / steps), y: 20 + (160 * i / steps) }); + // Bottom-right to bottom-left + for (let i = 0; i <= steps; i++) pts.push({ x: 180 - (160 * i / steps), y: 180 }); + // Bottom-left to top + for (let i = 0; i <= steps; i++) pts.push({ x: 20 + (80 * i / steps), y: 180 - (160 * i / steps) }); + return processTemplate(pts); +} + +function makeCheckTemplate(): Point2D[] { + const pts: Point2D[] = []; + for (let i = 0; i <= 6; i++) pts.push({ x: 20 + (40 * i / 6), y: 100 + (60 * i / 6) }); + for (let i = 0; i <= 8; i++) pts.push({ x: 60 + (120 * i / 8), y: 160 - (140 * i / 8) }); + return processTemplate(pts); +} + +const GESTURE_TEMPLATES = [ + { name: "circle", points: makeCircleTemplate() }, + { name: "rectangle", points: makeRectangleTemplate() }, + { name: "line", points: makeLineTemplate() }, + { name: "arrow", points: makeArrowTemplate() }, + { name: "triangle", points: makeTriangleTemplate() }, + { name: "check", points: makeCheckTemplate() }, +]; + +function recognizeGesture(rawPoints: Point2D[]): RecognizeResult | null { + if (rawPoints.length < 8) return null; // too few points + const processed = processTemplate(rawPoints); + const result = recognize(processed, GESTURE_TEMPLATES); + if (result.score < 0.7) return null; // low confidence + return result; +} +// --- End $1 Unistroke Recognizer --- + declare global { interface HTMLElementTagNameMap { "folk-drawfast": FolkDrawfast; @@ -364,6 +593,8 @@ export class FolkDrawfast extends FolkShape { #promptInput: HTMLInputElement | null = null; #generateBtn: HTMLButtonElement | null = null; #resultArea: HTMLElement | null = null; + #canvasArea: HTMLElement | null = null; + #gestureEnabled = true; get strokes() { return this.#strokes; @@ -443,6 +674,7 @@ export class FolkDrawfast extends FolkShape { const sizeSlider = wrapper.querySelector(".size-slider") as HTMLInputElement; const sizeLabel = wrapper.querySelector(".size-label") as HTMLElement; const canvasArea = wrapper.querySelector(".canvas-area") as HTMLElement; + this.#canvasArea = canvasArea; const autoCheckbox = wrapper.querySelector(".auto-checkbox") as HTMLInputElement; const providerSelect = wrapper.querySelector(".provider-select") as HTMLSelectElement; const strengthSlider = wrapper.querySelector(".strength-slider") as HTMLInputElement; @@ -533,10 +765,29 @@ export class FolkDrawfast extends FolkShape { if (!this.#isDrawing) return; this.#isDrawing = false; if (this.#currentStroke && this.#currentStroke.points.length > 0) { - this.#strokes.push(this.#currentStroke); - this.dispatchEvent(new CustomEvent("stroke-complete", { - detail: { stroke: this.#currentStroke }, - })); + // Try gesture recognition before adding stroke + let gestureResult: RecognizeResult | null = null; + if (this.#gestureEnabled && this.#currentStroke.tool === "pen") { + const rawPts = this.#currentStroke.points.map(p => ({ x: p.x, y: p.y })); + gestureResult = recognizeGesture(rawPts); + } + + if (gestureResult) { + // Replace freehand stroke with clean geometric shape + const cleanStroke = this.#makeCleanShape(gestureResult.name, this.#currentStroke); + this.#strokes.push(cleanStroke); + this.#redraw(); + this.#showGestureBadge(gestureResult.name, gestureResult.score); + this.dispatchEvent(new CustomEvent("stroke-complete", { + detail: { stroke: cleanStroke, gesture: gestureResult.name }, + })); + } else { + this.#strokes.push(this.#currentStroke); + this.dispatchEvent(new CustomEvent("stroke-complete", { + detail: { stroke: this.#currentStroke }, + })); + } + // Auto-generate on stroke complete if enabled if (this.#autoGenerate && this.#promptInput?.value.trim()) { this.#scheduleAutoGenerate(); @@ -774,6 +1025,95 @@ export class FolkDrawfast extends FolkShape { ctx.globalCompositeOperation = "source-over"; } + #makeCleanShape(gesture: string, original: Stroke): Stroke { + const pts = original.points; + const minX = Math.min(...pts.map(p => p.x)); + const maxX = Math.max(...pts.map(p => p.x)); + const minY = Math.min(...pts.map(p => p.y)); + const maxY = Math.max(...pts.map(p => p.y)); + const cx = (minX + maxX) / 2; + const cy = (minY + maxY) / 2; + const w = maxX - minX || 1; + const h = maxY - minY || 1; + const pressure = 0.5; + + const toStroke = (points: { x: number; y: number }[]): Stroke => ({ + points: points.map(p => ({ ...p, pressure })), + color: original.color, + size: original.size, + tool: "pen", + }); + + switch (gesture) { + case "circle": { + const rx = w / 2, ry = h / 2; + const circPts: { x: number; y: number }[] = []; + for (let i = 0; i <= 48; i++) { + const a = (i / 48) * Math.PI * 2; + circPts.push({ x: cx + rx * Math.cos(a), y: cy + ry * Math.sin(a) }); + } + return toStroke(circPts); + } + case "rectangle": { + return toStroke([ + { x: minX, y: minY }, { x: maxX, y: minY }, + { x: maxX, y: maxY }, { x: minX, y: maxY }, + { x: minX, y: minY }, + ]); + } + case "triangle": { + return toStroke([ + { x: cx, y: minY }, { x: maxX, y: maxY }, + { x: minX, y: maxY }, { x: cx, y: minY }, + ]); + } + case "line": { + // Use first and last point for direction + const first = pts[0], last = pts[pts.length - 1]; + return toStroke([ + { x: first.x, y: first.y }, + { x: last.x, y: last.y }, + ]); + } + case "arrow": { + const first = pts[0], last = pts[pts.length - 1]; + const dx = last.x - first.x, dy = last.y - first.y; + const len = Math.sqrt(dx * dx + dy * dy) || 1; + const ux = dx / len, uy = dy / len; + const headLen = Math.min(20, len * 0.3); + // Arrow shaft + two head lines + return toStroke([ + { x: first.x, y: first.y }, { x: last.x, y: last.y }, + { x: last.x - headLen * (ux + uy * 0.5), y: last.y - headLen * (uy - ux * 0.5) }, + { x: last.x, y: last.y }, + { x: last.x - headLen * (ux - uy * 0.5), y: last.y - headLen * (uy + ux * 0.5) }, + ]); + } + case "check": { + // V-shape: find the lowest point as the vertex + const lowestIdx = pts.reduce((best, p, i) => p.y > pts[best].y ? i : best, 0); + return toStroke([ + { x: pts[0].x, y: pts[0].y }, + { x: pts[lowestIdx].x, y: pts[lowestIdx].y }, + { x: pts[pts.length - 1].x, y: pts[pts.length - 1].y }, + ]); + } + default: + return original; + } + } + + #showGestureBadge(name: string, score: number) { + if (!this.#canvasArea) return; + // Remove existing badge + this.#canvasArea.querySelector(".gesture-badge")?.remove(); + const badge = document.createElement("div"); + badge.className = "gesture-badge"; + badge.textContent = `${name} (${Math.round(score * 100)}%)`; + this.#canvasArea.appendChild(badge); + setTimeout(() => badge.remove(), 1500); + } + #exportPNG() { if (!this.#canvas) return; const dataUrl = this.#canvas.toDataURL("image/png"); @@ -808,12 +1148,31 @@ export class FolkDrawfast extends FolkShape { size: s.size, tool: s.tool, })), + prompt: this.#promptInput?.value || "", lastResultUrl: this.#lastResultUrl, }; } override applyData(data: Record): void { super.applyData(data); + + // Restore strokes from sync data + if (data.strokes && Array.isArray(data.strokes)) { + this.#strokes = data.strokes.map((s: any) => ({ + points: Array.isArray(s.points) ? s.points : [], + color: s.color || "#0f172a", + size: s.size || 4, + tool: s.tool || "pen", + })); + this.#redraw(); + } + + // Restore prompt text + if (data.prompt !== undefined && this.#promptInput) { + this.#promptInput.value = data.prompt || ""; + } + + // Restore last generated image if (data.lastResultUrl && this.#resultArea) { this.#lastResultUrl = data.lastResultUrl; this.#renderResult(data.lastResultUrl, "");