feat: add text inpainting for zine page refinement

- Add TextSelectionCanvas component for drawing rectangle selections
- Add inpaint-text API endpoint using Fal.ai FLUX Pro Fill
- Integrate "Edit Text Region" UI in the refine step
- Users can now select text areas and replace them without regenerating the full page

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Jeff Emmett 2025-12-25 19:11:20 -05:00
parent c6eb3e5363
commit f09b1cc0b9
3 changed files with 580 additions and 0 deletions

View File

@ -0,0 +1,159 @@
import { NextRequest, NextResponse } from "next/server";
import { getZine, saveZine, getPageImagePath, readFileAsBase64, savePageImage } from "@/lib/storage";
export async function POST(request: NextRequest) {
try {
const body = await request.json();
const { zineId, pageNumber, maskBase64, newText, style, tone } = body;
// Validate required fields
if (!zineId || !pageNumber || !maskBase64 || !newText) {
return NextResponse.json(
{ error: "Missing required fields: zineId, pageNumber, maskBase64, newText" },
{ status: 400 }
);
}
// Validate Fal.ai API key
const falKey = process.env.FAL_KEY;
if (!falKey) {
return NextResponse.json(
{ error: "FAL_KEY not configured" },
{ status: 500 }
);
}
// Verify zine exists
const zine = await getZine(zineId);
if (!zine) {
return NextResponse.json(
{ error: "Zine not found" },
{ status: 404 }
);
}
// Get existing page image
const existingImagePath = await getPageImagePath(zineId, pageNumber);
if (!existingImagePath) {
return NextResponse.json(
{ error: "Page image not found" },
{ status: 404 }
);
}
const existingImageBase64 = await readFileAsBase64(existingImagePath);
// Build the text inpainting prompt
const textPrompt = buildTextPrompt(newText, style, tone);
console.log(`Inpainting text on page ${pageNumber}: "${newText.slice(0, 50)}..."`);
// Call Fal.ai FLUX Pro Fill for inpainting
const newImageBase64 = await inpaintWithFluxFill(
existingImageBase64,
maskBase64,
textPrompt,
falKey
);
// Save the inpainted image
await savePageImage(zineId, pageNumber, newImageBase64);
// Update zine metadata
zine.updatedAt = new Date().toISOString();
await saveZine(zine);
// Return success with cache-busted image URL
const imageUrl = `/api/zine/${zineId}?image=p${pageNumber}&t=${Date.now()}`;
return NextResponse.json({
pageNumber,
imageUrl,
success: true,
});
} catch (error) {
console.error("Text inpainting error:", error);
return NextResponse.json(
{ error: error instanceof Error ? error.message : "Failed to inpaint text" },
{ status: 500 }
);
}
}
function buildTextPrompt(newText: string, style: string, tone: string): string {
// Build a contextual prompt for text generation
const styleDescriptions: Record<string, string> = {
"punk-zine": "punk zine aesthetic, bold hand-drawn lettering, screen-printed style",
"collage": "collage art style, cut-out letters, mixed media typography",
"minimal": "clean minimal design, modern sans-serif typography",
"vintage": "vintage retro style, distressed typography, aged paper texture",
"psychedelic": "psychedelic art style, flowing organic letterforms, vibrant colors",
};
const toneDescriptions: Record<string, string> = {
"rebellious": "bold, confrontational, high-contrast",
"playful": "fun, whimsical, energetic",
"thoughtful": "contemplative, balanced, readable",
"informative": "clear, educational, professional",
"poetic": "artistic, expressive, lyrical",
};
const styleDesc = styleDescriptions[style] || "creative zine typography";
const toneDesc = toneDescriptions[tone] || "expressive";
return `${styleDesc}. The text clearly reads: "${newText}". ${toneDesc} aesthetic. Bold, clear lettering that integrates seamlessly with the surrounding design. High contrast for readability.`;
}
async function inpaintWithFluxFill(
imageBase64: string,
maskBase64: string,
prompt: string,
falKey: string
): Promise<string> {
console.log("Calling Fal.ai FLUX Pro Fill for inpainting...");
// Use fal.run for synchronous execution
const response = await fetch("https://fal.run/fal-ai/flux-pro/v1/fill", {
method: "POST",
headers: {
"Authorization": `Key ${falKey}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
image_url: `data:image/png;base64,${imageBase64}`,
mask_url: `data:image/png;base64,${maskBase64}`,
prompt: prompt,
num_inference_steps: 40,
guidance_scale: 7.0,
output_format: "png",
safety_tolerance: 3,
}),
});
if (!response.ok) {
const errorText = await response.text();
console.error("Fal.ai error:", response.status, errorText);
throw new Error(`Fal.ai API error: ${response.status} - ${errorText}`);
}
const result = await response.json();
console.log("Fal.ai inpainting response received");
// Extract the image URL from response
if (result.images && result.images.length > 0) {
const imageUrl = result.images[0].url;
console.log("Downloading inpainted image...");
return await fetchImageAsBase64(imageUrl);
}
console.error("Fal.ai response:", JSON.stringify(result).slice(0, 500));
throw new Error("No image in Fal.ai response");
}
async function fetchImageAsBase64(url: string): Promise<string> {
const response = await fetch(url);
if (!response.ok) {
throw new Error("Failed to fetch generated image");
}
const buffer = await response.arrayBuffer();
return Buffer.from(buffer).toString("base64");
}

View File

@ -14,7 +14,9 @@ import {
RefreshCw,
Copy,
CheckCircle,
Type,
} from "lucide-react";
import TextSelectionCanvas from "@/components/zine/TextSelectionCanvas";
// Helper to get correct path based on subdomain
function useZinePath() {
@ -71,6 +73,12 @@ export default function CreatePage() {
const [isListening, setIsListening] = useState(false);
const [copied, setCopied] = useState(false);
const [regenerateMode, setRegenerateMode] = useState<"refine" | "revise" | "regenerate">("revise");
const [isTextEditMode, setIsTextEditMode] = useState(false);
const [textEditSelection, setTextEditSelection] = useState<{
bounds: { x: number; y: number; width: number; height: number };
maskBase64: string;
} | null>(null);
const [newTextInput, setNewTextInput] = useState("");
// Initialize from session storage
useEffect(() => {
@ -212,6 +220,48 @@ export default function CreatePage() {
}
};
const handleInpaintText = async () => {
if (!state || !textEditSelection || !newTextInput.trim()) return;
setState((s) => (s ? { ...s, generatingPage: currentPage } : s));
try {
const response = await fetch("/api/zine/inpaint-text", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
zineId: state.id,
pageNumber: currentPage,
maskBase64: textEditSelection.maskBase64,
newText: newTextInput.trim(),
style: state.style,
tone: state.tone,
}),
});
if (!response.ok) {
const data = await response.json();
throw new Error(data.error || "Failed to update text");
}
const data = await response.json();
setState((s) => {
if (!s) return s;
const newPages = [...s.pages];
newPages[currentPage - 1] = data.imageUrl;
return { ...s, pages: newPages, generatingPage: null };
});
// Clear text edit state
setTextEditSelection(null);
setNewTextInput("");
} catch (err) {
setError(err instanceof Error ? err.message : "Failed to update text");
setState((s) => (s ? { ...s, generatingPage: null } : s));
}
};
const createPrintLayout = async () => {
if (!state) return;
@ -642,6 +692,67 @@ export default function CreatePage() {
regenerateMode === "revise" ? "Revise Page" : "Regenerate Page"}
</button>
</div>
{/* Text Edit Section */}
<div className="punk-border bg-white p-4">
<label className="block text-sm font-bold punk-text mb-2">
Edit Text Region
</label>
{textEditSelection ? (
<div className="space-y-3">
<p className="text-xs text-gray-600 punk-text">
Selection ready. Enter the new text below:
</p>
<input
type="text"
value={newTextInput}
onChange={(e) => setNewTextInput(e.target.value)}
placeholder="Enter new text..."
className="w-full p-3 border-2 border-black punk-text text-sm"
disabled={state.generatingPage !== null}
autoFocus
/>
<div className="flex gap-2">
<button
onClick={handleInpaintText}
disabled={!newTextInput.trim() || state.generatingPage !== null}
className="flex-1 py-2 bg-green-600 text-white punk-text flex items-center justify-center gap-2
hover:bg-green-700 disabled:opacity-50 disabled:cursor-not-allowed"
>
{state.generatingPage ? (
<Loader2 className="w-4 h-4 animate-spin" />
) : (
<Type className="w-4 h-4" />
)}
Apply Text
</button>
<button
onClick={() => {
setTextEditSelection(null);
setNewTextInput("");
}}
disabled={state.generatingPage !== null}
className="px-4 py-2 punk-border bg-white text-black hover:bg-gray-100 disabled:opacity-50"
>
Cancel
</button>
</div>
</div>
) : (
<button
onClick={() => setIsTextEditMode(true)}
disabled={state.generatingPage !== null}
className="w-full py-2 bg-white text-black punk-text border-2 border-black
flex items-center justify-center gap-2 hover:bg-gray-100 disabled:opacity-50"
>
<Type className="w-4 h-4" />
Select Text to Edit
</button>
)}
<p className="text-xs text-gray-500 mt-2 punk-text">
Draw a box around any text to change it
</p>
</div>
</div>
</div>
@ -669,6 +780,18 @@ export default function CreatePage() {
<ArrowRight className="w-4 h-4" />
</button>
</div>
{/* Text Selection Overlay */}
{isTextEditMode && (
<TextSelectionCanvas
imageUrl={state.pages[currentPage - 1]}
onSelectionComplete={(data) => {
setTextEditSelection(data);
setIsTextEditMode(false);
}}
onCancel={() => setIsTextEditMode(false)}
/>
)}
</div>
)}

View File

@ -0,0 +1,298 @@
"use client";
import { useRef, useState, useEffect, useCallback } from "react";
import { X } from "lucide-react";
interface SelectionBounds {
x: number;
y: number;
width: number;
height: number;
}
interface SelectionData {
bounds: SelectionBounds;
maskBase64: string;
}
interface TextSelectionCanvasProps {
imageUrl: string;
onSelectionComplete: (data: SelectionData) => void;
onCancel: () => void;
}
// Target dimensions for the mask (matching zine page size)
const MASK_WIDTH = 768;
const MASK_HEIGHT = 1024;
export default function TextSelectionCanvas({
imageUrl,
onSelectionComplete,
onCancel,
}: TextSelectionCanvasProps) {
const canvasRef = useRef<HTMLCanvasElement>(null);
const containerRef = useRef<HTMLDivElement>(null);
const [isDrawing, setIsDrawing] = useState(false);
const [startPoint, setStartPoint] = useState<{ x: number; y: number } | null>(null);
const [currentRect, setCurrentRect] = useState<SelectionBounds | null>(null);
const [imageLoaded, setImageLoaded] = useState(false);
const [canvasSize, setCanvasSize] = useState({ width: 0, height: 0 });
// Load and draw the background image
useEffect(() => {
const canvas = canvasRef.current;
const container = containerRef.current;
if (!canvas || !container) return;
const img = new Image();
img.crossOrigin = "anonymous";
img.onload = () => {
// Calculate canvas size to fit container while maintaining aspect ratio
const containerWidth = container.clientWidth;
const containerHeight = container.clientHeight;
const imgAspect = img.width / img.height;
const containerAspect = containerWidth / containerHeight;
let canvasWidth: number;
let canvasHeight: number;
if (imgAspect > containerAspect) {
canvasWidth = containerWidth;
canvasHeight = containerWidth / imgAspect;
} else {
canvasHeight = containerHeight;
canvasWidth = containerHeight * imgAspect;
}
canvas.width = canvasWidth;
canvas.height = canvasHeight;
setCanvasSize({ width: canvasWidth, height: canvasHeight });
const ctx = canvas.getContext("2d");
if (ctx) {
ctx.drawImage(img, 0, 0, canvasWidth, canvasHeight);
setImageLoaded(true);
}
};
img.src = imageUrl;
}, [imageUrl]);
// Redraw canvas with selection rectangle
const redrawCanvas = useCallback((rect: SelectionBounds | null) => {
const canvas = canvasRef.current;
if (!canvas) return;
const ctx = canvas.getContext("2d");
if (!ctx) return;
// Reload and draw the image
const img = new Image();
img.crossOrigin = "anonymous";
img.onload = () => {
ctx.clearRect(0, 0, canvas.width, canvas.height);
ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
// Draw semi-transparent overlay
ctx.fillStyle = "rgba(0, 0, 0, 0.4)";
ctx.fillRect(0, 0, canvas.width, canvas.height);
if (rect && rect.width > 0 && rect.height > 0) {
// Clear the selected area (show original image)
ctx.clearRect(rect.x, rect.y, rect.width, rect.height);
ctx.drawImage(
img,
(rect.x / canvas.width) * img.width,
(rect.y / canvas.height) * img.height,
(rect.width / canvas.width) * img.width,
(rect.height / canvas.height) * img.height,
rect.x,
rect.y,
rect.width,
rect.height
);
// Draw selection border
ctx.strokeStyle = "#22c55e";
ctx.lineWidth = 2;
ctx.setLineDash([5, 5]);
ctx.strokeRect(rect.x, rect.y, rect.width, rect.height);
// Draw corner handles
ctx.setLineDash([]);
ctx.fillStyle = "#22c55e";
const handleSize = 8;
const corners = [
{ x: rect.x, y: rect.y },
{ x: rect.x + rect.width, y: rect.y },
{ x: rect.x, y: rect.y + rect.height },
{ x: rect.x + rect.width, y: rect.y + rect.height },
];
corners.forEach(({ x, y }) => {
ctx.fillRect(x - handleSize / 2, y - handleSize / 2, handleSize, handleSize);
});
}
};
img.src = imageUrl;
}, [imageUrl]);
// Get canvas coordinates from mouse/touch event
const getCanvasCoords = (e: React.MouseEvent | React.TouchEvent) => {
const canvas = canvasRef.current;
if (!canvas) return { x: 0, y: 0 };
const rect = canvas.getBoundingClientRect();
const clientX = "touches" in e ? e.touches[0].clientX : e.clientX;
const clientY = "touches" in e ? e.touches[0].clientY : e.clientY;
return {
x: clientX - rect.left,
y: clientY - rect.top,
};
};
const handleStart = (e: React.MouseEvent | React.TouchEvent) => {
e.preventDefault();
const coords = getCanvasCoords(e);
setIsDrawing(true);
setStartPoint(coords);
setCurrentRect(null);
};
const handleMove = (e: React.MouseEvent | React.TouchEvent) => {
if (!isDrawing || !startPoint) return;
e.preventDefault();
const coords = getCanvasCoords(e);
const rect: SelectionBounds = {
x: Math.min(startPoint.x, coords.x),
y: Math.min(startPoint.y, coords.y),
width: Math.abs(coords.x - startPoint.x),
height: Math.abs(coords.y - startPoint.y),
};
setCurrentRect(rect);
redrawCanvas(rect);
};
const handleEnd = () => {
setIsDrawing(false);
setStartPoint(null);
};
// Generate mask and complete selection
const handleConfirm = () => {
if (!currentRect || currentRect.width < 20 || currentRect.height < 20) {
return;
}
// Create mask canvas at target dimensions
const maskCanvas = document.createElement("canvas");
maskCanvas.width = MASK_WIDTH;
maskCanvas.height = MASK_HEIGHT;
const maskCtx = maskCanvas.getContext("2d");
if (!maskCtx) return;
// Fill with black (preserve area)
maskCtx.fillStyle = "black";
maskCtx.fillRect(0, 0, MASK_WIDTH, MASK_HEIGHT);
// Scale selection coordinates to mask dimensions
const scaleX = MASK_WIDTH / canvasSize.width;
const scaleY = MASK_HEIGHT / canvasSize.height;
const scaledRect = {
x: currentRect.x * scaleX,
y: currentRect.y * scaleY,
width: currentRect.width * scaleX,
height: currentRect.height * scaleY,
};
// Draw white rectangle (inpaint area)
maskCtx.fillStyle = "white";
maskCtx.fillRect(scaledRect.x, scaledRect.y, scaledRect.width, scaledRect.height);
// Convert to base64 (without data: prefix)
const maskDataUrl = maskCanvas.toDataURL("image/png");
const maskBase64 = maskDataUrl.replace(/^data:image\/png;base64,/, "");
onSelectionComplete({
bounds: scaledRect,
maskBase64,
});
};
const hasValidSelection = currentRect && currentRect.width >= 20 && currentRect.height >= 20;
return (
<div className="fixed inset-0 z-50 bg-black/80 flex items-center justify-center p-4">
<div className="relative w-full max-w-2xl">
{/* Header */}
<div className="flex items-center justify-between mb-4">
<h3 className="text-white punk-text text-lg">Draw a rectangle around the text to edit</h3>
<button
onClick={onCancel}
className="p-2 bg-white/10 hover:bg-white/20 rounded-full text-white"
>
<X className="w-5 h-5" />
</button>
</div>
{/* Canvas container */}
<div
ref={containerRef}
className="relative w-full bg-gray-900 punk-border"
style={{ aspectRatio: "3/4" }}
>
<canvas
ref={canvasRef}
className={`absolute top-1/2 left-1/2 -translate-x-1/2 -translate-y-1/2 ${
imageLoaded ? "cursor-crosshair" : "cursor-wait"
}`}
onMouseDown={handleStart}
onMouseMove={handleMove}
onMouseUp={handleEnd}
onMouseLeave={handleEnd}
onTouchStart={handleStart}
onTouchMove={handleMove}
onTouchEnd={handleEnd}
/>
{!imageLoaded && (
<div className="absolute inset-0 flex items-center justify-center">
<div className="text-white punk-text">Loading image...</div>
</div>
)}
</div>
{/* Action buttons */}
<div className="flex gap-3 mt-4">
<button
onClick={onCancel}
className="flex-1 py-3 px-4 bg-white/10 text-white punk-text border-2 border-white/30 hover:bg-white/20"
>
Cancel
</button>
<button
onClick={handleConfirm}
disabled={!hasValidSelection}
className={`flex-1 py-3 px-4 punk-text border-2 ${
hasValidSelection
? "bg-green-600 text-white border-green-600 hover:bg-green-700"
: "bg-gray-600 text-gray-400 border-gray-600 cursor-not-allowed"
}`}
>
{hasValidSelection ? "Confirm Selection" : "Draw a selection"}
</button>
</div>
{/* Help text */}
<p className="text-white/60 text-sm text-center mt-3 punk-text">
Click and drag to select the text region you want to change
</p>
</div>
</div>
);
}