feat: add FLUX img2img for zine page refinement
- Add Refine/Revise/Regenerate mode selection in zine creator - Refine (0.25 strength): keeps most of image, minor tweaks - Revise (0.5 strength): keeps composition, changes elements - Regenerate: creates completely new image (existing behavior) - Use Fal.ai FLUX dev model for img2img processing - Better UX for iterative refinement without losing good work 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
parent
19764d7df4
commit
f78cf1d0cd
|
|
@ -1,12 +1,19 @@
|
||||||
import { NextRequest, NextResponse } from "next/server";
|
import { NextRequest, NextResponse } from "next/server";
|
||||||
import { GoogleGenerativeAI } from "@google/generative-ai";
|
import { GoogleGenerativeAI } from "@google/generative-ai";
|
||||||
import { getZine, saveZine } from "@/lib/storage";
|
import { getZine, saveZine, getPageImagePath, readFileAsBase64, savePageImage } from "@/lib/storage";
|
||||||
import type { PageOutline } from "@/lib/gemini";
|
import type { PageOutline } from "@/lib/gemini";
|
||||||
|
|
||||||
|
// Regeneration modes with denoising strengths
|
||||||
|
const MODE_STRENGTHS: Record<string, number> = {
|
||||||
|
refine: 0.25, // Keep most of image, minor tweaks
|
||||||
|
revise: 0.5, // Keep composition, change elements
|
||||||
|
regenerate: 1.0 // Completely new image
|
||||||
|
};
|
||||||
|
|
||||||
export async function POST(request: NextRequest) {
|
export async function POST(request: NextRequest) {
|
||||||
try {
|
try {
|
||||||
const body = await request.json();
|
const body = await request.json();
|
||||||
const { zineId, pageNumber, currentOutline, feedback, style, tone } = body;
|
const { zineId, pageNumber, currentOutline, feedback, style, tone, mode = "regenerate" } = body;
|
||||||
|
|
||||||
if (!zineId || !pageNumber || !currentOutline || !feedback) {
|
if (!zineId || !pageNumber || !currentOutline || !feedback) {
|
||||||
return NextResponse.json(
|
return NextResponse.json(
|
||||||
|
|
@ -76,8 +83,33 @@ Return ONLY valid JSON (no markdown, no code blocks):
|
||||||
|
|
||||||
const updatedOutline = JSON.parse(jsonStr.trim()) as PageOutline;
|
const updatedOutline = JSON.parse(jsonStr.trim()) as PageOutline;
|
||||||
|
|
||||||
// Generate new image with updated outline
|
let imageUrl: string;
|
||||||
// Forward to generate-page endpoint logic
|
|
||||||
|
// Use img2img for refine/revise modes, full generation for regenerate
|
||||||
|
if (mode === "refine" || mode === "revise") {
|
||||||
|
// Get existing image for img2img
|
||||||
|
const existingImagePath = await getPageImagePath(zineId, pageNumber);
|
||||||
|
if (!existingImagePath) {
|
||||||
|
throw new Error("Existing page image not found");
|
||||||
|
}
|
||||||
|
|
||||||
|
const existingImageBase64 = await readFileAsBase64(existingImagePath);
|
||||||
|
const strength = MODE_STRENGTHS[mode];
|
||||||
|
|
||||||
|
console.log(`Using FLUX img2img with strength ${strength} for ${mode} mode`);
|
||||||
|
|
||||||
|
// Generate with FLUX img2img via Fal.ai
|
||||||
|
const newImageBase64 = await generateWithFluxImg2Img(
|
||||||
|
existingImageBase64,
|
||||||
|
updatedOutline.imagePrompt,
|
||||||
|
strength
|
||||||
|
);
|
||||||
|
|
||||||
|
// Save the new image
|
||||||
|
await savePageImage(zineId, pageNumber, newImageBase64);
|
||||||
|
imageUrl = `/api/zine/${zineId}?image=p${pageNumber}&t=${Date.now()}`;
|
||||||
|
} else {
|
||||||
|
// Full regeneration - use existing generate-page endpoint
|
||||||
const generateResponse = await fetch(
|
const generateResponse = await fetch(
|
||||||
new URL("/api/zine/generate-page", request.url),
|
new URL("/api/zine/generate-page", request.url),
|
||||||
{
|
{
|
||||||
|
|
@ -100,6 +132,8 @@ Return ONLY valid JSON (no markdown, no code blocks):
|
||||||
}
|
}
|
||||||
|
|
||||||
const generateResult = await generateResponse.json();
|
const generateResult = await generateResponse.json();
|
||||||
|
imageUrl = generateResult.imageUrl;
|
||||||
|
}
|
||||||
|
|
||||||
// Update the zine outline
|
// Update the zine outline
|
||||||
zine.outline[pageNumber - 1] = updatedOutline;
|
zine.outline[pageNumber - 1] = updatedOutline;
|
||||||
|
|
@ -109,7 +143,8 @@ Return ONLY valid JSON (no markdown, no code blocks):
|
||||||
return NextResponse.json({
|
return NextResponse.json({
|
||||||
pageNumber,
|
pageNumber,
|
||||||
updatedOutline,
|
updatedOutline,
|
||||||
imageUrl: generateResult.imageUrl,
|
imageUrl,
|
||||||
|
mode,
|
||||||
success: true,
|
success: true,
|
||||||
});
|
});
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
|
|
@ -120,3 +155,107 @@ Return ONLY valid JSON (no markdown, no code blocks):
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FLUX img2img via Fal.ai
|
||||||
|
async function generateWithFluxImg2Img(
|
||||||
|
imageBase64: string,
|
||||||
|
prompt: string,
|
||||||
|
strength: number
|
||||||
|
): Promise<string> {
|
||||||
|
const falKey = process.env.FAL_KEY;
|
||||||
|
if (!falKey) {
|
||||||
|
throw new Error("FAL_KEY not configured");
|
||||||
|
}
|
||||||
|
|
||||||
|
console.log("Calling Fal.ai FLUX img2img API...");
|
||||||
|
|
||||||
|
// Submit the request
|
||||||
|
const submitResponse = await fetch("https://queue.fal.run/fal-ai/flux/dev/image-to-image", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
"Authorization": `Key ${falKey}`,
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
image_url: `data:image/png;base64,${imageBase64}`,
|
||||||
|
prompt: prompt,
|
||||||
|
strength: strength,
|
||||||
|
num_inference_steps: 28,
|
||||||
|
guidance_scale: 3.5,
|
||||||
|
image_size: {
|
||||||
|
width: 768,
|
||||||
|
height: 1024 // Portrait for zine pages
|
||||||
|
},
|
||||||
|
output_format: "png"
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!submitResponse.ok) {
|
||||||
|
const errorText = await submitResponse.text();
|
||||||
|
console.error("Fal.ai submit error:", submitResponse.status, errorText);
|
||||||
|
throw new Error(`Fal.ai API error: ${submitResponse.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await submitResponse.json();
|
||||||
|
|
||||||
|
// Check if we got a direct result or need to poll
|
||||||
|
if (result.images && result.images.length > 0) {
|
||||||
|
// Direct result
|
||||||
|
const imageUrl = result.images[0].url;
|
||||||
|
return await fetchImageAsBase64(imageUrl);
|
||||||
|
} else if (result.request_id) {
|
||||||
|
// Need to poll for result
|
||||||
|
return await pollForResult(result.request_id, falKey);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error("No image in Fal.ai response");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function pollForResult(requestId: string, falKey: string): Promise<string> {
|
||||||
|
const maxAttempts = 60;
|
||||||
|
const pollInterval = 2000;
|
||||||
|
|
||||||
|
for (let i = 0; i < maxAttempts; i++) {
|
||||||
|
await new Promise(resolve => setTimeout(resolve, pollInterval));
|
||||||
|
|
||||||
|
const statusResponse = await fetch(`https://queue.fal.run/fal-ai/flux/dev/image-to-image/requests/${requestId}/status`, {
|
||||||
|
headers: {
|
||||||
|
"Authorization": `Key ${falKey}`,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!statusResponse.ok) continue;
|
||||||
|
|
||||||
|
const status = await statusResponse.json();
|
||||||
|
|
||||||
|
if (status.status === "COMPLETED") {
|
||||||
|
// Get the result
|
||||||
|
const resultResponse = await fetch(`https://queue.fal.run/fal-ai/flux/dev/image-to-image/requests/${requestId}`, {
|
||||||
|
headers: {
|
||||||
|
"Authorization": `Key ${falKey}`,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
|
if (resultResponse.ok) {
|
||||||
|
const result = await resultResponse.json();
|
||||||
|
if (result.images && result.images.length > 0) {
|
||||||
|
const imageUrl = result.images[0].url;
|
||||||
|
return await fetchImageAsBase64(imageUrl);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (status.status === "FAILED") {
|
||||||
|
throw new Error("Fal.ai generation failed");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new Error("Fal.ai generation timed out");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fetchImageAsBase64(url: string): Promise<string> {
|
||||||
|
const response = await fetch(url);
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error("Failed to fetch generated image");
|
||||||
|
}
|
||||||
|
const buffer = await response.arrayBuffer();
|
||||||
|
return Buffer.from(buffer).toString("base64");
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -70,6 +70,7 @@ export default function CreatePage() {
|
||||||
const [feedback, setFeedback] = useState("");
|
const [feedback, setFeedback] = useState("");
|
||||||
const [isListening, setIsListening] = useState(false);
|
const [isListening, setIsListening] = useState(false);
|
||||||
const [copied, setCopied] = useState(false);
|
const [copied, setCopied] = useState(false);
|
||||||
|
const [regenerateMode, setRegenerateMode] = useState<"refine" | "revise" | "regenerate">("revise");
|
||||||
|
|
||||||
// Initialize from session storage
|
// Initialize from session storage
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
|
@ -184,6 +185,7 @@ export default function CreatePage() {
|
||||||
feedback: feedback.trim(),
|
feedback: feedback.trim(),
|
||||||
style: state.style,
|
style: state.style,
|
||||||
tone: state.tone,
|
tone: state.tone,
|
||||||
|
mode: regenerateMode,
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
@ -580,14 +582,64 @@ export default function CreatePage() {
|
||||||
{isListening ? <MicOff className="w-4 h-4" /> : <Mic className="w-4 h-4" />}
|
{isListening ? <MicOff className="w-4 h-4" /> : <Mic className="w-4 h-4" />}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
{/* Mode Selection */}
|
||||||
|
<div className="mt-3 mb-3">
|
||||||
|
<label className="block text-xs font-bold punk-text mb-2 text-gray-600">
|
||||||
|
Regeneration Mode
|
||||||
|
</label>
|
||||||
|
<div className="grid grid-cols-3 gap-2">
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => setRegenerateMode("refine")}
|
||||||
|
disabled={state.generatingPage !== null}
|
||||||
|
className={`py-2 px-3 text-xs punk-text border-2 transition-colors
|
||||||
|
${regenerateMode === "refine"
|
||||||
|
? "bg-green-600 text-white border-green-600"
|
||||||
|
: "bg-white text-black border-black hover:bg-gray-100"
|
||||||
|
} disabled:opacity-50`}
|
||||||
|
>
|
||||||
|
Refine
|
||||||
|
<span className="block text-[10px] opacity-75">Keep image</span>
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => setRegenerateMode("revise")}
|
||||||
|
disabled={state.generatingPage !== null}
|
||||||
|
className={`py-2 px-3 text-xs punk-text border-2 transition-colors
|
||||||
|
${regenerateMode === "revise"
|
||||||
|
? "bg-green-600 text-white border-green-600"
|
||||||
|
: "bg-white text-black border-black hover:bg-gray-100"
|
||||||
|
} disabled:opacity-50`}
|
||||||
|
>
|
||||||
|
Revise
|
||||||
|
<span className="block text-[10px] opacity-75">Same vibe</span>
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => setRegenerateMode("regenerate")}
|
||||||
|
disabled={state.generatingPage !== null}
|
||||||
|
className={`py-2 px-3 text-xs punk-text border-2 transition-colors
|
||||||
|
${regenerateMode === "regenerate"
|
||||||
|
? "bg-green-600 text-white border-green-600"
|
||||||
|
: "bg-white text-black border-black hover:bg-gray-100"
|
||||||
|
} disabled:opacity-50`}
|
||||||
|
>
|
||||||
|
New
|
||||||
|
<span className="block text-[10px] opacity-75">Start fresh</span>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
<button
|
<button
|
||||||
onClick={regeneratePage}
|
onClick={regeneratePage}
|
||||||
disabled={!feedback.trim() || state.generatingPage !== null}
|
disabled={!feedback.trim() || state.generatingPage !== null}
|
||||||
className="mt-3 w-full py-2 bg-black text-white punk-text flex items-center justify-center gap-2
|
className="w-full py-2 bg-black text-white punk-text flex items-center justify-center gap-2
|
||||||
hover:bg-green-600 disabled:opacity-50 disabled:cursor-not-allowed"
|
hover:bg-green-600 disabled:opacity-50 disabled:cursor-not-allowed"
|
||||||
>
|
>
|
||||||
<RefreshCw className={`w-4 h-4 ${state.generatingPage ? "animate-spin" : ""}`} />
|
<RefreshCw className={`w-4 h-4 ${state.generatingPage ? "animate-spin" : ""}`} />
|
||||||
Regenerate Page
|
{regenerateMode === "refine" ? "Refine Page" :
|
||||||
|
regenerateMode === "revise" ? "Revise Page" : "Regenerate Page"}
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue