feat: add Image Studio for brand-consistent img2img generation
New folk-image-studio canvas shape with multi-image style analysis (Gemini vision), img2img via fal.ai FLUX and Gemini Nano Banana models, and image attachment support in folk-prompt chat. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
3e4cdcee0e
commit
bc139de0cc
|
|
@ -0,0 +1,793 @@
|
|||
import { FolkShape } from "./folk-shape";
|
||||
import { css, html } from "./tags";
|
||||
|
||||
const styles = css`
|
||||
:host {
|
||||
background: white;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
|
||||
min-width: 420px;
|
||||
min-height: 520px;
|
||||
}
|
||||
|
||||
.header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
padding: 8px 12px;
|
||||
background: linear-gradient(135deg, #f59e0b, #ec4899);
|
||||
color: white;
|
||||
border-radius: 8px 8px 0 0;
|
||||
font-size: 12px;
|
||||
font-weight: 600;
|
||||
cursor: move;
|
||||
}
|
||||
|
||||
.header-title {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
}
|
||||
|
||||
.header-actions {
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
}
|
||||
|
||||
.header-actions button {
|
||||
background: transparent;
|
||||
border: none;
|
||||
color: white;
|
||||
cursor: pointer;
|
||||
padding: 2px 6px;
|
||||
border-radius: 4px;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.header-actions button:hover {
|
||||
background: rgba(255, 255, 255, 0.2);
|
||||
}
|
||||
|
||||
.content {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: calc(100% - 36px);
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.section {
|
||||
padding: 12px;
|
||||
border-bottom: 1px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.section-label {
|
||||
font-size: 11px;
|
||||
font-weight: 600;
|
||||
color: #64748b;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.5px;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
/* Brand references */
|
||||
.upload-zone {
|
||||
border: 2px dashed #cbd5e1;
|
||||
border-radius: 8px;
|
||||
padding: 16px;
|
||||
text-align: center;
|
||||
cursor: pointer;
|
||||
font-size: 12px;
|
||||
color: #94a3b8;
|
||||
transition: border-color 0.2s;
|
||||
}
|
||||
|
||||
.upload-zone:hover, .upload-zone.drag-over {
|
||||
border-color: #f59e0b;
|
||||
color: #f59e0b;
|
||||
}
|
||||
|
||||
.thumb-grid {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 6px;
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.thumb {
|
||||
position: relative;
|
||||
width: 60px;
|
||||
height: 60px;
|
||||
border-radius: 6px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.thumb img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.thumb .remove-btn {
|
||||
position: absolute;
|
||||
top: 2px;
|
||||
right: 2px;
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
background: rgba(0,0,0,0.6);
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 50%;
|
||||
font-size: 10px;
|
||||
line-height: 16px;
|
||||
text-align: center;
|
||||
cursor: pointer;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.analyze-btn, .generate-btn {
|
||||
padding: 8px 16px;
|
||||
background: linear-gradient(135deg, #f59e0b, #ec4899);
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 6px;
|
||||
font-size: 13px;
|
||||
font-weight: 600;
|
||||
cursor: pointer;
|
||||
transition: opacity 0.2s;
|
||||
}
|
||||
|
||||
.analyze-btn:hover, .generate-btn:hover {
|
||||
opacity: 0.9;
|
||||
}
|
||||
|
||||
.analyze-btn:disabled, .generate-btn:disabled {
|
||||
opacity: 0.5;
|
||||
cursor: not-allowed;
|
||||
}
|
||||
|
||||
/* Style analysis display */
|
||||
.style-display {
|
||||
margin-top: 8px;
|
||||
padding: 10px;
|
||||
background: #fefce8;
|
||||
border-radius: 6px;
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.palette {
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
margin: 6px 0;
|
||||
}
|
||||
|
||||
.swatch {
|
||||
width: 24px;
|
||||
height: 24px;
|
||||
border-radius: 4px;
|
||||
border: 1px solid #e2e8f0;
|
||||
}
|
||||
|
||||
.keywords {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 4px;
|
||||
margin: 6px 0;
|
||||
}
|
||||
|
||||
.keyword-tag {
|
||||
padding: 2px 8px;
|
||||
background: #fef3c7;
|
||||
border-radius: 12px;
|
||||
font-size: 11px;
|
||||
color: #92400e;
|
||||
}
|
||||
|
||||
.brand-prefix {
|
||||
width: 100%;
|
||||
padding: 6px 8px;
|
||||
border: 1px solid #e2e8f0;
|
||||
border-radius: 4px;
|
||||
font-size: 11px;
|
||||
resize: vertical;
|
||||
font-family: inherit;
|
||||
min-height: 40px;
|
||||
}
|
||||
|
||||
/* Source + Controls */
|
||||
.source-preview {
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.source-preview img {
|
||||
max-width: 100%;
|
||||
max-height: 150px;
|
||||
border-radius: 6px;
|
||||
}
|
||||
|
||||
.prompt-input {
|
||||
width: 100%;
|
||||
padding: 10px 12px;
|
||||
border: 2px solid #e2e8f0;
|
||||
border-radius: 8px;
|
||||
font-size: 13px;
|
||||
resize: none;
|
||||
outline: none;
|
||||
font-family: inherit;
|
||||
margin-top: 8px;
|
||||
}
|
||||
|
||||
.prompt-input:focus {
|
||||
border-color: #ec4899;
|
||||
}
|
||||
|
||||
.controls {
|
||||
display: flex;
|
||||
gap: 8px;
|
||||
margin-top: 8px;
|
||||
align-items: center;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.provider-select {
|
||||
padding: 6px 10px;
|
||||
border: 2px solid #e2e8f0;
|
||||
border-radius: 6px;
|
||||
font-size: 12px;
|
||||
background: white;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.strength-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
font-size: 11px;
|
||||
color: #64748b;
|
||||
}
|
||||
|
||||
.strength-slider {
|
||||
width: 80px;
|
||||
}
|
||||
|
||||
.brand-toggle {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
font-size: 11px;
|
||||
color: #64748b;
|
||||
}
|
||||
|
||||
/* Results */
|
||||
.image-area {
|
||||
flex: 1;
|
||||
padding: 12px;
|
||||
overflow-y: auto;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.placeholder {
|
||||
flex: 1;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
color: #94a3b8;
|
||||
text-align: center;
|
||||
gap: 8px;
|
||||
padding: 24px;
|
||||
}
|
||||
|
||||
.placeholder-icon {
|
||||
font-size: 48px;
|
||||
opacity: 0.5;
|
||||
}
|
||||
|
||||
.image-item {
|
||||
position: relative;
|
||||
}
|
||||
|
||||
.generated-image {
|
||||
width: 100%;
|
||||
border-radius: 8px;
|
||||
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.image-meta {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
font-size: 11px;
|
||||
color: #64748b;
|
||||
margin-top: 4px;
|
||||
padding: 4px 8px;
|
||||
background: #f1f5f9;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.provider-badge {
|
||||
padding: 1px 6px;
|
||||
border-radius: 8px;
|
||||
font-size: 10px;
|
||||
font-weight: 600;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.provider-badge.fal { background: #8b5cf6; }
|
||||
.provider-badge.gemini { background: #0ea5e9; }
|
||||
|
||||
.loading {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
padding: 24px;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.spinner {
|
||||
width: 32px;
|
||||
height: 32px;
|
||||
border: 3px solid #e2e8f0;
|
||||
border-top-color: #ec4899;
|
||||
border-radius: 50%;
|
||||
animation: spin 1s linear infinite;
|
||||
}
|
||||
|
||||
@keyframes spin {
|
||||
to { transform: rotate(360deg); }
|
||||
}
|
||||
|
||||
.error {
|
||||
color: #ef4444;
|
||||
padding: 12px;
|
||||
background: #fef2f2;
|
||||
border-radius: 6px;
|
||||
font-size: 13px;
|
||||
}
|
||||
`;
|
||||
|
||||
interface StyleAnalysis {
|
||||
style_description: string;
|
||||
color_palette: string[];
|
||||
style_keywords: string[];
|
||||
brand_prompt_prefix: string;
|
||||
}
|
||||
|
||||
interface StudioResult {
|
||||
id: string;
|
||||
prompt: string;
|
||||
url: string;
|
||||
provider: string;
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
declare global {
|
||||
interface HTMLElementTagNameMap {
|
||||
"folk-image-studio": FolkImageStudio;
|
||||
}
|
||||
}
|
||||
|
||||
export class FolkImageStudio extends FolkShape {
|
||||
static override tagName = "folk-image-studio";
|
||||
|
||||
static {
|
||||
const sheet = new CSSStyleSheet();
|
||||
const parentRules = Array.from(FolkShape.styles.cssRules)
|
||||
.map((r) => r.cssText)
|
||||
.join("\n");
|
||||
const childRules = Array.from(styles.cssRules)
|
||||
.map((r) => r.cssText)
|
||||
.join("\n");
|
||||
sheet.replaceSync(`${parentRules}\n${childRules}`);
|
||||
this.styles = sheet;
|
||||
}
|
||||
|
||||
#brandReferences: string[] = []; // server paths or data URLs
|
||||
#styleAnalysis: StyleAnalysis | null = null;
|
||||
#sourceImage: string | null = null;
|
||||
#results: StudioResult[] = [];
|
||||
#provider: string = "fal";
|
||||
#strength = 0.85;
|
||||
#applyBrandStyle = true;
|
||||
#isAnalyzing = false;
|
||||
#isGenerating = false;
|
||||
#error: string | null = null;
|
||||
|
||||
// DOM refs
|
||||
#wrapper: HTMLElement | null = null;
|
||||
#refUploadInput: HTMLInputElement | null = null;
|
||||
#sourceUploadInput: HTMLInputElement | null = null;
|
||||
#promptInput: HTMLTextAreaElement | null = null;
|
||||
#brandPrefixInput: HTMLTextAreaElement | null = null;
|
||||
|
||||
override createRenderRoot() {
|
||||
const root = super.createRenderRoot();
|
||||
|
||||
this.#wrapper = document.createElement("div");
|
||||
this.#wrapper.innerHTML = this.#renderAll();
|
||||
|
||||
const slot = root.querySelector("slot");
|
||||
const containerDiv = slot?.parentElement as HTMLElement;
|
||||
if (containerDiv) containerDiv.replaceWith(this.#wrapper);
|
||||
|
||||
this.#bindEvents();
|
||||
return root;
|
||||
}
|
||||
|
||||
#renderAll(): string {
|
||||
return html`
|
||||
<div class="header">
|
||||
<span class="header-title">
|
||||
<span>🖌️</span>
|
||||
<span>Image Studio</span>
|
||||
</span>
|
||||
<div class="header-actions">
|
||||
<button class="close-btn" title="Close">×</button>
|
||||
</div>
|
||||
</div>
|
||||
<div class="content">
|
||||
<!-- Brand References -->
|
||||
<div class="section">
|
||||
<div class="section-label">Brand References</div>
|
||||
<div class="upload-zone ref-upload-zone">
|
||||
Click or drag images to add brand references
|
||||
</div>
|
||||
<input type="file" class="ref-upload-input" accept="image/*" multiple hidden />
|
||||
${this.#renderThumbs()}
|
||||
${this.#brandReferences.length >= 1
|
||||
? `<div style="margin-top:8px"><button class="analyze-btn" ${this.#isAnalyzing ? "disabled" : ""}>${this.#isAnalyzing ? "Analyzing..." : "Analyze Style"}</button></div>`
|
||||
: ""}
|
||||
${this.#renderStyleDisplay()}
|
||||
</div>
|
||||
|
||||
<!-- Source + Controls -->
|
||||
<div class="section">
|
||||
<div class="section-label">Source Image + Prompt</div>
|
||||
<div class="upload-zone source-upload-zone" style="padding:10px;">
|
||||
${this.#sourceImage
|
||||
? `<div class="source-preview"><img src="${this.#escapeHtml(this.#sourceImage)}" /></div>`
|
||||
: "Click or drag a source image"}
|
||||
</div>
|
||||
<input type="file" class="source-upload-input" accept="image/*" hidden />
|
||||
<textarea class="prompt-input" placeholder="Describe how to transform the image..." rows="2"></textarea>
|
||||
<div class="controls">
|
||||
<select class="provider-select">
|
||||
<option value="fal" ${this.#provider === "fal" ? "selected" : ""}>fal.ai FLUX</option>
|
||||
<option value="gemini" ${this.#provider === "gemini" ? "selected" : ""}>Nano Banana</option>
|
||||
<option value="gemini-pro" ${this.#provider === "gemini-pro" ? "selected" : ""}>Nano Banana Pro</option>
|
||||
<option value="gemini-flash-2" ${this.#provider === "gemini-flash-2" ? "selected" : ""}>Nano Banana 2</option>
|
||||
</select>
|
||||
<div class="strength-group" style="${this.#provider === "fal" ? "" : "display:none"}">
|
||||
<label>Strength</label>
|
||||
<input type="range" class="strength-slider" min="0" max="1" step="0.05" value="${this.#strength}" />
|
||||
<span class="strength-val">${this.#strength}</span>
|
||||
</div>
|
||||
<label class="brand-toggle">
|
||||
<input type="checkbox" class="brand-check" ${this.#applyBrandStyle ? "checked" : ""} />
|
||||
Apply Brand Style
|
||||
</label>
|
||||
<button class="generate-btn" ${this.#isGenerating ? "disabled" : ""}>${this.#isGenerating ? "Generating..." : "Generate"}</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Results -->
|
||||
<div class="image-area">
|
||||
${this.#error ? `<div class="error">${this.#escapeHtml(this.#error)}</div>` : ""}
|
||||
${this.#isGenerating ? '<div class="loading"><div class="spinner"></div><span>Generating image...</span></div>' : ""}
|
||||
${this.#results.length > 0 ? this.#renderImageList() : (!this.#isGenerating ? '<div class="placeholder"><span class="placeholder-icon">🖌️</span><span>Upload a source image and generate</span></div>' : "")}
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
#renderThumbs(): string {
|
||||
if (!this.#brandReferences.length) return "";
|
||||
return `<div class="thumb-grid">${this.#brandReferences
|
||||
.map(
|
||||
(ref, i) => `<div class="thumb">
|
||||
<img src="${this.#escapeHtml(ref)}" />
|
||||
<button class="remove-btn" data-ref-index="${i}">×</button>
|
||||
</div>`
|
||||
)
|
||||
.join("")}</div>`;
|
||||
}
|
||||
|
||||
#renderStyleDisplay(): string {
|
||||
if (!this.#styleAnalysis) return "";
|
||||
const s = this.#styleAnalysis;
|
||||
return `
|
||||
<div class="style-display">
|
||||
<div style="margin-bottom:4px;font-weight:600;">Style Analysis</div>
|
||||
<div style="margin-bottom:6px;">${this.#escapeHtml(s.style_description)}</div>
|
||||
<div class="palette">${s.color_palette.map((c) => `<div class="swatch" style="background:${c}" title="${c}"></div>`).join("")}</div>
|
||||
<div class="keywords">${s.style_keywords.map((k) => `<span class="keyword-tag">${this.#escapeHtml(k)}</span>`).join("")}</div>
|
||||
<div class="section-label" style="margin-top:6px;">Brand Prompt Prefix</div>
|
||||
<textarea class="brand-prefix">${this.#escapeHtml(s.brand_prompt_prefix)}</textarea>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
||||
#renderImageList(): string {
|
||||
return this.#results
|
||||
.map(
|
||||
(r) => `
|
||||
<div class="image-item">
|
||||
<img class="generated-image" src="${this.#escapeHtml(r.url)}" loading="lazy" />
|
||||
<div class="image-meta">
|
||||
<span class="provider-badge ${r.provider === "fal" ? "fal" : "gemini"}">${this.#escapeHtml(r.provider)}</span>
|
||||
<span>${this.#escapeHtml(r.prompt)}</span>
|
||||
</div>
|
||||
</div>
|
||||
`
|
||||
)
|
||||
.join("");
|
||||
}
|
||||
|
||||
#rerender() {
|
||||
if (!this.#wrapper) return;
|
||||
this.#wrapper.innerHTML = this.#renderAll();
|
||||
this.#bindEvents();
|
||||
}
|
||||
|
||||
#bindEvents() {
|
||||
if (!this.#wrapper) return;
|
||||
|
||||
const w = this.#wrapper;
|
||||
this.#refUploadInput = w.querySelector(".ref-upload-input");
|
||||
this.#sourceUploadInput = w.querySelector(".source-upload-input");
|
||||
this.#promptInput = w.querySelector(".prompt-input");
|
||||
this.#brandPrefixInput = w.querySelector(".brand-prefix");
|
||||
|
||||
// Close
|
||||
w.querySelector(".close-btn")?.addEventListener("click", (e) => {
|
||||
e.stopPropagation();
|
||||
this.dispatchEvent(new CustomEvent("close"));
|
||||
});
|
||||
|
||||
// Ref upload zone
|
||||
const refZone = w.querySelector(".ref-upload-zone") as HTMLElement | null;
|
||||
refZone?.addEventListener("click", (e) => {
|
||||
e.stopPropagation();
|
||||
this.#refUploadInput?.click();
|
||||
});
|
||||
refZone?.addEventListener("dragover", (e) => { e.preventDefault(); refZone.classList.add("drag-over"); });
|
||||
refZone?.addEventListener("dragleave", () => refZone.classList.remove("drag-over"));
|
||||
refZone?.addEventListener("drop", (e) => {
|
||||
e.preventDefault();
|
||||
refZone.classList.remove("drag-over");
|
||||
if (e.dataTransfer?.files) this.#handleRefFiles(e.dataTransfer.files);
|
||||
});
|
||||
this.#refUploadInput?.addEventListener("change", () => {
|
||||
if (this.#refUploadInput?.files) this.#handleRefFiles(this.#refUploadInput.files);
|
||||
});
|
||||
|
||||
// Source upload zone
|
||||
const srcZone = w.querySelector(".source-upload-zone") as HTMLElement | null;
|
||||
srcZone?.addEventListener("click", (e) => {
|
||||
e.stopPropagation();
|
||||
this.#sourceUploadInput?.click();
|
||||
});
|
||||
srcZone?.addEventListener("dragover", (e) => { e.preventDefault(); srcZone.classList.add("drag-over"); });
|
||||
srcZone?.addEventListener("dragleave", () => srcZone.classList.remove("drag-over"));
|
||||
srcZone?.addEventListener("drop", (e) => {
|
||||
e.preventDefault();
|
||||
srcZone.classList.remove("drag-over");
|
||||
if (e.dataTransfer?.files?.[0]) this.#handleSourceFile(e.dataTransfer.files[0]);
|
||||
});
|
||||
this.#sourceUploadInput?.addEventListener("change", () => {
|
||||
if (this.#sourceUploadInput?.files?.[0]) this.#handleSourceFile(this.#sourceUploadInput.files[0]);
|
||||
});
|
||||
|
||||
// Remove ref buttons
|
||||
w.querySelectorAll(".remove-btn[data-ref-index]").forEach((btn) => {
|
||||
btn.addEventListener("click", (e) => {
|
||||
e.stopPropagation();
|
||||
const idx = parseInt((btn as HTMLElement).dataset.refIndex || "0", 10);
|
||||
this.#brandReferences.splice(idx, 1);
|
||||
this.#rerender();
|
||||
});
|
||||
});
|
||||
|
||||
// Analyze button
|
||||
w.querySelector(".analyze-btn")?.addEventListener("click", (e) => {
|
||||
e.stopPropagation();
|
||||
this.#analyzeStyle();
|
||||
});
|
||||
|
||||
// Provider select
|
||||
const provSelect = w.querySelector(".provider-select") as HTMLSelectElement | null;
|
||||
provSelect?.addEventListener("change", () => {
|
||||
this.#provider = provSelect.value;
|
||||
this.#rerender();
|
||||
});
|
||||
|
||||
// Strength slider
|
||||
const slider = w.querySelector(".strength-slider") as HTMLInputElement | null;
|
||||
slider?.addEventListener("input", () => {
|
||||
this.#strength = parseFloat(slider.value);
|
||||
const valEl = w.querySelector(".strength-val");
|
||||
if (valEl) valEl.textContent = String(this.#strength);
|
||||
});
|
||||
|
||||
// Brand toggle
|
||||
const brandCheck = w.querySelector(".brand-check") as HTMLInputElement | null;
|
||||
brandCheck?.addEventListener("change", () => {
|
||||
this.#applyBrandStyle = brandCheck.checked;
|
||||
});
|
||||
|
||||
// Brand prefix textarea
|
||||
this.#brandPrefixInput?.addEventListener("input", () => {
|
||||
if (this.#styleAnalysis && this.#brandPrefixInput) {
|
||||
this.#styleAnalysis.brand_prompt_prefix = this.#brandPrefixInput.value;
|
||||
}
|
||||
});
|
||||
|
||||
// Generate button
|
||||
w.querySelector(".generate-btn")?.addEventListener("click", (e) => {
|
||||
e.stopPropagation();
|
||||
this.#generate();
|
||||
});
|
||||
|
||||
// Prevent drag on interactive elements
|
||||
this.#promptInput?.addEventListener("pointerdown", (e) => e.stopPropagation());
|
||||
this.#brandPrefixInput?.addEventListener("pointerdown", (e) => e.stopPropagation());
|
||||
slider?.addEventListener("pointerdown", (e) => e.stopPropagation());
|
||||
provSelect?.addEventListener("pointerdown", (e) => e.stopPropagation());
|
||||
brandCheck?.addEventListener("pointerdown", (e) => e.stopPropagation());
|
||||
}
|
||||
|
||||
#fileToDataUrl(file: File): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => resolve(reader.result as string);
|
||||
reader.onerror = reject;
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
}
|
||||
|
||||
async #handleRefFiles(files: FileList) {
|
||||
for (const file of Array.from(files)) {
|
||||
if (!file.type.startsWith("image/")) continue;
|
||||
try {
|
||||
// Upload to server for persistence
|
||||
const dataUrl = await this.#fileToDataUrl(file);
|
||||
const res = await fetch("/api/image-upload", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ image: dataUrl }),
|
||||
});
|
||||
if (res.ok) {
|
||||
const { url } = await res.json();
|
||||
this.#brandReferences.push(url);
|
||||
} else {
|
||||
this.#brandReferences.push(dataUrl);
|
||||
}
|
||||
} catch {
|
||||
const dataUrl = await this.#fileToDataUrl(file);
|
||||
this.#brandReferences.push(dataUrl);
|
||||
}
|
||||
}
|
||||
this.#rerender();
|
||||
}
|
||||
|
||||
async #handleSourceFile(file: File) {
|
||||
if (!file.type.startsWith("image/")) return;
|
||||
try {
|
||||
const dataUrl = await this.#fileToDataUrl(file);
|
||||
const res = await fetch("/api/image-upload", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ image: dataUrl }),
|
||||
});
|
||||
if (res.ok) {
|
||||
const { url } = await res.json();
|
||||
this.#sourceImage = url;
|
||||
} else {
|
||||
this.#sourceImage = dataUrl;
|
||||
}
|
||||
} catch {
|
||||
this.#sourceImage = await this.#fileToDataUrl(file);
|
||||
}
|
||||
this.#rerender();
|
||||
}
|
||||
|
||||
async #analyzeStyle() {
|
||||
if (this.#isAnalyzing || !this.#brandReferences.length) return;
|
||||
this.#isAnalyzing = true;
|
||||
this.#error = null;
|
||||
this.#rerender();
|
||||
|
||||
try {
|
||||
const res = await fetch("/api/image-gen/analyze-style", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ images: this.#brandReferences }),
|
||||
});
|
||||
if (!res.ok) throw new Error("Style analysis failed");
|
||||
this.#styleAnalysis = await res.json();
|
||||
} catch (e: any) {
|
||||
this.#error = e.message || "Analysis failed";
|
||||
} finally {
|
||||
this.#isAnalyzing = false;
|
||||
this.#rerender();
|
||||
}
|
||||
}
|
||||
|
||||
async #generate() {
|
||||
const prompt = this.#promptInput?.value.trim();
|
||||
if (!prompt || !this.#sourceImage || this.#isGenerating) return;
|
||||
|
||||
this.#isGenerating = true;
|
||||
this.#error = null;
|
||||
this.#rerender();
|
||||
|
||||
try {
|
||||
let fullPrompt = prompt;
|
||||
if (this.#applyBrandStyle && this.#styleAnalysis?.brand_prompt_prefix) {
|
||||
fullPrompt = this.#styleAnalysis.brand_prompt_prefix + " " + prompt;
|
||||
}
|
||||
|
||||
const isGemini = this.#provider !== "fal";
|
||||
const body: any = {
|
||||
prompt: fullPrompt,
|
||||
source_image: this.#sourceImage,
|
||||
provider: isGemini ? "gemini" : "fal",
|
||||
strength: this.#strength,
|
||||
};
|
||||
|
||||
if (isGemini && this.#provider !== "gemini") {
|
||||
body.model = this.#provider; // "gemini-pro" or "gemini-flash-2"
|
||||
}
|
||||
|
||||
if (this.#brandReferences.length) {
|
||||
body.reference_images = this.#brandReferences;
|
||||
}
|
||||
|
||||
const res = await fetch("/api/image-gen/img2img", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (!res.ok) throw new Error("Generation failed");
|
||||
const data = await res.json();
|
||||
const url = data.url || data.image_url;
|
||||
if (!url) throw new Error("No image returned");
|
||||
|
||||
this.#results.unshift({
|
||||
id: crypto.randomUUID(),
|
||||
prompt,
|
||||
url,
|
||||
provider: this.#provider,
|
||||
timestamp: new Date().toISOString(),
|
||||
});
|
||||
|
||||
if (this.#promptInput) this.#promptInput.value = "";
|
||||
} catch (e: any) {
|
||||
this.#error = e.message || "Generation failed";
|
||||
} finally {
|
||||
this.#isGenerating = false;
|
||||
this.#rerender();
|
||||
}
|
||||
}
|
||||
|
||||
#escapeHtml(text: string): string {
|
||||
const div = document.createElement("div");
|
||||
div.textContent = text;
|
||||
return div.innerHTML;
|
||||
}
|
||||
|
||||
override toJSON() {
|
||||
return {
|
||||
...super.toJSON(),
|
||||
type: "folk-image-studio",
|
||||
provider: this.#provider,
|
||||
strength: this.#strength,
|
||||
styleAnalysis: this.#styleAnalysis,
|
||||
brandReferences: this.#brandReferences,
|
||||
results: this.#results,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
|
@ -223,6 +223,70 @@ const styles = css`
|
|||
background: #e2e8f0;
|
||||
}
|
||||
|
||||
.attach-btn {
|
||||
padding: 10px 12px;
|
||||
background: transparent;
|
||||
border: 2px solid #e2e8f0;
|
||||
border-radius: 8px;
|
||||
font-size: 14px;
|
||||
cursor: pointer;
|
||||
transition: all 0.2s;
|
||||
}
|
||||
|
||||
.attach-btn:hover {
|
||||
border-color: #6366f1;
|
||||
}
|
||||
|
||||
.pending-images {
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.pending-thumb {
|
||||
position: relative;
|
||||
width: 48px;
|
||||
height: 48px;
|
||||
border-radius: 6px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.pending-thumb img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.pending-thumb .remove-img {
|
||||
position: absolute;
|
||||
top: 1px;
|
||||
right: 1px;
|
||||
width: 14px;
|
||||
height: 14px;
|
||||
background: rgba(0,0,0,0.6);
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 50%;
|
||||
font-size: 9px;
|
||||
line-height: 14px;
|
||||
text-align: center;
|
||||
cursor: pointer;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.msg-images {
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
margin-top: 6px;
|
||||
}
|
||||
|
||||
.msg-images img {
|
||||
max-width: 80px;
|
||||
max-height: 80px;
|
||||
border-radius: 4px;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
pre {
|
||||
background: #1e293b;
|
||||
color: #e2e8f0;
|
||||
|
|
@ -242,6 +306,7 @@ export interface ChatMessage {
|
|||
id: string;
|
||||
role: "user" | "assistant";
|
||||
content: string;
|
||||
images?: string[];
|
||||
timestamp: Date;
|
||||
}
|
||||
|
||||
|
|
@ -270,11 +335,14 @@ export class FolkPrompt extends FolkShape {
|
|||
#isStreaming = false;
|
||||
#error: string | null = null;
|
||||
#model = "gemini-flash";
|
||||
#pendingImages: string[] = [];
|
||||
|
||||
#messagesEl: HTMLElement | null = null;
|
||||
#promptInput: HTMLTextAreaElement | null = null;
|
||||
#modelSelect: HTMLSelectElement | null = null;
|
||||
#sendBtn: HTMLButtonElement | null = null;
|
||||
#attachInput: HTMLInputElement | null = null;
|
||||
#pendingImagesEl: HTMLElement | null = null;
|
||||
|
||||
get messages() {
|
||||
return this.#messages;
|
||||
|
|
@ -316,8 +384,11 @@ export class FolkPrompt extends FolkShape {
|
|||
<option value="mistral-small">Mistral Small (24B)</option>
|
||||
</optgroup>
|
||||
</select>
|
||||
<div class="pending-images"></div>
|
||||
<div class="prompt-row">
|
||||
<textarea class="prompt-input" placeholder="Type your message..." rows="2"></textarea>
|
||||
<button class="attach-btn" title="Attach image">📎</button>
|
||||
<input type="file" class="attach-input" accept="image/*" multiple hidden />
|
||||
${SpeechDictation.isSupported() ? '<button class="mic-btn" title="Voice dictation">🎤</button>' : ''}
|
||||
<button class="send-btn">→</button>
|
||||
</div>
|
||||
|
|
@ -336,8 +407,30 @@ export class FolkPrompt extends FolkShape {
|
|||
this.#promptInput = wrapper.querySelector(".prompt-input");
|
||||
this.#modelSelect = wrapper.querySelector(".model-select");
|
||||
this.#sendBtn = wrapper.querySelector(".send-btn");
|
||||
this.#attachInput = wrapper.querySelector(".attach-input");
|
||||
this.#pendingImagesEl = wrapper.querySelector(".pending-images");
|
||||
const clearBtn = wrapper.querySelector(".clear-btn") as HTMLButtonElement;
|
||||
const closeBtn = wrapper.querySelector(".close-btn") as HTMLButtonElement;
|
||||
const attachBtn = wrapper.querySelector(".attach-btn") as HTMLButtonElement | null;
|
||||
|
||||
// Attach button
|
||||
attachBtn?.addEventListener("click", (e) => {
|
||||
e.stopPropagation();
|
||||
this.#attachInput?.click();
|
||||
});
|
||||
this.#attachInput?.addEventListener("change", () => {
|
||||
if (!this.#attachInput?.files) return;
|
||||
for (const file of Array.from(this.#attachInput.files)) {
|
||||
if (!file.type.startsWith("image/")) continue;
|
||||
const reader = new FileReader();
|
||||
reader.onload = () => {
|
||||
this.#pendingImages.push(reader.result as string);
|
||||
this.#renderPendingImages();
|
||||
};
|
||||
reader.readAsDataURL(file);
|
||||
}
|
||||
this.#attachInput.value = "";
|
||||
});
|
||||
|
||||
// Send button
|
||||
this.#sendBtn?.addEventListener("click", (e) => {
|
||||
|
|
@ -415,21 +508,48 @@ export class FolkPrompt extends FolkShape {
|
|||
return root;
|
||||
}
|
||||
|
||||
#renderPendingImages() {
|
||||
if (!this.#pendingImagesEl) return;
|
||||
if (!this.#pendingImages.length) {
|
||||
this.#pendingImagesEl.innerHTML = "";
|
||||
return;
|
||||
}
|
||||
this.#pendingImagesEl.innerHTML = this.#pendingImages
|
||||
.map(
|
||||
(img, i) => `<div class="pending-thumb">
|
||||
<img src="${this.#escapeHtml(img)}" />
|
||||
<button class="remove-img" data-img-index="${i}">×</button>
|
||||
</div>`
|
||||
)
|
||||
.join("");
|
||||
this.#pendingImagesEl.querySelectorAll(".remove-img").forEach((btn) => {
|
||||
btn.addEventListener("click", (e) => {
|
||||
e.stopPropagation();
|
||||
const idx = parseInt((btn as HTMLElement).dataset.imgIndex || "0", 10);
|
||||
this.#pendingImages.splice(idx, 1);
|
||||
this.#renderPendingImages();
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async #send() {
|
||||
const content = this.#promptInput?.value.trim();
|
||||
if (!content || this.#isStreaming) return;
|
||||
|
||||
// Add user message
|
||||
// Add user message with any pending images
|
||||
const userMessage: ChatMessage = {
|
||||
id: crypto.randomUUID(),
|
||||
role: "user",
|
||||
content,
|
||||
images: this.#pendingImages.length ? [...this.#pendingImages] : undefined,
|
||||
timestamp: new Date(),
|
||||
};
|
||||
this.#messages.push(userMessage);
|
||||
|
||||
// Clear input
|
||||
// Clear input and pending images
|
||||
if (this.#promptInput) this.#promptInput.value = "";
|
||||
this.#pendingImages = [];
|
||||
this.#renderPendingImages();
|
||||
|
||||
this.#isStreaming = true;
|
||||
this.#error = null;
|
||||
|
|
@ -444,6 +564,7 @@ export class FolkPrompt extends FolkShape {
|
|||
messages: this.#messages.map((m) => ({
|
||||
role: m.role,
|
||||
content: m.content,
|
||||
...(m.images?.length ? { images: m.images } : {}),
|
||||
})),
|
||||
model: this.#model,
|
||||
}),
|
||||
|
|
@ -498,7 +619,13 @@ export class FolkPrompt extends FolkShape {
|
|||
}
|
||||
|
||||
let messagesHtml = this.#messages
|
||||
.map((msg) => `<div class="message ${msg.role}">${this.#formatContent(msg.content)}</div>`)
|
||||
.map((msg) => {
|
||||
let imgHtml = "";
|
||||
if (msg.images?.length) {
|
||||
imgHtml = `<div class="msg-images">${msg.images.map((src) => `<img src="${this.#escapeHtml(src)}" />`).join("")}</div>`;
|
||||
}
|
||||
return `<div class="message ${msg.role}">${this.#formatContent(msg.content)}${imgHtml}</div>`;
|
||||
})
|
||||
.join("");
|
||||
|
||||
if (streaming) {
|
||||
|
|
@ -544,7 +671,9 @@ export class FolkPrompt extends FolkShape {
|
|||
type: "folk-prompt",
|
||||
model: this.#model,
|
||||
messages: this.messages.map((msg) => ({
|
||||
...msg,
|
||||
role: msg.role,
|
||||
content: msg.content,
|
||||
...(msg.images?.length ? { images: msg.images } : {}),
|
||||
timestamp: msg.timestamp.toISOString(),
|
||||
})),
|
||||
};
|
||||
|
|
|
|||
|
|
@ -35,6 +35,7 @@ export * from "./folk-map";
|
|||
|
||||
// AI Integration Shapes
|
||||
export * from "./folk-image-gen";
|
||||
export * from "./folk-image-studio";
|
||||
export * from "./folk-video-gen";
|
||||
export * from "./folk-prompt";
|
||||
export * from "./folk-zine-gen";
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ const TOOL_HINTS: ToolHint[] = [
|
|||
{ tagName: "folk-calendar", label: "Calendar", icon: "📅", keywords: ["calendar", "date", "schedule", "event"] },
|
||||
{ tagName: "folk-map", label: "Map", icon: "🗺️", keywords: ["map", "location", "place", "geo"] },
|
||||
{ tagName: "folk-image-gen", label: "AI Image", icon: "🎨", keywords: ["image", "picture", "photo", "generate", "art", "draw"] },
|
||||
{ tagName: "folk-image-studio", label: "Image Studio", icon: "🖌️", keywords: ["image", "brand", "style", "redesign", "img2img", "reference", "consistency", "studio"] },
|
||||
{ tagName: "folk-video-gen", label: "AI Video", icon: "🎬", keywords: ["video", "clip", "animate", "movie", "film"] },
|
||||
{ tagName: "folk-prompt", label: "AI Chat", icon: "🤖", keywords: ["ai", "prompt", "llm", "assistant", "gpt"] },
|
||||
{ tagName: "folk-transcription", label: "Transcribe", icon: "🎙️", keywords: ["transcribe", "audio", "speech", "voice", "record"] },
|
||||
|
|
|
|||
280
server/index.ts
280
server/index.ts
|
|
@ -707,6 +707,31 @@ app.post("/api/x402-test", async (c) => {
|
|||
const FAL_KEY = process.env.FAL_KEY || "";
|
||||
const GEMINI_API_KEY = process.env.GEMINI_API_KEY || "";
|
||||
|
||||
// ── Image helpers ──
|
||||
|
||||
/** Read a /data/files/generated/... path from disk → base64 */
|
||||
async function readFileAsBase64(serverPath: string): Promise<string> {
|
||||
const filename = serverPath.split("/").pop();
|
||||
if (!filename) throw new Error("Invalid path");
|
||||
const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
|
||||
const file = Bun.file(resolve(dir, filename));
|
||||
if (!(await file.exists())) throw new Error("File not found: " + serverPath);
|
||||
const buf = await file.arrayBuffer();
|
||||
return Buffer.from(buf).toString("base64");
|
||||
}
|
||||
|
||||
/** Save a data:image/... URL to disk → return server-relative URL */
|
||||
async function saveDataUrlToDisk(dataUrl: string, prefix: string): Promise<string> {
|
||||
const match = dataUrl.match(/^data:image\/(\w+);base64,(.+)$/);
|
||||
if (!match) throw new Error("Invalid data URL");
|
||||
const ext = match[1] === "jpeg" ? "jpg" : match[1];
|
||||
const b64 = match[2];
|
||||
const filename = `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 6)}.${ext}`;
|
||||
const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
|
||||
await Bun.write(resolve(dir, filename), Buffer.from(b64, "base64"));
|
||||
return `/data/files/generated/${filename}`;
|
||||
}
|
||||
|
||||
// Image generation via fal.ai Flux Pro
|
||||
app.post("/api/image-gen", async (c) => {
|
||||
if (!FAL_KEY) return c.json({ error: "FAL_KEY not configured" }, 503);
|
||||
|
|
@ -750,6 +775,219 @@ app.post("/api/image-gen", async (c) => {
|
|||
return c.json({ url: imageUrl, image_url: imageUrl });
|
||||
});
|
||||
|
||||
// Upload image (data URL → disk)
|
||||
app.post("/api/image-upload", async (c) => {
|
||||
const { image } = await c.req.json();
|
||||
if (!image) return c.json({ error: "image required" }, 400);
|
||||
try {
|
||||
const url = await saveDataUrlToDisk(image, "upload");
|
||||
return c.json({ url });
|
||||
} catch (e: any) {
|
||||
console.error("[image-upload]", e.message);
|
||||
return c.json({ error: e.message }, 400);
|
||||
}
|
||||
});
|
||||
|
||||
// Analyze style from reference images via Gemini vision
|
||||
app.post("/api/image-gen/analyze-style", async (c) => {
|
||||
if (!GEMINI_API_KEY) return c.json({ error: "GEMINI_API_KEY not configured" }, 503);
|
||||
|
||||
const { images, context } = await c.req.json();
|
||||
if (!images?.length) return c.json({ error: "images required" }, 400);
|
||||
|
||||
const { GoogleGenerativeAI } = await import("@google/generative-ai");
|
||||
const genAI = new GoogleGenerativeAI(GEMINI_API_KEY);
|
||||
const model = genAI.getGenerativeModel({
|
||||
model: "gemini-2.5-flash",
|
||||
generationConfig: { responseMimeType: "application/json" },
|
||||
});
|
||||
|
||||
// Build multimodal parts
|
||||
const parts: any[] = [];
|
||||
for (const img of images) {
|
||||
let b64: string, mimeType: string;
|
||||
if (img.startsWith("data:")) {
|
||||
const match = img.match(/^data:(image\/\w+);base64,(.+)$/);
|
||||
if (!match) continue;
|
||||
mimeType = match[1];
|
||||
b64 = match[2];
|
||||
} else {
|
||||
// Server path
|
||||
b64 = await readFileAsBase64(img);
|
||||
mimeType = img.endsWith(".png") ? "image/png" : "image/jpeg";
|
||||
}
|
||||
parts.push({ inlineData: { data: b64, mimeType } });
|
||||
}
|
||||
|
||||
parts.push({
|
||||
text: `Analyze the visual style of these reference images. ${context ? `Context: ${context}` : ""}
|
||||
|
||||
Return JSON with these fields:
|
||||
{
|
||||
"style_description": "A detailed paragraph describing the overall visual style",
|
||||
"color_palette": ["#hex1", "#hex2", ...up to 6 dominant colors],
|
||||
"style_keywords": ["keyword1", "keyword2", ...up to 8 keywords],
|
||||
"brand_prompt_prefix": "A concise prompt prefix (under 50 words) that can be prepended to any image generation prompt to reproduce this style"
|
||||
}
|
||||
|
||||
Focus on: color palette, textures, composition patterns, mood, typography style, illustration approach.`,
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await model.generateContent({ contents: [{ role: "user", parts }] });
|
||||
const text = result.response.text();
|
||||
const parsed = JSON.parse(text);
|
||||
return c.json(parsed);
|
||||
} catch (e: any) {
|
||||
console.error("[analyze-style] error:", e.message);
|
||||
return c.json({ error: "Style analysis failed" }, 502);
|
||||
}
|
||||
});
|
||||
|
||||
// img2img generation via fal.ai or Gemini
|
||||
app.post("/api/image-gen/img2img", async (c) => {
|
||||
const { prompt, source_image, reference_images, provider = "fal", strength = 0.85, style, model: modelOverride } = await c.req.json();
|
||||
if (!prompt) return c.json({ error: "prompt required" }, 400);
|
||||
if (!source_image) return c.json({ error: "source_image required" }, 400);
|
||||
|
||||
const fullPrompt = (style ? style + " " : "") + prompt;
|
||||
|
||||
if (provider === "fal") {
|
||||
if (!FAL_KEY) return c.json({ error: "FAL_KEY not configured" }, 503);
|
||||
|
||||
try {
|
||||
// Save source to disk if data URL
|
||||
let sourceUrl: string;
|
||||
if (source_image.startsWith("data:")) {
|
||||
const path = await saveDataUrlToDisk(source_image, "img2img-src");
|
||||
sourceUrl = `https://rspace.online${path}`;
|
||||
} else if (source_image.startsWith("/")) {
|
||||
sourceUrl = `https://rspace.online${source_image}`;
|
||||
} else {
|
||||
sourceUrl = source_image;
|
||||
}
|
||||
|
||||
let falEndpoint: string;
|
||||
let falBody: any;
|
||||
|
||||
if (reference_images?.length) {
|
||||
// Multi-reference edit mode
|
||||
const imageUrls: string[] = [sourceUrl];
|
||||
for (const ref of reference_images) {
|
||||
if (ref.startsWith("data:")) {
|
||||
const path = await saveDataUrlToDisk(ref, "img2img-ref");
|
||||
imageUrls.push(`https://rspace.online${path}`);
|
||||
} else if (ref.startsWith("/")) {
|
||||
imageUrls.push(`https://rspace.online${ref}`);
|
||||
} else {
|
||||
imageUrls.push(ref);
|
||||
}
|
||||
}
|
||||
falEndpoint = "https://fal.run/fal-ai/flux-2/edit";
|
||||
falBody = { image_urls: imageUrls, prompt: fullPrompt };
|
||||
} else {
|
||||
// Single source img2img
|
||||
falEndpoint = "https://fal.run/fal-ai/flux/dev/image-to-image";
|
||||
falBody = { image_url: sourceUrl, prompt: fullPrompt, strength, num_inference_steps: 28 };
|
||||
}
|
||||
|
||||
const res = await fetch(falEndpoint, {
|
||||
method: "POST",
|
||||
headers: { Authorization: `Key ${FAL_KEY}`, "Content-Type": "application/json" },
|
||||
body: JSON.stringify(falBody),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const err = await res.text();
|
||||
console.error("[img2img] fal.ai error:", err);
|
||||
return c.json({ error: "img2img generation failed" }, 502);
|
||||
}
|
||||
|
||||
const data = await res.json();
|
||||
const resultUrl = data.images?.[0]?.url || data.output?.url;
|
||||
if (!resultUrl) return c.json({ error: "No image returned" }, 502);
|
||||
|
||||
// Download and save locally
|
||||
const imgRes = await fetch(resultUrl);
|
||||
if (!imgRes.ok) return c.json({ url: resultUrl, image_url: resultUrl });
|
||||
const imgBuf = await imgRes.arrayBuffer();
|
||||
const filename = `img2img-${Date.now()}.png`;
|
||||
const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
|
||||
await Bun.write(resolve(dir, filename), Buffer.from(imgBuf));
|
||||
const localUrl = `/data/files/generated/${filename}`;
|
||||
return c.json({ url: localUrl, image_url: localUrl });
|
||||
} catch (e: any) {
|
||||
console.error("[img2img] fal error:", e.message);
|
||||
return c.json({ error: "img2img generation failed" }, 502);
|
||||
}
|
||||
}
|
||||
|
||||
if (provider === "gemini") {
|
||||
if (!GEMINI_API_KEY) return c.json({ error: "GEMINI_API_KEY not configured" }, 503);
|
||||
|
||||
const { GoogleGenAI } = await import("@google/genai");
|
||||
const ai = new GoogleGenAI({ apiKey: GEMINI_API_KEY });
|
||||
|
||||
const geminiModelMap: Record<string, string> = {
|
||||
"gemini-flash": "gemini-2.5-flash-image",
|
||||
"gemini-pro": "gemini-3-pro-image-preview",
|
||||
"gemini-flash-2": "gemini-3.1-flash-image-preview",
|
||||
};
|
||||
const modelName = geminiModelMap[modelOverride || "gemini-flash"] || "gemini-2.5-flash-image";
|
||||
|
||||
try {
|
||||
// Build multimodal parts
|
||||
const parts: any[] = [{ text: fullPrompt }];
|
||||
|
||||
// Add source image
|
||||
const addImage = async (img: string) => {
|
||||
let b64: string, mimeType: string;
|
||||
if (img.startsWith("data:")) {
|
||||
const match = img.match(/^data:(image\/\w+);base64,(.+)$/);
|
||||
if (!match) return;
|
||||
mimeType = match[1];
|
||||
b64 = match[2];
|
||||
} else {
|
||||
b64 = await readFileAsBase64(img);
|
||||
mimeType = img.endsWith(".png") ? "image/png" : "image/jpeg";
|
||||
}
|
||||
parts.push({ inlineData: { data: b64, mimeType } });
|
||||
};
|
||||
|
||||
await addImage(source_image);
|
||||
if (reference_images?.length) {
|
||||
for (const ref of reference_images) await addImage(ref);
|
||||
}
|
||||
|
||||
const result = await ai.models.generateContent({
|
||||
model: modelName,
|
||||
contents: [{ role: "user", parts }],
|
||||
config: { responseModalities: ["Text", "Image"] },
|
||||
});
|
||||
|
||||
const resParts = result.candidates?.[0]?.content?.parts || [];
|
||||
for (const part of resParts) {
|
||||
if ((part as any).inlineData) {
|
||||
const { data: b64, mimeType } = (part as any).inlineData;
|
||||
const ext = mimeType?.includes("png") ? "png" : "jpg";
|
||||
const filename = `img2img-gemini-${Date.now()}.${ext}`;
|
||||
const dir = resolve(process.env.FILES_DIR || "./data/files", "generated");
|
||||
await Bun.write(resolve(dir, filename), Buffer.from(b64, "base64"));
|
||||
const url = `/data/files/generated/${filename}`;
|
||||
return c.json({ url, image_url: url });
|
||||
}
|
||||
}
|
||||
|
||||
return c.json({ error: "No image in Gemini response" }, 502);
|
||||
} catch (e: any) {
|
||||
console.error("[img2img] Gemini error:", e.message);
|
||||
return c.json({ error: "Gemini img2img failed" }, 502);
|
||||
}
|
||||
}
|
||||
|
||||
return c.json({ error: `Unknown provider: ${provider}` }, 400);
|
||||
});
|
||||
|
||||
// Text-to-video via fal.ai WAN 2.1
|
||||
app.post("/api/video-gen/t2v", async (c) => {
|
||||
if (!FAL_KEY) return c.json({ error: "FAL_KEY not configured" }, 503);
|
||||
|
|
@ -992,11 +1230,19 @@ app.post("/api/prompt", async (c) => {
|
|||
const genAI = new GoogleGenerativeAI(GEMINI_API_KEY);
|
||||
const geminiModel = genAI.getGenerativeModel({ model: GEMINI_MODELS[model] });
|
||||
|
||||
// Convert chat messages to Gemini contents format
|
||||
const contents = messages.map((m: { role: string; content: string }) => ({
|
||||
role: m.role === "assistant" ? "model" : "user",
|
||||
parts: [{ text: m.content }],
|
||||
}));
|
||||
// Convert chat messages to Gemini contents format (with optional images)
|
||||
const contents = messages.map((m: { role: string; content: string; images?: string[] }) => {
|
||||
const parts: any[] = [{ text: m.content }];
|
||||
if (m.images?.length) {
|
||||
for (const img of m.images) {
|
||||
const match = img.match(/^data:(image\/\w+);base64,(.+)$/);
|
||||
if (match) {
|
||||
parts.push({ inlineData: { data: match[2], mimeType: match[1] } });
|
||||
}
|
||||
}
|
||||
}
|
||||
return { role: m.role === "assistant" ? "model" : "user", parts };
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await geminiModel.generateContent({ contents });
|
||||
|
|
@ -1044,7 +1290,7 @@ app.post("/api/prompt", async (c) => {
|
|||
app.post("/api/gemini/image", async (c) => {
|
||||
if (!GEMINI_API_KEY) return c.json({ error: "GEMINI_API_KEY not configured" }, 503);
|
||||
|
||||
const { prompt, style, aspect_ratio } = await c.req.json();
|
||||
const { prompt, style, aspect_ratio, reference_images } = await c.req.json();
|
||||
if (!prompt) return c.json({ error: "prompt required" }, 400);
|
||||
|
||||
const styleHints: Record<string, string> = {
|
||||
|
|
@ -1062,13 +1308,33 @@ app.post("/api/gemini/image", async (c) => {
|
|||
const { GoogleGenAI } = await import("@google/genai");
|
||||
const ai = new GoogleGenAI({ apiKey: GEMINI_API_KEY });
|
||||
|
||||
// Build multimodal contents if reference images provided
|
||||
let contentsPayload: any = enhancedPrompt;
|
||||
if (reference_images?.length) {
|
||||
const parts: any[] = [{ text: enhancedPrompt + "\n\nUse these reference images as style guidance:" }];
|
||||
for (const ref of reference_images) {
|
||||
let b64: string, mimeType: string;
|
||||
if (ref.startsWith("data:")) {
|
||||
const match = ref.match(/^data:(image\/\w+);base64,(.+)$/);
|
||||
if (!match) continue;
|
||||
mimeType = match[1];
|
||||
b64 = match[2];
|
||||
} else {
|
||||
b64 = await readFileAsBase64(ref);
|
||||
mimeType = ref.endsWith(".png") ? "image/png" : "image/jpeg";
|
||||
}
|
||||
parts.push({ inlineData: { data: b64, mimeType } });
|
||||
}
|
||||
contentsPayload = [{ role: "user", parts }];
|
||||
}
|
||||
|
||||
const models = ["gemini-2.5-flash-image", "imagen-3.0-generate-002"];
|
||||
for (const modelName of models) {
|
||||
try {
|
||||
if (modelName.startsWith("gemini")) {
|
||||
const result = await ai.models.generateContent({
|
||||
model: modelName,
|
||||
contents: enhancedPrompt,
|
||||
contents: contentsPayload,
|
||||
config: { responseModalities: ["Text", "Image"] },
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue