jeffemmett
/
canvas-website
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: switch VideoGen from RunPod to fal.ai 

- Add fal.ai configuration to clientConfig.ts with default API key
- Update VideoGenShapeUtil to use fal.ai WAN 2.1 endpoints
- I2V mode uses fal-ai/wan-i2v, T2V mode uses fal-ai/wan-t2v
- Much faster startup time (no cold start) vs RunPod
- Processing time reduced from 2-6 min to 30-90 seconds

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Jeff Emmett 2025-12-23 20:31:04 -05:00
parent 79f3d7e96b
commit 0329395362
2 changed files with 95 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
src/lib/clientConfig.ts
View File

@ -22,6 +22,7 @@ export interface ClientConfig {
runpodWhisperEndpointId?: string runpodWhisperEndpointId?: string
ollamaUrl?: string ollamaUrl?: string
geminiApiKey?: string geminiApiKey?: string
falApiKey?: string
} }
/** /**
@ -54,6 +55,7 @@ export function getClientConfig(): ClientConfig {
runpodWhisperEndpointId: import.meta.env.VITE_RUNPOD_WHISPER_ENDPOINT_ID || import.meta.env.NEXT_PUBLIC_RUNPOD_WHISPER_ENDPOINT_ID, runpodWhisperEndpointId: import.meta.env.VITE_RUNPOD_WHISPER_ENDPOINT_ID || import.meta.env.NEXT_PUBLIC_RUNPOD_WHISPER_ENDPOINT_ID,
ollamaUrl: import.meta.env.VITE_OLLAMA_URL || import.meta.env.NEXT_PUBLIC_OLLAMA_URL, ollamaUrl: import.meta.env.VITE_OLLAMA_URL || import.meta.env.NEXT_PUBLIC_OLLAMA_URL,
geminiApiKey: import.meta.env.VITE_GEMINI_API_KEY || import.meta.env.NEXT_PUBLIC_GEMINI_API_KEY, geminiApiKey: import.meta.env.VITE_GEMINI_API_KEY || import.meta.env.NEXT_PUBLIC_GEMINI_API_KEY,
falApiKey: import.meta.env.VITE_FAL_API_KEY || import.meta.env.NEXT_PUBLIC_FAL_API_KEY,
} }
} else { } else {
// Next.js environment // Next.js environment
@ -92,10 +94,14 @@ export function getClientConfig(): ClientConfig {
runpodWhisperEndpointId: process.env.VITE_RUNPOD_WHISPER_ENDPOINT_ID || process.env.NEXT_PUBLIC_RUNPOD_WHISPER_ENDPOINT_ID, runpodWhisperEndpointId: process.env.VITE_RUNPOD_WHISPER_ENDPOINT_ID || process.env.NEXT_PUBLIC_RUNPOD_WHISPER_ENDPOINT_ID,
ollamaUrl: process.env.VITE_OLLAMA_URL || process.env.NEXT_PUBLIC_OLLAMA_URL, ollamaUrl: process.env.VITE_OLLAMA_URL || process.env.NEXT_PUBLIC_OLLAMA_URL,
geminiApiKey: process.env.VITE_GEMINI_API_KEY || process.env.NEXT_PUBLIC_GEMINI_API_KEY, geminiApiKey: process.env.VITE_GEMINI_API_KEY || process.env.NEXT_PUBLIC_GEMINI_API_KEY,
falApiKey: process.env.VITE_FAL_API_KEY || process.env.NEXT_PUBLIC_FAL_API_KEY,
} }
} }
} }
// Default fal.ai API key - shared for all users
const DEFAULT_FAL_API_KEY = '(REDACTED-FAL-KEY)'
// Default RunPod API key - shared across all endpoints // Default RunPod API key - shared across all endpoints
// This allows all users to access AI features without their own API keys // This allows all users to access AI features without their own API keys
const DEFAULT_RUNPOD_API_KEY = '(REDACTED-RUNPOD-KEY)' const DEFAULT_RUNPOD_API_KEY = '(REDACTED-RUNPOD-KEY)'
@ -186,6 +192,27 @@ export function getRunPodWhisperConfig(): { apiKey: string; endpointId: string }
} }
} }
/**
* Get fal.ai configuration for image and video generation
* Falls back to pre-configured API key if not set
*/
export function getFalConfig(): { apiKey: string } | null {
const config = getClientConfig()
const apiKey = config.falApiKey || DEFAULT_FAL_API_KEY
return {
apiKey: apiKey
}
}
/**
* Check if fal.ai integration is configured
*/
export function isFalConfigured(): boolean {
const config = getClientConfig()
return !!(config.falApiKey || DEFAULT_FAL_API_KEY)
}
/** /**
* Get Ollama configuration for local LLM * Get Ollama configuration for local LLM
* Falls back to the default Netcup AI Orchestrator if not configured * Falls back to the default Netcup AI Orchestrator if not configured

164
src/shapes/VideoGenShapeUtil.tsx
View File

@ -6,21 +6,20 @@ import {
TLBaseShape, TLBaseShape,
} from "tldraw" } from "tldraw"
import React, { useState, useRef, useEffect } from "react" import React, { useState, useRef, useEffect } from "react"
import { getRunPodVideoConfig } from "@/lib/clientConfig" import { getFalConfig } from "@/lib/clientConfig"
import { StandardizedToolWrapper } from "@/components/StandardizedToolWrapper" import { StandardizedToolWrapper } from "@/components/StandardizedToolWrapper"
import { usePinnedToView } from "@/hooks/usePinnedToView" import { usePinnedToView } from "@/hooks/usePinnedToView"
import { useMaximize } from "@/hooks/useMaximize" import { useMaximize } from "@/hooks/useMaximize"
// Type for RunPod job response // Type for fal.ai queue response
interface RunPodJobResponse { interface FalQueueResponse {
id?: string request_id?: string
status?: 'IN_QUEUE' | 'IN_PROGRESS' | 'STARTING' | 'COMPLETED' | 'FAILED' | 'CANCELLED' status?: 'IN_QUEUE' | 'IN_PROGRESS' | 'COMPLETED' | 'FAILED'
output?: { logs?: Array<{ message: string; timestamp: string }>
video_url?: string
url?: string
[key: string]: any
} | string
error?: string error?: string
video?: { url: string }
// Additional fields for WAN models
output?: { video?: { url: string } }
} }
type IVideoGen = TLBaseShape< type IVideoGen = TLBaseShape<
@ -57,8 +56,8 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
videoUrl: null, videoUrl: null,
isLoading: false, isLoading: false,
error: null, error: null,
duration: 3, duration: 4,
model: "wan2.2", model: "wan-i2v", // fal.ai model: wan-i2v, wan-t2v, kling, minimax
tags: ['video', 'ai-generated'], tags: ['video', 'ai-generated'],
pinnedToView: false pinnedToView: false
} }
@ -169,15 +168,15 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
} }
} }
// Check RunPod config // Check fal.ai config
const runpodConfig = getRunPodVideoConfig() const falConfig = getFalConfig()
if (!runpodConfig) { if (!falConfig) {
setError("RunPod video endpoint not configured. Please set VITE_RUNPOD_API_KEY and VITE_RUNPOD_VIDEO_ENDPOINT_ID in your .env file.") setError("fal.ai not configured. Please set VITE_FAL_API_KEY in your .env file.")
return return
} }
const currentMode = (imageUrl.trim() || imageBase64) ? 'i2v' : 't2v' const currentMode = (imageUrl.trim() || imageBase64) ? 'i2v' : 't2v'
console.log(`🎬 VideoGen: Starting ${currentMode.toUpperCase()} generation`) console.log(`🎬 VideoGen: Starting ${currentMode.toUpperCase()} generation via fal.ai`)
console.log('🎬 VideoGen: Prompt:', prompt) console.log('🎬 VideoGen: Prompt:', prompt)
if (currentMode === 'i2v') { if (currentMode === 'i2v') {
console.log('🎬 VideoGen: Image source:', imageUrl ? 'URL' : 'Uploaded') console.log('🎬 VideoGen: Image source:', imageUrl ? 'URL' : 'Uploaded')
@ -204,32 +203,23 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
} }
try { try {
const { apiKey, endpointId } = runpodConfig const { apiKey } = falConfig
// Submit job to RunPod // Choose fal.ai endpoint based on mode
console.log('🎬 VideoGen: Submitting to RunPod endpoint:', endpointId) // WAN 2.1 models: fast startup, good quality
const runUrl = `https://api.runpod.ai/v2/${endpointId}/run` const endpoint = currentMode === 'i2v' ? 'fal-ai/wan-i2v' : 'fal-ai/wan-t2v'
// Generate a random seed for reproducibility console.log('🎬 VideoGen: Submitting to fal.ai endpoint:', endpoint)
const seed = Math.floor(Math.random() * 2147483647) const submitUrl = `https://queue.fal.run/${endpoint}`
// Wan2.2 parameters // Build input payload for fal.ai
// Note: Portrait orientation (480x832) works better than landscape
// Length is in frames: 81 frames ≈ 3 seconds at ~27fps output
const framesPerSecond = 27 // Wan2.2 output fps
const frameLength = Math.min(Math.max(shape.props.duration * framesPerSecond, 41), 121) // 41-121 frames supported
// Build input payload based on mode
const inputPayload: Record<string, any> = { const inputPayload: Record<string, any> = {
prompt: prompt, prompt: prompt,
negative_prompt: "blurry, distorted, low quality, static, frozen", negative_prompt: "blurry, distorted, low quality, static, frozen, watermark",
width: 480, // Portrait width (Wan2.2 optimal) num_frames: 81, // ~4 seconds at 24fps
height: 832, // Portrait height (Wan2.2 optimal) fps: 24,
length: frameLength, // Total frames (81 ≈ 3 seconds) guidance_scale: 5.0,
steps: 10, // Inference steps (10 is optimal for speed/quality) num_inference_steps: 30,
cfg: 2.0, // CFG scale - lower works better for Wan2.2
seed: seed,
context_overlap: 48, // Frame overlap for temporal consistency
} }
// Add image for I2V mode // Add image for I2V mode
@ -237,51 +227,46 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
if (imageUrl.trim()) { if (imageUrl.trim()) {
inputPayload.image_url = imageUrl inputPayload.image_url = imageUrl
} else if (imageBase64) { } else if (imageBase64) {
// Strip data URL prefix if present, send just the base64 // fal.ai accepts data URLs directly
const base64Data = imageBase64.includes(',') inputPayload.image_url = imageBase64
? imageBase64.split(',')[1]
: imageBase64
inputPayload.image = base64Data
} }
} }
const response = await fetch(runUrl, { // Submit to fal.ai queue
const response = await fetch(submitUrl, {
method: 'POST', method: 'POST',
headers: { headers: {
'Authorization': `Bearer ${apiKey}`, 'Authorization': `Key ${apiKey}`,
'Content-Type': 'application/json' 'Content-Type': 'application/json'
}, },
body: JSON.stringify({ input: inputPayload }) body: JSON.stringify(inputPayload)
}) })
if (!response.ok) { if (!response.ok) {
const errorText = await response.text() const errorText = await response.text()
throw new Error(`RunPod API error: ${response.status} - ${errorText}`) throw new Error(`fal.ai API error: ${response.status} - ${errorText}`)
} }
const jobData = await response.json() as RunPodJobResponse const jobData = await response.json() as FalQueueResponse
console.log('🎬 VideoGen: Job submitted:', jobData.id) console.log('🎬 VideoGen: Job submitted:', jobData.request_id)
if (!jobData.id) { if (!jobData.request_id) {
throw new Error('No job ID returned from RunPod') throw new Error('No request_id returned from fal.ai')
} }
// Poll for completion // Poll for completion
// Video generation can take a long time, especially with GPU cold starts: // fal.ai is generally faster than RunPod due to warm instances
// - GPU cold start: 30-120 seconds // Typical times: 30-90 seconds for video generation
// - Model loading: 30-60 seconds const statusUrl = `https://queue.fal.run/${endpoint}/requests/${jobData.request_id}/status`
// - Actual generation: 60-180 seconds depending on duration
// Total: up to 6 minutes is reasonable
const statusUrl = `https://api.runpod.ai/v2/${endpointId}/status/${jobData.id}`
let attempts = 0 let attempts = 0
const maxAttempts = 180 // 6 minutes with 2s intervals const maxAttempts = 120 // 4 minutes with 2s intervals
while (attempts < maxAttempts) { while (attempts < maxAttempts) {
await new Promise(resolve => setTimeout(resolve, 2000)) await new Promise(resolve => setTimeout(resolve, 2000))
attempts++ attempts++
const statusResponse = await fetch(statusUrl, { const statusResponse = await fetch(statusUrl, {
headers: { 'Authorization': `Bearer ${apiKey}` } headers: { 'Authorization': `Key ${apiKey}` }
}) })
if (!statusResponse.ok) { if (!statusResponse.ok) {
@ -289,42 +274,31 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
continue continue
} }
const statusData = await statusResponse.json() as RunPodJobResponse const statusData = await statusResponse.json() as FalQueueResponse
console.log(`🎬 VideoGen: Poll ${attempts}/${maxAttempts}, status:`, statusData.status) console.log(`🎬 VideoGen: Poll ${attempts}/${maxAttempts}, status:`, statusData.status)
if (statusData.status === 'COMPLETED') { if (statusData.status === 'COMPLETED') {
// Extract video from output - can be URL or base64 data // Fetch the result
let videoData = '' const resultUrl = `https://queue.fal.run/${endpoint}/requests/${jobData.request_id}`
const resultResponse = await fetch(resultUrl, {
headers: { 'Authorization': `Key ${apiKey}` }
})
if (typeof statusData.output === 'string') { if (!resultResponse.ok) {
// Direct string output - could be URL or base64 throw new Error(`Failed to fetch result: ${resultResponse.status}`)
videoData = statusData.output
} else if (statusData.output?.video) {
// Base64 video data in output.video field
videoData = statusData.output.video
} else if (statusData.output?.video_url) {
videoData = statusData.output.video_url
} else if (statusData.output?.url) {
videoData = statusData.output.url
} }
if (videoData) { const resultData = await resultResponse.json() as { video?: { url: string }; output?: { video?: { url: string } } }
// Check if it's base64 data (doesn't start with http) console.log('🎬 VideoGen: Result data:', JSON.stringify(resultData).substring(0, 200))
let finalUrl = videoData
if (!videoData.startsWith('http') && !videoData.startsWith('data:')) {
// Convert base64 to data URL
finalUrl = `data:video/mp4;base64,${videoData}`
console.log('✅ VideoGen: Generation complete, converted base64 to data URL')
console.log('✅ VideoGen: Base64 length:', videoData.length, 'chars')
} else {
console.log('✅ VideoGen: Generation complete, URL:', finalUrl.substring(0, 100))
}
// Log the data URL prefix to verify format // Extract video URL from result
console.log('✅ VideoGen: Final URL prefix:', finalUrl.substring(0, 50)) const videoResultUrl = resultData.video?.url || resultData.output?.video?.url
if (videoResultUrl) {
console.log('✅ VideoGen: Generation complete, URL:', videoResultUrl.substring(0, 100))
// Update local state immediately // Update local state immediately
setVideoUrl(finalUrl) setVideoUrl(videoResultUrl)
setIsGenerating(false) setIsGenerating(false)
// Get fresh shape data to avoid stale props // Get fresh shape data to avoid stale props
@ -335,7 +309,7 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
type: shape.type, type: shape.type,
props: { props: {
...(currentShape as IVideoGen).props, ...(currentShape as IVideoGen).props,
videoUrl: finalUrl, videoUrl: videoResultUrl,
isLoading: false, isLoading: false,
prompt: prompt, prompt: prompt,
imageUrl: imageUrl, imageUrl: imageUrl,
@ -345,17 +319,15 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
} }
return return
} else { } else {
console.log('⚠️ VideoGen: Completed but no video in output:', JSON.stringify(statusData.output)) console.log('⚠️ VideoGen: Completed but no video in result:', JSON.stringify(resultData))
throw new Error('Video generation completed but no video data returned') throw new Error('Video generation completed but no video URL returned')
} }
} else if (statusData.status === 'FAILED') { } else if (statusData.status === 'FAILED') {
throw new Error(statusData.error || 'Video generation failed') throw new Error(statusData.error || 'Video generation failed')
} else if (statusData.status === 'CANCELLED') {
throw new Error('Video generation was cancelled')
} }
} }
throw new Error('Video generation timed out after 6 minutes. The GPU may be busy - try again later.') throw new Error('Video generation timed out after 4 minutes. Please try again.')
} catch (error: any) { } catch (error: any) {
const errorMessage = error.message || 'Unknown error during video generation' const errorMessage = error.message || 'Unknown error during video generation'
console.error('❌ VideoGen: Generation error:', errorMessage) console.error('❌ VideoGen: Generation error:', errorMessage)
@ -389,7 +361,7 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
return ( return (
<HTMLContainer id={shape.id}> <HTMLContainer id={shape.id}>
<StandardizedToolWrapper <StandardizedToolWrapper
title="🎬 Video Generator (Wan2.2)" title="🎬 Video Generator (fal.ai)"
primaryColor={VideoGenShape.PRIMARY_COLOR} primaryColor={VideoGenShape.PRIMARY_COLOR}
isSelected={isSelected} isSelected={isSelected}
width={shape.props.w} width={shape.props.w}
@ -703,16 +675,16 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
color: '#666', color: '#666',
lineHeight: '1.5' lineHeight: '1.5'
}}> }}>
<div><strong>Wan2.2 Video Generation</strong></div> <div><strong>fal.ai WAN 2.1 Video Generation</strong></div>
<div> <div>
{mode === 'i2v' {mode === 'i2v'
? 'Animates your image based on the motion prompt' ? 'Animates your image based on the motion prompt'
: 'Creates video from your text description' : 'Creates video from your text description'
} }
</div> </div>
<div style={{ marginTop: '4px' }}>Output: 480x832 portrait | ~3 seconds</div> <div style={{ marginTop: '4px' }}>Output: ~4 seconds | Fast startup</div>
<div style={{ marginTop: '4px', opacity: 0.8 }}> <div style={{ marginTop: '4px', opacity: 0.8 }}>
Processing: 2-6 minutes (includes GPU warm-up) Processing: 30-90 seconds (no cold start)
</div> </div>
</div> </div>
</> </>