feat: add direct RunPod integration for video generation
- Add RunPod config helpers for image, video, text, whisper endpoints - Update VideoGenShapeUtil to call RunPod video endpoint directly - Add Ollama URL config for local LLM support - Remove dependency on AI orchestrator backend (not yet built) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
05197f8430
commit
083095c821
|
|
@ -16,6 +16,11 @@ export interface ClientConfig {
|
|||
openaiApiKey?: string
|
||||
runpodApiKey?: string
|
||||
runpodEndpointId?: string
|
||||
runpodImageEndpointId?: string
|
||||
runpodVideoEndpointId?: string
|
||||
runpodTextEndpointId?: string
|
||||
runpodWhisperEndpointId?: string
|
||||
ollamaUrl?: string
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -41,7 +46,12 @@ export function getClientConfig(): ClientConfig {
|
|||
webhookSecret: import.meta.env.VITE_QUARTZ_WEBHOOK_SECRET || import.meta.env.NEXT_PUBLIC_QUARTZ_WEBHOOK_SECRET,
|
||||
openaiApiKey: import.meta.env.VITE_OPENAI_API_KEY || import.meta.env.NEXT_PUBLIC_OPENAI_API_KEY,
|
||||
runpodApiKey: import.meta.env.VITE_RUNPOD_API_KEY || import.meta.env.NEXT_PUBLIC_RUNPOD_API_KEY,
|
||||
runpodEndpointId: import.meta.env.VITE_RUNPOD_ENDPOINT_ID || import.meta.env.NEXT_PUBLIC_RUNPOD_ENDPOINT_ID,
|
||||
runpodEndpointId: import.meta.env.VITE_RUNPOD_ENDPOINT_ID || import.meta.env.VITE_RUNPOD_IMAGE_ENDPOINT_ID || import.meta.env.NEXT_PUBLIC_RUNPOD_ENDPOINT_ID,
|
||||
runpodImageEndpointId: import.meta.env.VITE_RUNPOD_IMAGE_ENDPOINT_ID || import.meta.env.NEXT_PUBLIC_RUNPOD_IMAGE_ENDPOINT_ID,
|
||||
runpodVideoEndpointId: import.meta.env.VITE_RUNPOD_VIDEO_ENDPOINT_ID || import.meta.env.NEXT_PUBLIC_RUNPOD_VIDEO_ENDPOINT_ID,
|
||||
runpodTextEndpointId: import.meta.env.VITE_RUNPOD_TEXT_ENDPOINT_ID || import.meta.env.NEXT_PUBLIC_RUNPOD_TEXT_ENDPOINT_ID,
|
||||
runpodWhisperEndpointId: import.meta.env.VITE_RUNPOD_WHISPER_ENDPOINT_ID || import.meta.env.NEXT_PUBLIC_RUNPOD_WHISPER_ENDPOINT_ID,
|
||||
ollamaUrl: import.meta.env.VITE_OLLAMA_URL || import.meta.env.NEXT_PUBLIC_OLLAMA_URL,
|
||||
}
|
||||
} else {
|
||||
// Next.js environment
|
||||
|
|
@ -73,13 +83,18 @@ export function getClientConfig(): ClientConfig {
|
|||
webhookUrl: process.env.VITE_QUARTZ_WEBHOOK_URL || process.env.NEXT_PUBLIC_QUARTZ_WEBHOOK_URL,
|
||||
webhookSecret: process.env.VITE_QUARTZ_WEBHOOK_SECRET || process.env.NEXT_PUBLIC_QUARTZ_WEBHOOK_SECRET,
|
||||
runpodApiKey: process.env.VITE_RUNPOD_API_KEY || process.env.NEXT_PUBLIC_RUNPOD_API_KEY,
|
||||
runpodEndpointId: process.env.VITE_RUNPOD_ENDPOINT_ID || process.env.NEXT_PUBLIC_RUNPOD_ENDPOINT_ID,
|
||||
runpodEndpointId: process.env.VITE_RUNPOD_ENDPOINT_ID || process.env.VITE_RUNPOD_IMAGE_ENDPOINT_ID || process.env.NEXT_PUBLIC_RUNPOD_ENDPOINT_ID,
|
||||
runpodImageEndpointId: process.env.VITE_RUNPOD_IMAGE_ENDPOINT_ID || process.env.NEXT_PUBLIC_RUNPOD_IMAGE_ENDPOINT_ID,
|
||||
runpodVideoEndpointId: process.env.VITE_RUNPOD_VIDEO_ENDPOINT_ID || process.env.NEXT_PUBLIC_RUNPOD_VIDEO_ENDPOINT_ID,
|
||||
runpodTextEndpointId: process.env.VITE_RUNPOD_TEXT_ENDPOINT_ID || process.env.NEXT_PUBLIC_RUNPOD_TEXT_ENDPOINT_ID,
|
||||
runpodWhisperEndpointId: process.env.VITE_RUNPOD_WHISPER_ENDPOINT_ID || process.env.NEXT_PUBLIC_RUNPOD_WHISPER_ENDPOINT_ID,
|
||||
ollamaUrl: process.env.VITE_OLLAMA_URL || process.env.NEXT_PUBLIC_OLLAMA_URL,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get RunPod configuration for API calls
|
||||
* Get RunPod configuration for API calls (defaults to image endpoint)
|
||||
*/
|
||||
export function getRunPodConfig(): { apiKey: string; endpointId: string } | null {
|
||||
const config = getClientConfig()
|
||||
|
|
@ -94,6 +109,86 @@ export function getRunPodConfig(): { apiKey: string; endpointId: string } | null
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get RunPod configuration for image generation
|
||||
*/
|
||||
export function getRunPodImageConfig(): { apiKey: string; endpointId: string } | null {
|
||||
const config = getClientConfig()
|
||||
const endpointId = config.runpodImageEndpointId || config.runpodEndpointId
|
||||
|
||||
if (!config.runpodApiKey || !endpointId) {
|
||||
return null
|
||||
}
|
||||
|
||||
return {
|
||||
apiKey: config.runpodApiKey,
|
||||
endpointId: endpointId
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get RunPod configuration for video generation
|
||||
*/
|
||||
export function getRunPodVideoConfig(): { apiKey: string; endpointId: string } | null {
|
||||
const config = getClientConfig()
|
||||
|
||||
if (!config.runpodApiKey || !config.runpodVideoEndpointId) {
|
||||
return null
|
||||
}
|
||||
|
||||
return {
|
||||
apiKey: config.runpodApiKey,
|
||||
endpointId: config.runpodVideoEndpointId
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get RunPod configuration for text generation (vLLM)
|
||||
*/
|
||||
export function getRunPodTextConfig(): { apiKey: string; endpointId: string } | null {
|
||||
const config = getClientConfig()
|
||||
|
||||
if (!config.runpodApiKey || !config.runpodTextEndpointId) {
|
||||
return null
|
||||
}
|
||||
|
||||
return {
|
||||
apiKey: config.runpodApiKey,
|
||||
endpointId: config.runpodTextEndpointId
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get RunPod configuration for Whisper transcription
|
||||
*/
|
||||
export function getRunPodWhisperConfig(): { apiKey: string; endpointId: string } | null {
|
||||
const config = getClientConfig()
|
||||
|
||||
if (!config.runpodApiKey || !config.runpodWhisperEndpointId) {
|
||||
return null
|
||||
}
|
||||
|
||||
return {
|
||||
apiKey: config.runpodApiKey,
|
||||
endpointId: config.runpodWhisperEndpointId
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Ollama configuration for local LLM
|
||||
*/
|
||||
export function getOllamaConfig(): { url: string } | null {
|
||||
const config = getClientConfig()
|
||||
|
||||
if (!config.ollamaUrl) {
|
||||
return null
|
||||
}
|
||||
|
||||
return {
|
||||
url: config.ollamaUrl
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if RunPod integration is configured
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -6,9 +6,21 @@ import {
|
|||
TLBaseShape,
|
||||
} from "tldraw"
|
||||
import React, { useState } from "react"
|
||||
import { aiOrchestrator, isAIOrchestratorAvailable } from "@/lib/aiOrchestrator"
|
||||
import { getRunPodVideoConfig } from "@/lib/clientConfig"
|
||||
import { StandardizedToolWrapper } from "@/components/StandardizedToolWrapper"
|
||||
|
||||
// Type for RunPod job response
|
||||
interface RunPodJobResponse {
|
||||
id?: string
|
||||
status?: 'IN_QUEUE' | 'IN_PROGRESS' | 'STARTING' | 'COMPLETED' | 'FAILED' | 'CANCELLED'
|
||||
output?: {
|
||||
video_url?: string
|
||||
url?: string
|
||||
[key: string]: any
|
||||
} | string
|
||||
error?: string
|
||||
}
|
||||
|
||||
type IVideoGen = TLBaseShape<
|
||||
"VideoGen",
|
||||
{
|
||||
|
|
@ -66,6 +78,13 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
|
|||
return
|
||||
}
|
||||
|
||||
// Check RunPod config
|
||||
const runpodConfig = getRunPodVideoConfig()
|
||||
if (!runpodConfig) {
|
||||
setError("RunPod video endpoint not configured. Please set VITE_RUNPOD_API_KEY and VITE_RUNPOD_VIDEO_ENDPOINT_ID in your .env file.")
|
||||
return
|
||||
}
|
||||
|
||||
console.log('🎬 VideoGen: Starting generation with prompt:', prompt)
|
||||
setIsGenerating(true)
|
||||
setError(null)
|
||||
|
|
@ -78,28 +97,76 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
|
|||
})
|
||||
|
||||
try {
|
||||
// Check if AI Orchestrator is available
|
||||
const orchestratorAvailable = await isAIOrchestratorAvailable()
|
||||
const { apiKey, endpointId } = runpodConfig
|
||||
|
||||
if (orchestratorAvailable) {
|
||||
console.log('🎬 VideoGen: Using AI Orchestrator for video generation')
|
||||
// Submit job to RunPod
|
||||
console.log('🎬 VideoGen: Submitting to RunPod endpoint:', endpointId)
|
||||
const runUrl = `https://api.runpod.ai/v2/${endpointId}/run`
|
||||
|
||||
// Use AI Orchestrator (always routes to RunPod for video)
|
||||
const job = await aiOrchestrator.generateVideo(prompt, {
|
||||
model: shape.props.model,
|
||||
const response = await fetch(runUrl, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${apiKey}`,
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({
|
||||
input: {
|
||||
prompt: prompt,
|
||||
duration: shape.props.duration,
|
||||
wait: true // Wait for completion
|
||||
model: shape.props.model
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
if (job.status === 'completed' && job.result?.video_url) {
|
||||
const url = job.result.video_url
|
||||
console.log('✅ VideoGen: Generation complete, URL:', url)
|
||||
console.log(`💰 VideoGen: Cost: $${job.cost?.toFixed(4) || '0.00'}`)
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text()
|
||||
throw new Error(`RunPod API error: ${response.status} - ${errorText}`)
|
||||
}
|
||||
|
||||
const jobData = await response.json() as RunPodJobResponse
|
||||
console.log('🎬 VideoGen: Job submitted:', jobData.id)
|
||||
|
||||
if (!jobData.id) {
|
||||
throw new Error('No job ID returned from RunPod')
|
||||
}
|
||||
|
||||
// Poll for completion
|
||||
const statusUrl = `https://api.runpod.ai/v2/${endpointId}/status/${jobData.id}`
|
||||
let attempts = 0
|
||||
const maxAttempts = 120 // 4 minutes with 2s intervals (video can take a while)
|
||||
|
||||
while (attempts < maxAttempts) {
|
||||
await new Promise(resolve => setTimeout(resolve, 2000))
|
||||
attempts++
|
||||
|
||||
const statusResponse = await fetch(statusUrl, {
|
||||
headers: { 'Authorization': `Bearer ${apiKey}` }
|
||||
})
|
||||
|
||||
if (!statusResponse.ok) {
|
||||
console.warn(`🎬 VideoGen: Poll error (attempt ${attempts}):`, statusResponse.status)
|
||||
continue
|
||||
}
|
||||
|
||||
const statusData = await statusResponse.json() as RunPodJobResponse
|
||||
console.log(`🎬 VideoGen: Poll ${attempts}/${maxAttempts}, status:`, statusData.status)
|
||||
|
||||
if (statusData.status === 'COMPLETED') {
|
||||
// Extract video URL from output
|
||||
let url = ''
|
||||
if (typeof statusData.output === 'string') {
|
||||
url = statusData.output
|
||||
} else if (statusData.output?.video_url) {
|
||||
url = statusData.output.video_url
|
||||
} else if (statusData.output?.url) {
|
||||
url = statusData.output.url
|
||||
}
|
||||
|
||||
if (url) {
|
||||
console.log('✅ VideoGen: Generation complete, URL:', url)
|
||||
setVideoUrl(url)
|
||||
setIsGenerating(false)
|
||||
|
||||
// Update shape with video URL
|
||||
this.editor.updateShape({
|
||||
id: shape.id,
|
||||
type: shape.type,
|
||||
|
|
@ -110,21 +177,25 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
|
|||
prompt: prompt
|
||||
}
|
||||
})
|
||||
return
|
||||
} else {
|
||||
throw new Error('Video generation job did not return a video URL')
|
||||
console.log('⚠️ VideoGen: Completed but no video URL in output:', statusData.output)
|
||||
throw new Error('Video generation completed but no video URL returned')
|
||||
}
|
||||
} else {
|
||||
throw new Error(
|
||||
'AI Orchestrator not available. Please configure VITE_AI_ORCHESTRATOR_URL or set up the orchestrator on your Netcup RS 8000 server.'
|
||||
)
|
||||
} else if (statusData.status === 'FAILED') {
|
||||
throw new Error(statusData.error || 'Video generation failed')
|
||||
} else if (statusData.status === 'CANCELLED') {
|
||||
throw new Error('Video generation was cancelled')
|
||||
}
|
||||
}
|
||||
|
||||
throw new Error('Video generation timed out after 4 minutes')
|
||||
} catch (error: any) {
|
||||
const errorMessage = error.message || 'Unknown error during video generation'
|
||||
console.error('❌ VideoGen: Generation error:', errorMessage)
|
||||
setError(errorMessage)
|
||||
setIsGenerating(false)
|
||||
|
||||
// Update shape with error
|
||||
this.editor.updateShape({
|
||||
id: shape.id,
|
||||
type: shape.type,
|
||||
|
|
|
|||
Loading…
Reference in New Issue