feat: add direct RunPod integration for video generation

- Add RunPod config helpers for image, video, text, whisper endpoints
- Update VideoGenShapeUtil to call RunPod video endpoint directly
- Add Ollama URL config for local LLM support
- Remove dependency on AI orchestrator backend (not yet built)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

src/lib/clientConfig.ts

@@ -16,6 +16,11 @@ export interface ClientConfig {
   openaiApiKey?: string
   runpodApiKey?: string
   runpodEndpointId?: string
+  runpodImageEndpointId?: string
+  runpodVideoEndpointId?: string
+  runpodTextEndpointId?: string
+  runpodWhisperEndpointId?: string
+  ollamaUrl?: string
 }
 
 /**
@@ -41,7 +46,12 @@ export function getClientConfig(): ClientConfig {
         webhookSecret: import.meta.env.VITE_QUARTZ_WEBHOOK_SECRET || import.meta.env.NEXT_PUBLIC_QUARTZ_WEBHOOK_SECRET,
         openaiApiKey: import.meta.env.VITE_OPENAI_API_KEY || import.meta.env.NEXT_PUBLIC_OPENAI_API_KEY,
         runpodApiKey: import.meta.env.VITE_RUNPOD_API_KEY || import.meta.env.NEXT_PUBLIC_RUNPOD_API_KEY,
-        runpodEndpointId: import.meta.env.VITE_RUNPOD_ENDPOINT_ID || import.meta.env.NEXT_PUBLIC_RUNPOD_ENDPOINT_ID,
+        runpodEndpointId: import.meta.env.VITE_RUNPOD_ENDPOINT_ID || import.meta.env.VITE_RUNPOD_IMAGE_ENDPOINT_ID || import.meta.env.NEXT_PUBLIC_RUNPOD_ENDPOINT_ID,
+        runpodImageEndpointId: import.meta.env.VITE_RUNPOD_IMAGE_ENDPOINT_ID || import.meta.env.NEXT_PUBLIC_RUNPOD_IMAGE_ENDPOINT_ID,
+        runpodVideoEndpointId: import.meta.env.VITE_RUNPOD_VIDEO_ENDPOINT_ID || import.meta.env.NEXT_PUBLIC_RUNPOD_VIDEO_ENDPOINT_ID,
+        runpodTextEndpointId: import.meta.env.VITE_RUNPOD_TEXT_ENDPOINT_ID || import.meta.env.NEXT_PUBLIC_RUNPOD_TEXT_ENDPOINT_ID,
+        runpodWhisperEndpointId: import.meta.env.VITE_RUNPOD_WHISPER_ENDPOINT_ID || import.meta.env.NEXT_PUBLIC_RUNPOD_WHISPER_ENDPOINT_ID,
+        ollamaUrl: import.meta.env.VITE_OLLAMA_URL || import.meta.env.NEXT_PUBLIC_OLLAMA_URL,
       }
     } else {
       // Next.js environment
@@ -73,27 +83,112 @@ export function getClientConfig(): ClientConfig {
       webhookUrl: process.env.VITE_QUARTZ_WEBHOOK_URL || process.env.NEXT_PUBLIC_QUARTZ_WEBHOOK_URL,
       webhookSecret: process.env.VITE_QUARTZ_WEBHOOK_SECRET || process.env.NEXT_PUBLIC_QUARTZ_WEBHOOK_SECRET,
       runpodApiKey: process.env.VITE_RUNPOD_API_KEY || process.env.NEXT_PUBLIC_RUNPOD_API_KEY,
-      runpodEndpointId: process.env.VITE_RUNPOD_ENDPOINT_ID || process.env.NEXT_PUBLIC_RUNPOD_ENDPOINT_ID,
+      runpodEndpointId: process.env.VITE_RUNPOD_ENDPOINT_ID || process.env.VITE_RUNPOD_IMAGE_ENDPOINT_ID || process.env.NEXT_PUBLIC_RUNPOD_ENDPOINT_ID,
+      runpodImageEndpointId: process.env.VITE_RUNPOD_IMAGE_ENDPOINT_ID || process.env.NEXT_PUBLIC_RUNPOD_IMAGE_ENDPOINT_ID,
+      runpodVideoEndpointId: process.env.VITE_RUNPOD_VIDEO_ENDPOINT_ID || process.env.NEXT_PUBLIC_RUNPOD_VIDEO_ENDPOINT_ID,
+      runpodTextEndpointId: process.env.VITE_RUNPOD_TEXT_ENDPOINT_ID || process.env.NEXT_PUBLIC_RUNPOD_TEXT_ENDPOINT_ID,
+      runpodWhisperEndpointId: process.env.VITE_RUNPOD_WHISPER_ENDPOINT_ID || process.env.NEXT_PUBLIC_RUNPOD_WHISPER_ENDPOINT_ID,
+      ollamaUrl: process.env.VITE_OLLAMA_URL || process.env.NEXT_PUBLIC_OLLAMA_URL,
     }
   }
 }
 
 /**
- * Get RunPod configuration for API calls
+ * Get RunPod configuration for API calls (defaults to image endpoint)
  */
 export function getRunPodConfig(): { apiKey: string; endpointId: string } | null {
   const config = getClientConfig()
-  
+
   if (!config.runpodApiKey || !config.runpodEndpointId) {
     return null
   }
-  
+
   return {
     apiKey: config.runpodApiKey,
     endpointId: config.runpodEndpointId
   }
 }
 
+/**
+ * Get RunPod configuration for image generation
+ */
+export function getRunPodImageConfig(): { apiKey: string; endpointId: string } | null {
+  const config = getClientConfig()
+  const endpointId = config.runpodImageEndpointId || config.runpodEndpointId
+
+  if (!config.runpodApiKey || !endpointId) {
+    return null
+  }
+
+  return {
+    apiKey: config.runpodApiKey,
+    endpointId: endpointId
+  }
+}
+
+/**
+ * Get RunPod configuration for video generation
+ */
+export function getRunPodVideoConfig(): { apiKey: string; endpointId: string } | null {
+  const config = getClientConfig()
+
+  if (!config.runpodApiKey || !config.runpodVideoEndpointId) {
+    return null
+  }
+
+  return {
+    apiKey: config.runpodApiKey,
+    endpointId: config.runpodVideoEndpointId
+  }
+}
+
+/**
+ * Get RunPod configuration for text generation (vLLM)
+ */
+export function getRunPodTextConfig(): { apiKey: string; endpointId: string } | null {
+  const config = getClientConfig()
+
+  if (!config.runpodApiKey || !config.runpodTextEndpointId) {
+    return null
+  }
+
+  return {
+    apiKey: config.runpodApiKey,
+    endpointId: config.runpodTextEndpointId
+  }
+}
+
+/**
+ * Get RunPod configuration for Whisper transcription
+ */
+export function getRunPodWhisperConfig(): { apiKey: string; endpointId: string } | null {
+  const config = getClientConfig()
+
+  if (!config.runpodApiKey || !config.runpodWhisperEndpointId) {
+    return null
+  }
+
+  return {
+    apiKey: config.runpodApiKey,
+    endpointId: config.runpodWhisperEndpointId
+  }
+}
+
+/**
+ * Get Ollama configuration for local LLM
+ */
+export function getOllamaConfig(): { url: string } | null {
+  const config = getClientConfig()
+
+  if (!config.ollamaUrl) {
+    return null
+  }
+
+  return {
+    url: config.ollamaUrl
+  }
+}
+
 /**
  * Check if RunPod integration is configured
  */

src/shapes/VideoGenShapeUtil.tsx

@@ -6,9 +6,21 @@ import {
   TLBaseShape,
 } from "tldraw"
 import React, { useState } from "react"
-import { aiOrchestrator, isAIOrchestratorAvailable } from "@/lib/aiOrchestrator"
+import { getRunPodVideoConfig } from "@/lib/clientConfig"
 import { StandardizedToolWrapper } from "@/components/StandardizedToolWrapper"
 
+// Type for RunPod job response
+interface RunPodJobResponse {
+  id?: string
+  status?: 'IN_QUEUE' | 'IN_PROGRESS' | 'STARTING' | 'COMPLETED' | 'FAILED' | 'CANCELLED'
+  output?: {
+    video_url?: string
+    url?: string
+    [key: string]: any
+  } | string
+  error?: string
+}
+
 type IVideoGen = TLBaseShape<
   "VideoGen",
   {
@@ -66,6 +78,13 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
         return
       }
 
+      // Check RunPod config
+      const runpodConfig = getRunPodVideoConfig()
+      if (!runpodConfig) {
+        setError("RunPod video endpoint not configured. Please set VITE_RUNPOD_API_KEY and VITE_RUNPOD_VIDEO_ENDPOINT_ID in your .env file.")
+        return
+      }
+
       console.log('🎬 VideoGen: Starting generation with prompt:', prompt)
       setIsGenerating(true)
       setError(null)
@@ -78,53 +97,105 @@ export class VideoGenShape extends BaseBoxShapeUtil<IVideoGen> {
       })
 
       try {
-        // Check if AI Orchestrator is available
+        const { apiKey, endpointId } = runpodConfig
-        const orchestratorAvailable = await isAIOrchestratorAvailable()
 
-        if (orchestratorAvailable) {
+        // Submit job to RunPod
-          console.log('🎬 VideoGen: Using AI Orchestrator for video generation')
+        console.log('🎬 VideoGen: Submitting to RunPod endpoint:', endpointId)
+        const runUrl = `https://api.runpod.ai/v2/${endpointId}/run`
 
-          // Use AI Orchestrator (always routes to RunPod for video)
+        const response = await fetch(runUrl, {
-          const job = await aiOrchestrator.generateVideo(prompt, {
+          method: 'POST',
-            model: shape.props.model,
+          headers: {
-            duration: shape.props.duration,
+            'Authorization': `Bearer ${apiKey}`,
-            wait: true // Wait for completion
+            'Content-Type': 'application/json'
+          },
+          body: JSON.stringify({
+            input: {
+              prompt: prompt,
+              duration: shape.props.duration,
+              model: shape.props.model
+            }
+          })
+        })
+
+        if (!response.ok) {
+          const errorText = await response.text()
+          throw new Error(`RunPod API error: ${response.status} - ${errorText}`)
+        }
+
+        const jobData = await response.json() as RunPodJobResponse
+        console.log('🎬 VideoGen: Job submitted:', jobData.id)
+
+        if (!jobData.id) {
+          throw new Error('No job ID returned from RunPod')
+        }
+
+        // Poll for completion
+        const statusUrl = `https://api.runpod.ai/v2/${endpointId}/status/${jobData.id}`
+        let attempts = 0
+        const maxAttempts = 120 // 4 minutes with 2s intervals (video can take a while)
+
+        while (attempts < maxAttempts) {
+          await new Promise(resolve => setTimeout(resolve, 2000))
+          attempts++
+
+          const statusResponse = await fetch(statusUrl, {
+            headers: { 'Authorization': `Bearer ${apiKey}` }
           })
 
-          if (job.status === 'completed' && job.result?.video_url) {
+          if (!statusResponse.ok) {
-            const url = job.result.video_url
+            console.warn(`🎬 VideoGen: Poll error (attempt ${attempts}):`, statusResponse.status)
-            console.log('✅ VideoGen: Generation complete, URL:', url)
+            continue
-            console.log(`💰 VideoGen: Cost: $${job.cost?.toFixed(4) || '0.00'}`)
+          }
-
+
-            setVideoUrl(url)
+          const statusData = await statusResponse.json() as RunPodJobResponse
-            setIsGenerating(false)
+          console.log(`🎬 VideoGen: Poll ${attempts}/${maxAttempts}, status:`, statusData.status)
-
+
-            // Update shape with video URL
+          if (statusData.status === 'COMPLETED') {
-            this.editor.updateShape({
+            // Extract video URL from output
-              id: shape.id,
+            let url = ''
-              type: shape.type,
+            if (typeof statusData.output === 'string') {
-              props: {
+              url = statusData.output
-                ...shape.props,
+            } else if (statusData.output?.video_url) {
-                videoUrl: url,
+              url = statusData.output.video_url
-                isLoading: false,
+            } else if (statusData.output?.url) {
-                prompt: prompt
+              url = statusData.output.url
-              }
+            }
-            })
+
-          } else {
+            if (url) {
-            throw new Error('Video generation job did not return a video URL')
+              console.log('✅ VideoGen: Generation complete, URL:', url)
+              setVideoUrl(url)
+              setIsGenerating(false)
+
+              this.editor.updateShape({
+                id: shape.id,
+                type: shape.type,
+                props: {
+                  ...shape.props,
+                  videoUrl: url,
+                  isLoading: false,
+                  prompt: prompt
+                }
+              })
+              return
+            } else {
+              console.log('⚠️ VideoGen: Completed but no video URL in output:', statusData.output)
+              throw new Error('Video generation completed but no video URL returned')
+            }
+          } else if (statusData.status === 'FAILED') {
+            throw new Error(statusData.error || 'Video generation failed')
+          } else if (statusData.status === 'CANCELLED') {
+            throw new Error('Video generation was cancelled')
           }
-        } else {
-          throw new Error(
-            'AI Orchestrator not available. Please configure VITE_AI_ORCHESTRATOR_URL or set up the orchestrator on your Netcup RS 8000 server.'
-          )
         }
+
+        throw new Error('Video generation timed out after 4 minutes')
       } catch (error: any) {
         const errorMessage = error.message || 'Unknown error during video generation'
         console.error('❌ VideoGen: Generation error:', errorMessage)
         setError(errorMessage)
         setIsGenerating(false)
 
-        // Update shape with error
         this.editor.updateShape({
           id: shape.id,
           type: shape.type,