diff --git a/docker-compose.yml b/docker-compose.yml index 77987e1..84a5af5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,7 @@ services: restart: unless-stopped environment: - OLLAMA_HOST=http://ollama:11434 + - RUNPOD_API_KEY=${RUNPOD_API_KEY} - INFISICAL_CLIENT_ID=${INFISICAL_CLIENT_ID} - INFISICAL_CLIENT_SECRET=${INFISICAL_CLIENT_SECRET} - INFISICAL_PROJECT_SLUG=ai-orchestrator diff --git a/server.py b/server.py index 3dbbb46..ea732fe 100644 --- a/server.py +++ b/server.py @@ -663,8 +663,10 @@ async def generate_text(request: TextRequest): cost_tracker["runpod_requests"] += 1 payload = { "prompt": request.prompt, - "max_tokens": request.max_tokens, - "temperature": request.temperature, + "sampling_params": { + "max_tokens": request.max_tokens, + "temperature": request.temperature, + }, } result = await submit_job(ENDPOINTS["llm"]["id"], payload) if "id" in result: