commit 2466db43b3ddbf239f479ceec75dd863a12e92df Author: sakhiy Date: Wed May 27 00:55:23 2026 +0000 Initial commit: LiteLLM config + docker-compose + helper scripts diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..a9a768e --- /dev/null +++ b/.env.example @@ -0,0 +1,35 @@ +# AI PROXY CONFIG +# Copy file ini ke .env dan isi API key masing-masing + +# Gemini (Tier 1 - main) +GEMINI_API_KEY_1= +GEMINI_API_KEY_2= +GEMINI_API_KEY_3= +GEMINI_API_KEY_4= +GEMINI_API_KEY_5= + +# GitHub Models (Tier 2 - fallback) +GITHUB_TOKEN_1= +GITHUB_TOKEN_2= + +# Groq (Tier 3 - emergency) +GROQ_API_KEY_1= +GROQ_API_KEY_2= + +# OpenRouter (free tier) +OPENROUTER_API_KEY_1= +OPENROUTER_API_KEY_2= +OPENROUTER_API_KEY_3= + +# Others +DEEPSEEK_API_KEY= +TOGETHER_API_KEY= +HUGGINGFACE_API_KEY= + +# LiteLLM Admin +LITELLM_MASTER_KEY= +UI_USERNAME=admin +UI_PASSWORD= + +# Database +POSTGRES_PASSWORD=litellm diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b5d0376 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +# Sensitive files — jangan di-commit +.env + +# Docker runtime data +litellm_db_data/ + +# Python cache +__pycache__/ +*.pyc diff --git a/README.md b/README.md new file mode 100644 index 0000000..cc8f853 --- /dev/null +++ b/README.md @@ -0,0 +1,28 @@ +# LiteLLM Config + +Konfigurasi LiteLLM proxy Alcozaky Cookies. + +## Struktur + +``` +├── docker-compose.yml ← Cara jalanin LiteLLM + DB + Redis +├── config.yaml ← Daftar model AI & load balancing +├── .env.example ← Template API key (isi sendiri di .env) +├── .gitignore +├── scripts/ ← Helper Python (add/remove key, proxy) +└── README.md +``` + +## Cara Pake + +1. Copy `.env.example` ke `.env`, isi API key masing-masing +2. `docker compose up -d` +3. Proxy available di `http://localhost:4000` + +## Model Tiers + +| Tier | Model | Provider | Akun | +|------|-------|----------|------| +| 1 | `hermes-engine` | Gemini 2.5 Flash | 5 (load balanced) | +| 2 | `tier2-fallback` | GPT-4o Mini (GitHub) | 2 | +| 3 | `tier3-emergency` | Llama 3 70B (Groq) | 2 | diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..6fd8607 --- /dev/null +++ b/config.yaml @@ -0,0 +1,59 @@ +model_list: + # --- TIER 1: GEMINI (5 Akun Load Balancing) --- + - model_name: hermes-engine + litellm_params: + model: gemini/gemini-2.5-flash + api_key: os.environ/GEMINI_API_KEY_1 + tpm_fallback: true + - model_name: hermes-engine + litellm_params: + model: gemini/gemini-2.5-flash + api_key: os.environ/GEMINI_API_KEY_2 + tpm_fallback: true + - model_name: hermes-engine + litellm_params: + model: gemini/gemini-2.5-flash + api_key: os.environ/GEMINI_API_KEY_3 + tpm_fallback: true + - model_name: hermes-engine + litellm_params: + model: gemini/gemini-2.5-flash + api_key: os.environ/GEMINI_API_KEY_4 + tpm_fallback: true + - model_name: hermes-engine + litellm_params: + model: gemini/gemini-2.5-flash + api_key: os.environ/GEMINI_API_KEY_5 + tpm_fallback: true + + # --- TIER 2: GITHUB MODELS (2 Akun Cadangan) --- + - model_name: tier2-fallback + litellm_params: + model: github/gpt-4o-mini + api_key: os.environ/GITHUB_TOKEN_1 + - model_name: tier2-fallback + litellm_params: + model: github/gpt-4o-mini + api_key: os.environ/GITHUB_TOKEN_2 + + # --- TIER 3: GROQ (2 Akun Darurat) --- + - model_name: tier3-emergency + litellm_params: + model: groq/llama3-70b-8192 + api_key: os.environ/GROQ_API_KEY_1 + - model_name: tier3-emergency + litellm_params: + model: groq/llama3-70b-8192 + api_key: os.environ/GROQ_API_KEY_2 + +router_settings: + routing_strategy: simple-shuffle + fallbacks: + - hermes-engine: + - tier2-fallback + - tier3-emergency + retry_after: 0 + num_retries: 2 + timeout: 60 + allowed_fails: 1 + cooldown_time: 60 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..df2f007 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,39 @@ +version: '3.9' + +services: + litellm: + image: ghcr.io/berriai/litellm:main-latest + container_name: litellm-gateway + ports: + - "4000:4000" + volumes: + - ./config.yaml:/app/config.yaml + env_file: + - .env + environment: + - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY} + - DATABASE_URL=postgresql://litellm:${POSTGRES_PASSWORD}@db:5432/litellm + - REDIS_URL=redis://redis:6379 + - UI_USERNAME=${UI_USERNAME} + - UI_PASSWORD=${UI_PASSWORD} + depends_on: + - db + - redis + command: [ "--config", "/app/config.yaml", "--detailed_debug" ] + restart: unless-stopped + + db: + image: postgres:15-alpine + container_name: litellm-db + environment: + POSTGRES_DB: litellm + POSTGRES_USER: litellm + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + volumes: + - ./litellm_db_data:/var/lib/postgresql/data + restart: unless-stopped + + redis: + image: redis:alpine + container_name: litellm-redis + restart: unless-stopped diff --git a/scripts/add-groq3.py b/scripts/add-groq3.py new file mode 100644 index 0000000..170dd86 --- /dev/null +++ b/scripts/add-groq3.py @@ -0,0 +1,34 @@ +import re, subprocess + +# Read current proxy +with open("/opt/ai-proxy/proxy.py") as f: + code = f.read() + +# Add groq-3 provider after groq-2 +old = ''' "priority": 11, + },''' + +new = ''' "priority": 11, + }, + { + "name": "groq-3", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("GROQ_API_KEY_3"), + base_url="https://api.groq.com/openai/v1" + ), + "models": [ + "llama-3.1-8b-instant", + "llama-3.3-70b-versatile", + ], + "priority": 12, + },''' + +code = code.replace(old, new) + +with open("/opt/ai-proxy/proxy.py", "w") as f: + f.write(code) + +print("Added groq-3 provider!") + +subprocess.run(["systemctl", "restart", "ai-proxy"], capture_output=True) +print("Proxy restarted!") diff --git a/scripts/add-keys.py b/scripts/add-keys.py new file mode 100644 index 0000000..35df164 --- /dev/null +++ b/scripts/add-keys.py @@ -0,0 +1,35 @@ +import re, subprocess + +env_path = "/etc/litellm/.env" + +with open(env_path) as f: + content = f.read() + +# Add/replace key2 +k2 = "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" +k3 = "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" + +if "OPENROUTER_API_KEY_2=" in content: + content = re.sub(r"OPENROUTER_API_KEY_2=.*", f"OPENROUTER_API_KEY_2={k2}", content) +else: + content += f"\nOPENROUTER_API_KEY_2={k2}" + +if "OPENROUTER_API_KEY_3=" in content: + content = re.sub(r"OPENROUTER_API_KEY_3=.*", f"OPENROUTER_API_KEY_3={k3}", content) +else: + content += f"\nOPENROUTER_API_KEY_3={k3}" + +with open(env_path, "w") as f: + f.write(content) + +# Verify +with open(env_path) as f: + for line in f: + if "OPENROUTER" in line: + parts = line.strip().split("=", 1) + key = parts[0] + val = parts[1] if len(parts) > 1 else "" + print(f"{key}={val[:15]}...{val[-5:]}" if len(val) > 25 else f"{key}={val}") + +subprocess.run(["systemctl", "restart", "ai-proxy"], capture_output=True) +print("\nProxy restarted!") diff --git a/scripts/copy-keys.py b/scripts/copy-keys.py new file mode 100644 index 0000000..d34b57b --- /dev/null +++ b/scripts/copy-keys.py @@ -0,0 +1,19 @@ +import re + +with open("/opt/litellm/config.yaml") as f: + c = f.read() + +keys = re.findall(r'api_key:\s*"([^"]+)"', c) +gemini = [k for k in keys if k.startswith("AIza")] +github = [k for k in keys if k.startswith("ghp_")] +groq = [k for k in keys if k.startswith("gsk_")] + +with open("/etc/litellm/.env", "a") as f: + for i, k in enumerate(gemini): + f.write(f"GEMINI_API_KEY_{i+1}={k}\n") + for i, k in enumerate(github): + f.write(f"GITHUB_TOKEN_{i+1}={k}\n") + for i, k in enumerate(groq): + f.write(f"GROQ_API_KEY_{i+1}={k}\n") + +print(f"Added: {len(gemini)} Gemini, {len(github)} GitHub, {len(groq)} Groq") diff --git a/scripts/extract-keys.py b/scripts/extract-keys.py new file mode 100644 index 0000000..1bb3a3c --- /dev/null +++ b/scripts/extract-keys.py @@ -0,0 +1,40 @@ +import re, os, sys + +# Read the litellm config +with open("/opt/litellm/config.yaml") as f: + content = f.read() + +# Helper to extract keys by source name +def extract_keys_before(content, keyword, count=5): + """Extract api_key values that appear before a given keyword.""" + lines = content.split("\n") + keys = [] + for i, line in enumerate(lines): + if keyword in line.lower(): + # Look backwards for api_key + for j in range(i-1, max(0, i-10), -1): + if "api_key" in lines[j]: + key = lines[j].split('"')[1] if '"' in lines[j] else None + if key and key not in keys: + keys.append(key) + break + return keys[:count] + +# Extract by model group +gemini_keys = extract_keys_before(content, "gemini") +github_keys = extract_keys_before(content, "github") +groq_keys = extract_keys_before(content, "groq") + +# Print as env file format +print("# GENERATED ENV FILE") +print("# Copy these to /etc/litellm/.env") + +for i, k in enumerate(gemini_keys): + print(f"GEMINI_API_KEY_{i+1}={k}") + +for i, k in enumerate(github_keys): + # GitHub models use a different env var format + print(f"GITHUB_TOKEN_{i+1}={k}") + +for i, k in enumerate(groq_keys): + print(f"GROQ_API_KEY_{i+1}={k}") diff --git a/scripts/proxy.py b/scripts/proxy.py new file mode 100644 index 0000000..2c96958 --- /dev/null +++ b/scripts/proxy.py @@ -0,0 +1,335 @@ +#!/usr/bin/env python3 +""" +Alcozaky AI Proxy — OpenAI-compatible proxy +Routes & rotates between multiple free AI providers. +Usage: python3 /opt/ai-proxy/proxy.py +""" + +import os, json, time, asyncio, random, hashlib +from fastapi import FastAPI, Request, HTTPException +from fastapi.responses import JSONResponse, StreamingResponse +from openai import AsyncOpenAI +import httpx + +app = FastAPI(title="Alcozaky AI Proxy") + +# ============================================================ +# PROVIDERS CONFIG +# ============================================================ +PROVIDERS = [ + { + "name": "openrouter-1", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("OPENROUTER_API_KEY_1"), + base_url="https://openrouter.ai/api/v1" + ), + "models": [ + "openai/gpt-4o-mini", + "google/gemini-2.0-flash-001", + "deepseek/deepseek-chat", + "anthropic/claude-3-haiku", + "meta-llama/llama-3.2-3b-instruct:free", + ], + "priority": 1, + }, + { + "name": "openrouter-2", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("OPENROUTER_API_KEY_2"), + base_url="https://openrouter.ai/api/v1" + ), + "models": [ + "openai/gpt-4o-mini", + "meta-llama/llama-3.2-3b-instruct:free", + "google/gemini-2.0-flash-001", + ], + "priority": 2, + }, + { + "name": "openrouter-3", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("OPENROUTER_API_KEY_3"), + base_url="https://openrouter.ai/api/v1" + ), + "models": [ + "openai/gpt-4o-mini", + "deepseek/deepseek-chat", + "meta-llama/llama-3.2-3b-instruct:free", + ], + "priority": 3, + }, + { + "name": "github-1", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("GITHUB_TOKEN_1"), + base_url="https://models.inference.ai.azure.com" + ), + "models": ["gpt-4o-mini", "gpt-4o"], + "priority": 4, + }, + { + "name": "github-2", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("GITHUB_TOKEN_2"), + base_url="https://models.inference.ai.azure.com" + ), + "models": ["gpt-4o-mini"], + "priority": 5, + }, + { + "name": "github-3", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("GITHUB_TOKEN_3"), + base_url="https://models.inference.ai.azure.com" + ), + "models": ["gpt-4o-mini"], + "priority": 6, + }, + { + "name": "openrouter-2", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("OPENROUTER_API_KEY_2"), + base_url="https://openrouter.ai/api/v1" + ), + "models": [ + "openai/gpt-4o-mini", + "meta-llama/llama-3.2-3b-instruct:free", + "anthropic/claude-3-haiku", + ], + "priority": 2, + }, + { + "name": "groq-1", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("GROQ_API_KEY_1"), + base_url="https://api.groq.com/openai/v1" + ), + "models": [ + "llama-3.3-70b-versatile", + "mixtral-8x7b-32768", + "llama-3.1-8b-instant", + ], + "priority": 10, + }, + { + "name": "groq-2", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("GROQ_API_KEY_2"), + base_url="https://api.groq.com/openai/v1" + ), + "models": [ + "llama-3.1-8b-instant", + ], + "priority": 11, + }, + { + "name": "groq-3", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("GROQ_API_KEY_3"), + base_url="https://api.groq.com/openai/v1" + ), + "models": [ + "llama-3.1-8b-instant", + "llama-3.3-70b-versatile", + ], + "priority": 12, + }, + { + "name": "deepseek", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("DEEPSEEK_API_KEY"), + base_url="https://api.deepseek.com/v1" + ), + "models": ["deepseek-chat"], + "priority": 30, + }, + { + "name": "together", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("TOGETHER_API_KEY"), + base_url="https://api.together.xyz/v1" + ), + "models": ["meta-llama/Llama-3.2-3B-Instruct-Turbo"], + "priority": 40, + }, + { + "name": "huggingface", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("HUGGINGFACE_API_KEY"), + base_url="https://api-inference.huggingface.co/v1" + ), + "models": ["meta-llama/Llama-3.2-3B-Instruct"], + "priority": 50, + }, +] + +# Filter providers with valid API keys +active_providers = [] +for p in PROVIDERS: + try: + c = p["client"]() + if c.api_key: + active_providers.append(p) + except Exception: + pass +provider_failures = {} # name -> timestamp of last failure +provider_cooldowns = {} # name -> cooldown until + +# Model alias mapping +MODEL_ALIASES = { + "gpt-4o-mini": "openai/gpt-4o-mini", + "gpt4": "openai/gpt-4o-mini", + "fast": "llama-3.3-70b-versatile", + "cheap": "meta-llama/Llama-3.2-3B-Instruct-Turbo", + "gemini": "gemini-2.0-flash-exp", + "claude": "anthropic/claude-3-haiku", +} + + +def get_active_providers(): + """Return providers sorted by priority, excluding those in cooldown.""" + now = time.time() + result = [] + for p in sorted(active_providers, key=lambda x: x["priority"]): + # Check cooldown + if p["name"] in provider_cooldowns: + if now < provider_cooldowns[p["name"]]: + continue # Still in cooldown + else: + del provider_cooldowns[p["name"]] + result.append(p) + return result + + +def resolve_model(model_name): + """Resolve model alias to actual model name.""" + return MODEL_ALIASES.get(model_name, model_name) + + +@app.get("/health") +@app.get("/") +async def health(): + return {"status": "ok", "providers": len(active_providers)} + + +@app.get("/v1/models") +async def list_models(): + models = [] + for p in get_active_providers(): + for m in p["models"]: + models.append({ + "id": m, + "provider": p["name"], + "object": "model", + "created": int(time.time()), + "owned_by": p["name"], + }) + return {"object": "list", "data": models} + + +@app.post("/v1/chat/completions") +async def chat_completions(request: Request): + body = await request.json() + model = resolve_model(body.get("model", "gpt-4o-mini")) + messages = body.get("messages", []) + stream = body.get("stream", False) + max_tokens = body.get("max_tokens", 4096) + temperature = body.get("temperature", 0.7) + + # Try each provider + providers = get_active_providers() + first_provider = None + last_error = None + + for provider in providers: + if not first_provider: + first_provider = provider + + client = provider["client"]() + actual_model = model + + # Check if this provider has this model + # For OpenRouter: model names include prefix (e.g. openai/gpt-4o-mini) + # For others: use first available model if model not found + if model not in provider["models"]: + # Try to find closest match or use first model + if model.startswith("openai/") or model.startswith("google/") or model.startswith("anthropic/"): + # Only OpenRouter supports these prefixed models + if "openrouter" not in provider["name"]: + continue + else: + # Non-prefixed model - check if any provider model contains it + matches = [m for m in provider["models"] if model in m] + if not matches: + continue + actual_model = matches[0] + + try: + if stream: + return await stream_response(client, actual_model, messages, max_tokens, temperature) + else: + response = await client.chat.completions.create( + model=actual_model, + messages=messages, + max_tokens=max_tokens, + temperature=temperature, + ) + # Success! Reset failure count + if provider["name"] in provider_failures: + del provider_failures[provider["name"]] + return JSONResponse(content=response.model_dump()) + + except Exception as e: + last_error = str(e) + provider_failures[provider["name"]] = time.time() + # Cooldown for 30 seconds + provider_cooldowns[provider["name"]] = time.time() + 30 + continue + + # All providers failed + raise HTTPException( + status_code=503, + detail={ + "error": "All providers failed", + "last_error": last_error, + "active_providers": len(providers), + } + ) + + +async def stream_response(client, model, messages, max_tokens, temperature): + """Handle streaming responses.""" + response = await client.chat.completions.create( + model=model, + messages=messages, + max_tokens=max_tokens, + temperature=temperature, + stream=True, + ) + + async def generate(): + async for chunk in response: + yield f"data: {chunk.model_dump_json()}\n\n" + yield "data: [DONE]\n\n" + + return StreamingResponse( + generate(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + "X-Accel-Buffering": "no", + } + ) + + +if __name__ == "__main__": + import uvicorn + port = int(os.getenv("PORT", "4000")) + host = os.getenv("HOST", "0.0.0.0") + print(f"🚀 Alcozaky AI Proxy starting on {host}:{port}") + print(f"📡 Providers loaded: {len(active_providers)}") + for p in active_providers: + print(f" ├ {p['name']}: {len(p['models'])} models") + print(f"🔀 Rotation: latency-based auto-fallback") + uvicorn.run(app, host=host, port=port) \ No newline at end of file diff --git a/scripts/rm-gemini.py b/scripts/rm-gemini.py new file mode 100644 index 0000000..f170456 --- /dev/null +++ b/scripts/rm-gemini.py @@ -0,0 +1,22 @@ +import subprocess + +# Read current proxy +with open("/opt/ai-proxy/proxy.py") as f: + code = f.read() + +# Remove gemini-1 provider (from "name": "gemini-1" to the closing },) +import re +code = re.sub( + r'\s*\{\s*\n\s*"name":\s*"gemini-1",.*?\n\s*\},', + '', + code, + flags=re.DOTALL +) + +with open("/opt/ai-proxy/proxy.py", "w") as f: + f.write(code) + +print("Gemini removed!") + +subprocess.run(["systemctl", "restart", "ai-proxy"], capture_output=True) +print("Proxy restarted!") diff --git a/scripts/setup-github.py b/scripts/setup-github.py new file mode 100644 index 0000000..b870531 --- /dev/null +++ b/scripts/setup-github.py @@ -0,0 +1,80 @@ +import httpx, json, re, subprocess + +# Add tokens to env +with open("/etc/litellm/.env", "a") as f: + f.write("GITHUB_TOKEN_1=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\n") + f.write("GITHUB_TOKEN_2=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\n") + f.write("GITHUB_TOKEN_3=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\n") + +print("Tokens added to .env") + +# Test each GitHub token +tokens = [ + "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", + "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", + "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", +] + +for i, token in enumerate(tokens, 1): + try: + r = httpx.post( + "https://models.inference.ai.azure.com/chat/completions", + headers={"Authorization": f"Bearer {token}", "Content-Type": "application/json"}, + json={"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hai"}], "max_tokens": 10}, + timeout=15 + ) + if r.status_code == 200: + content = r.json()["choices"][0]["message"]["content"] + print(f"✅ GitHub {i}: {content[:50]}") + else: + print(f"❌ GitHub {i}: HTTP {r.status_code} - {r.text[:80]}") + except Exception as e: + print(f"❌ GitHub {i}: {str(e)[:60]}") + +# Now add GitHub providers to proxy script +with open("/opt/ai-proxy/proxy.py") as f: + code = f.read() + +# Add github providers after openrouter-3 section +github_block = ''' + { + "name": "github-1", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("GITHUB_TOKEN_1"), + base_url="https://models.inference.ai.azure.com" + ), + "models": ["gpt-4o-mini", "gpt-4o"], + "priority": 4, + }, + { + "name": "github-2", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("GITHUB_TOKEN_2"), + base_url="https://models.inference.ai.azure.com" + ), + "models": ["gpt-4o-mini"], + "priority": 5, + }, + { + "name": "github-3", + "client": lambda: AsyncOpenAI( + api_key=os.getenv("GITHUB_TOKEN_3"), + base_url="https://models.inference.ai.azure.com" + ), + "models": ["gpt-4o-mini"], + "priority": 6, + },''' + +# Insert after openrouter-3 (after priority 3) +code = code.replace( + ' "priority": 3,\n },', + ' "priority": 3,\n },' + github_block +) + +with open("/opt/ai-proxy/proxy.py", "w") as f: + f.write(code) + +print("GitHub providers added to proxy!") + +subprocess.run(["systemctl", "restart", "ai-proxy"], capture_output=True) +print("Proxy restarted!") diff --git a/scripts/test.py b/scripts/test.py new file mode 100644 index 0000000..5e48559 --- /dev/null +++ b/scripts/test.py @@ -0,0 +1,27 @@ +import httpx, json + +# Test each provider +tests = [ + ("openrouter-1", "openai/gpt-4o-mini"), + ("openrouter-2", "openai/gpt-4o-mini"), + ("openrouter-3", "openai/gpt-4o-mini"), + ("groq-1", "llama-3.3-70b-versatile"), + ("groq-2", "llama-3.1-8b-instant"), + ("groq-3", "llama-3.1-8b-instant"), + ("gemini-1", "gemini-2.0-flash-exp"), +] + +for provider, model in tests: + try: + r = httpx.post( + "http://localhost:4000/v1/chat/completions", + json={"model": model, "messages": [{"role": "user", "content": "hai"}], "max_tokens": 10}, + timeout=30 + ) + if r.status_code == 200: + content = r.json().get("choices", [{}])[0].get("message", {}).get("content", "") + print(f"✅ {provider} ({model}): {content[:50]}") + else: + print(f"❌ {provider} ({model}): HTTP {r.status_code}") + except Exception as e: + print(f"❌ {provider} ({model}): {str(e)[:60]}")