Initial commit: LiteLLM config + docker-compose + helper scripts

2026-05-27 00:55:23 +00:00
commit 2466db43b3
13 changed files with 762 additions and 0 deletions
@@ -0,0 +1,59 @@
+model_list:
+  # --- TIER 1: GEMINI (5 Akun Load Balancing) ---
+  - model_name: hermes-engine
+    litellm_params:
+      model: gemini/gemini-2.5-flash
+      api_key: os.environ/GEMINI_API_KEY_1
+      tpm_fallback: true
+  - model_name: hermes-engine
+    litellm_params:
+      model: gemini/gemini-2.5-flash
+      api_key: os.environ/GEMINI_API_KEY_2
+      tpm_fallback: true
+  - model_name: hermes-engine
+    litellm_params:
+      model: gemini/gemini-2.5-flash
+      api_key: os.environ/GEMINI_API_KEY_3
+      tpm_fallback: true
+  - model_name: hermes-engine
+    litellm_params:
+      model: gemini/gemini-2.5-flash
+      api_key: os.environ/GEMINI_API_KEY_4
+      tpm_fallback: true
+  - model_name: hermes-engine
+    litellm_params:
+      model: gemini/gemini-2.5-flash
+      api_key: os.environ/GEMINI_API_KEY_5
+      tpm_fallback: true
+
+  # --- TIER 2: GITHUB MODELS (2 Akun Cadangan) ---
+  - model_name: tier2-fallback
+    litellm_params:
+      model: github/gpt-4o-mini
+      api_key: os.environ/GITHUB_TOKEN_1
+  - model_name: tier2-fallback
+    litellm_params:
+      model: github/gpt-4o-mini
+      api_key: os.environ/GITHUB_TOKEN_2
+
+  # --- TIER 3: GROQ (2 Akun Darurat) ---
+  - model_name: tier3-emergency
+    litellm_params:
+      model: groq/llama3-70b-8192
+      api_key: os.environ/GROQ_API_KEY_1
+  - model_name: tier3-emergency
+    litellm_params:
+      model: groq/llama3-70b-8192
+      api_key: os.environ/GROQ_API_KEY_2
+
+router_settings:
+  routing_strategy: simple-shuffle
+  fallbacks:
+    - hermes-engine:
+        - tier2-fallback
+        - tier3-emergency
+  retry_after: 0
+  num_retries: 2
+  timeout: 60
+  allowed_fails: 1
+  cooldown_time: 60