Open source the proxy

Author: xzrderek

.gitignore

@@ -105,6 +105,9 @@ env.bak/
 venv.bak/
 *.backup
 
+# Secrets
+secrets.json
+
 # Spyder project settings
 .spyderproject
 .spyproject

eval_protocol/adapters/fireworks_tracing.py

@@ -7,9 +7,9 @@
 from __future__ import annotations
 import logging
 import requests
-import time
 from datetime import datetime
 from typing import Any, Dict, List, Optional, Protocol
+import os
 
 from eval_protocol.models import EvaluationRow, InputMetadata, ExecutionMetadata, Message
 from .base import BaseAdapter
@@ -349,9 +349,11 @@ def get_evaluation_rows(
         else:
             url = f"{self.base_url}/v1/traces"
 
+        headers = {"Authorization": f"Bearer {os.environ.get('FIREWORKS_API_KEY')}"}
+
         result = None
         try:
-            response = requests.get(url, params=params, timeout=self.timeout)
+            response = requests.get(url, params=params, timeout=self.timeout, headers=headers)
             response.raise_for_status()
             result = response.json()
         except requests.exceptions.HTTPError as e:

eval_protocol/proxy/Dockerfile.gateway

@@ -0,0 +1,26 @@
+# Metadata Extraction Gateway - Sits in front of LiteLLM
+FROM python:3.11-slim
+
+WORKDIR /app
+
+# Prevent Python from buffering stdout/stderr
+ENV PYTHONUNBUFFERED=1
+
+# Set secrets path to proxy directory
+ENV SECRETS_PATH=/app/proxy_core/secrets.json
+
+# Copy requirements file
+COPY ./requirements.txt /app/requirements.txt
+
+# Install dependencies
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy the proxy package
+COPY ./proxy_core /app/proxy_core
+
+# Expose port
+EXPOSE 4000
+
+# Run the gateway as a module
+# LITELLM_URL will be set by environment (docker-compose or Cloud Run)
+CMD ["python", "-m", "proxy_core.main"]

eval_protocol/proxy/config_no_cache.yaml

@@ -0,0 +1,10 @@
+model_list:
+  - model_name: "*"
+    litellm_params:
+      model: "*"
+litellm_settings:
+  success_callback: ["langfuse"]
+  failure_callback: ["langfuse"]
+  drop_params: True
+general_settings:
+  allow_client_side_credentials: true

eval_protocol/proxy/docker-compose.yml

@@ -0,0 +1,65 @@
+services:
+  # Redis - For tracking assistant message counts
+  redis:
+    image: redis:7-alpine
+    platform: linux/amd64
+    container_name: proxy-redis
+    ports:
+      - "6379:6379"  # Expose for debugging if needed
+    networks:
+      - litellm-network
+    restart: unless-stopped
+    command: redis-server --appendonly yes
+    volumes:
+      - redis-data:/data
+
+  # LiteLLM Backend - Handles actual LLM proxying
+  litellm-backend:
+    image: litellm/litellm:v1.77.3-stable
+    platform: linux/amd64
+    container_name: litellm-backend
+    command: ["--config", "/app/config.yaml", "--port", "4000", "--host", "0.0.0.0"]
+    environment:
+      - LANGFUSE_PUBLIC_KEY=dummy  # Set dummy public and private key so Langfuse instance initializes in LiteLLM, then real keys get sent in proxy
+      - LANGFUSE_SECRET_KEY=dummy
+      - LANGFUSE_HOST=https://langfuse.fireworks.ai
+    volumes:
+      - ./config_no_cache.yaml:/app/config.yaml:ro
+    ports:
+      - "4001:4000"  # Expose on 4001 for direct access if needed
+    networks:
+      - litellm-network
+    restart: unless-stopped
+
+  # Metadata Gateway - Public-facing service that extracts metadata from URLs
+  metadata-gateway:
+    build:
+      context: .
+      dockerfile: Dockerfile.gateway
+    container_name: metadata-gateway
+    environment:
+      # Point to the LiteLLM backend service
+      - LITELLM_URL=http://litellm-backend:4000
+      - PORT=4000
+      # Redis configuration for assistant message counting
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      # No password for local Redis
+      - REQUEST_TIMEOUT=300
+      # Logging level: INFO (default)
+      - LOG_LEVEL=INFO
+    ports:
+      - "4000:4000"  # Main public-facing port
+    networks:
+      - litellm-network
+    depends_on:
+      - litellm-backend
+      - redis
+    restart: unless-stopped
+
+networks:
+  litellm-network:
+    driver: bridge
+
+volumes:
+  redis-data:

eval_protocol/proxy/proxy_core/__init__.py

@@ -0,0 +1,10 @@
+from .models import ProxyConfig
+from .app import create_app
+from .auth import AuthProvider, NoAuthProvider
+
+__all__ = [
+    "ProxyConfig",
+    "create_app",
+    "AuthProvider",
+    "NoAuthProvider",
+]