Feat/update openrouter proxy latest main

alembic/versions/2026_05_05_openrouter_secrets_and_eval_run_cost.py

@@ -0,0 +1,85 @@
+"""add openrouter secrets and persisted evaluation run cost
+
+Revision ID: 34e1f2c7a9bd
+Revises: e3f8c1d4a2b9
+Create Date: 2026-05-05 12:30:00.000000
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+from alembic import op
+
+revision: str = "34e1f2c7a9bd"
+down_revision: Union[str, None] = "e3f8c1d4a2b9"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.create_table(
+        "agent_openrouter_secrets",
+        sa.Column("agent_id", sa.UUID(), nullable=False),
+        sa.Column("runtime_api_key_ciphertext", postgresql.BYTEA(), nullable=False),
+        sa.Column("management_api_key_ciphertext", postgresql.BYTEA(), nullable=False),
+        sa.Column("workspace_id", sa.Text(), nullable=False),
+        sa.Column("api_key_label", sa.Text(), nullable=False),
+        sa.Column("api_key_creator_user_id", sa.Text(), nullable=False),
+        sa.Column("validated_at", sa.TIMESTAMP(timezone=True), nullable=False),
+        sa.Column(
+            "created_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("NOW()"),
+            nullable=False,
+        ),
+        sa.Column(
+            "updated_at",
+            sa.TIMESTAMP(timezone=True),
+            server_default=sa.text("NOW()"),
+            nullable=False,
+        ),
+        sa.ForeignKeyConstraint(["agent_id"], ["agents.agent_id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("agent_id"),
+    )
+
+    op.add_column("evaluation_runs", sa.Column("cost_usd", sa.Double(), nullable=True))
+
+    op.execute("DROP VIEW IF EXISTS evaluation_runs_with_cost;")
+    op.execute(
+        """
+        CREATE VIEW evaluation_runs_with_cost AS
+        SELECT
+            er.*,
+            COALESCE(er.cost_usd, 0) AS total_cost_usd,
+            COALESCE(SUM(i.num_input_tokens), 0) AS total_input_tokens,
+            COALESCE(SUM(i.num_output_tokens), 0) AS total_output_tokens,
+            COUNT(*) as num_inferences
+        FROM evaluation_runs er
+        LEFT JOIN inferences i ON er.evaluation_run_id = i.evaluation_run_id
+        GROUP BY er.evaluation_run_id;
+    """
+    )
+
+
+def downgrade() -> None:
+    op.execute("DROP VIEW IF EXISTS evaluation_runs_with_cost;")
+    op.execute(
+        """
+        CREATE VIEW evaluation_runs_with_cost AS
+        SELECT
+            er.*,
+            COALESCE(SUM(i.cost_usd), 0) AS total_cost_usd,
+            COALESCE(SUM(i.num_input_tokens), 0) AS total_input_tokens,
+            COALESCE(SUM(i.num_output_tokens), 0) AS total_output_tokens,
+            COUNT(*) as num_inferences
+        FROM evaluation_runs er
+        LEFT JOIN inferences i ON er.evaluation_run_id = i.evaluation_run_id
+        GROUP BY er.evaluation_run_id;
+    """
+    )
+
+    op.drop_column("evaluation_runs", "cost_usd")
+    op.drop_table("agent_openrouter_secrets")

api/.env.example

@@ -23,6 +23,9 @@ AWS_ACCESS_KEY_ID=
 AWS_SECRET_ACCESS_KEY=
 AWS_REGION=
 
+# python3 -c "import os,base64; print(base64.b64encode(os.urandom(32)).decode())"
+RIDGES_AGENT_KEY_ENCRYPTION_KEY=
+
 
 
 S3_BUCKET_NAME=

api/config.py

@@ -80,6 +80,13 @@
 if not AWS_REGION:
     logger.fatal("AWS_REGION is not set in .env")
 
+RIDGES_AGENT_KEY_ENCRYPTION_KEY = os.getenv("RIDGES_AGENT_KEY_ENCRYPTION_KEY")
+if not RIDGES_AGENT_KEY_ENCRYPTION_KEY:
+    logger.fatal(
+        "RIDGES_AGENT_KEY_ENCRYPTION_KEY is not set in .env; miner OpenRouter secret encryption/decryption "
+        "will be unavailable until it is configured."
+    )
+
 
 # Load S3 configuration
 S3_BUCKET_NAME = os.getenv("S3_BUCKET_NAME")

api/endpoints/validator.py

@@ -37,9 +37,11 @@
 from models.evaluation import Evaluation, EvaluationStatus
 from models.evaluation_run import EvaluationRunLogType, EvaluationRunStatus
 from models.harbor_task import read_execution_spec_metadata
+from models.openrouter import OpenRouterRuntimeConfig
 from queries.agent import (
     get_agent_by_id,
     get_next_agent_id_awaiting_evaluation_for_validator_hotkey,
+    get_openrouter_secrets_for_agent_id,
     get_top_agents,
     update_agent_status,
 )
@@ -57,6 +59,7 @@
     get_evaluation_run_by_id,
     update_evaluation_run_by_id,
 )
+from utils.agent_secrets import AgentKeyDecryptError, AgentKeyEncryptionConfigError, sha256_hex
 from utils.bittensor import validate_signed_timestamp
 from utils.debug_lock import DebugLock
 from utils.git import COMMIT_HASH
@@ -377,6 +380,28 @@ async def validator_request_evaluation(
                 logger.error(f"Failed to download agent code for {agent_id}: {exc}")
                 return None
 
+            openrouter_config = None
+
+            try:
+                openrouter_secrets = await get_openrouter_secrets_for_agent_id(agent_id)
+            except AgentKeyEncryptionConfigError as exc:
+                logger.error(
+                    f"OpenRouter secret decryption is unavailable for agent {agent_id} due to encryption "
+                    f"configuration: {exc}"
+                )
+                return None
+            except AgentKeyDecryptError as exc:
+                logger.error(f"OpenRouter secret unreadable for agent {agent_id}: {exc}")
+                return None
+            else:
+                if openrouter_secrets is not None:
+                    openrouter_config = OpenRouterRuntimeConfig(
+                        api_key=openrouter_secrets.runtime_api_key,
+                        management_key=openrouter_secrets.management_api_key,
+                        workspace_id=openrouter_secrets.workspace_id,
+                        expected_api_key_sha256=sha256_hex(openrouter_secrets.runtime_api_key),
+                    )
+
             # Create a new evaluation and evaluation runs for this agent & validator
             evaluation_bundle = await create_new_evaluation_and_evaluation_runs(agent_id, validator.hotkey)
             if evaluation_bundle is None:
@@ -427,6 +452,7 @@ async def validator_request_evaluation(
         agent_code=agent_code,
         evaluation_runs=response_runs,
         artifact_upload_urls=artifact_upload_urls,
+        openrouter_config=openrouter_config,
     )
 
 
@@ -681,6 +707,7 @@ async def validator_update_evaluation_run(
                 evaluation_run.patch = request.patch
             evaluation_run.verifier_reward = request.verifier_reward
             evaluation_run.test_results = request.test_results
+            evaluation_run.cost_usd = request.cost_usd
             evaluation_run.finished_or_errored_at = now
 
             if not agent_logs_exist:
@@ -719,6 +746,7 @@ async def validator_update_evaluation_run(
             evaluation_run.status = EvaluationRunStatus.error
             evaluation_run.error_code = request.error_code
             evaluation_run.error_message = request.error_message
+            evaluation_run.cost_usd = request.cost_usd
             evaluation_run.finished_or_errored_at = datetime.now(timezone.utc)
 
             # Create evaluation run logs

api/endpoints/validator_models.py

@@ -7,6 +7,7 @@
 from models.agent import Agent
 from models.evaluation import Evaluation
 from models.evaluation_run import EvaluationRunStatus
+from models.openrouter import OpenRouterRuntimeConfig
 from models.problem import ProblemTestResult
 from utils.system_metrics import SystemMetrics
 
@@ -54,6 +55,7 @@ class ValidatorRequestEvaluationResponse(BaseModel):
     agent_code: str
     evaluation_runs: List[ValidatorRequestEvaluationResponseEvaluationRun]
     artifact_upload_urls: dict[str, str] = Field(default_factory=dict)
+    openrouter_config: OpenRouterRuntimeConfig | None = None
 
 
 class ValidatorTaskDownloadUrlRequest(BaseModel):
@@ -85,6 +87,7 @@ class ValidatorUpdateEvaluationRunRequest(BaseModel):
 
     error_code: Optional[int] = None
     error_message: Optional[str] = None
+    cost_usd: Optional[float] = None
 
 
 class ValidatorUpdateEvaluationRunResponse(BaseModel):

api/src/endpoints/upload.py

@@ -10,6 +10,7 @@
 
 import utils.logger as logger
 from api import config
+from api.src.utils.openrouter_validation import validate_openrouter_keys
 from api.src.utils.request_cache import hourly_cache
 from api.src.utils.upload_agent_helpers import (
     check_agent_banned,
@@ -28,6 +29,7 @@
 )
 from queries.banned_hotkey import get_banned_hotkey
 from queries.payments import record_evaluation_payment, retrieve_payment_by_hash
+from utils.agent_secrets import encrypt_agent_secret
 from utils.coingecko import get_tao_price
 from utils.debug_lock import DebugLock
 
@@ -81,6 +83,10 @@ async def check_agent_post(
     signature: str = Form(..., description="Signature to verify the authenticity of the upload"),
     name: str = Form(..., description="Name of the agent"),
     payment_time: float = Form(..., description="Timestamp of the payment"),
+    openrouter_api_key: str = Form(..., description="OpenRouter API key for inference during evaluation"),
+    openrouter_management_key: str = Form(
+        ..., description="OpenRouter management key used to validate workspace privacy settings"
+    ),
 ) -> AgentUploadResponse:
     if config.DISALLOW_UPLOADS:
         raise HTTPException(status_code=503, detail=config.DISALLOW_UPLOADS_REASON)
@@ -102,6 +108,10 @@ async def check_agent_post(
             status_code=402,
             detail=f"Insufficient balance. You need {payment_cost.amount_rao} RAO to upload this agent. You have {miner_balance} RAO.",
         )
+    await validate_openrouter_keys(
+        openrouter_api_key=openrouter_api_key,
+        openrouter_management_key=openrouter_management_key,
+    )
     return AgentUploadResponse(status="success", message="Agent check successful")
 
 
@@ -130,6 +140,10 @@ async def post_agent(
     payment_block_hash: str = Form(..., description="Block hash in which payment was made"),
     payment_extrinsic_index: str = Form(..., description="Index in the block for payment extrinsic"),
     payment_time: float = Form(..., description="Timestamp of the payment"),
+    openrouter_api_key: str = Form(..., description="OpenRouter API key for inference during evaluation"),
+    openrouter_management_key: str = Form(
+        ..., description="OpenRouter management key used to validate workspace privacy settings"
+    ),
 ) -> AgentUploadResponse:
     """
     Upload a new agent version for evaluation
@@ -239,6 +253,11 @@ async def post_agent(
                     detail=f"Destination does not match. The payment should be sent to {config.UPLOAD_SEND_ADDRESS}",
                 )
 
+        validated_openrouter_keys = await validate_openrouter_keys(
+            openrouter_api_key=openrouter_api_key,
+            openrouter_management_key=openrouter_management_key,
+        )
+
         agent_text = (await agent_file.read()).decode("utf-8")
 
         hotkey_lock = await get_hotkey_lock(miner_hotkey)
@@ -251,6 +270,9 @@ async def post_agent(
 
             if prod and latest_agent_created_at_in_latest_set_id:
                 check_rate_limit(latest_agent_created_at_in_latest_set_id)
+
+            encrypted_openrouter_api_key = encrypt_agent_secret(validated_openrouter_keys.runtime_api_key)
+            encrypted_openrouter_management_key = encrypt_agent_secret(validated_openrouter_keys.management_api_key)
             agent = Agent(
                 agent_id=uuid.uuid4(),
                 miner_hotkey=miner_hotkey,
@@ -260,7 +282,16 @@ async def post_agent(
                 status=AgentStatus.screening_1,
                 ip_address=request.client.host if request.client else None,
             )
-            await create_agent(agent, agent_text)
+            await create_agent(
+                agent,
+                agent_text,
+                runtime_openrouter_api_key_ciphertext=encrypted_openrouter_api_key,
+                management_openrouter_api_key_ciphertext=encrypted_openrouter_management_key,
+                openrouter_workspace_id=validated_openrouter_keys.workspace_id,
+                openrouter_api_key_label=validated_openrouter_keys.api_key_label,
+                openrouter_api_key_creator_user_id=validated_openrouter_keys.api_key_creator_user_id,
+                openrouter_validated_at=validated_openrouter_keys.validated_at,
+            )
 
         if prod:
             await record_evaluation_payment(