make completionparams optional and clean up tests

Author: xzrderek

eval_protocol/models.py

@@ -181,7 +181,7 @@ class InputMetadata(BaseModel):
     model_config = ConfigDict(extra="allow")
 
     row_id: Optional[str] = Field(None, description="Unique string to ID the row")
-    completion_params: CompletionParams = Field(..., description="Completion endpoint parameters used")
+    completion_params: Optional[CompletionParams] = Field(None, description="Completion endpoint parameters used")
     dataset_info: Optional[Dict[str, Any]] = Field(
         None, description="Dataset row details: seed, system_prompt, environment_context, etc"
     )

tests/pytest/test_frozen_lake.py

@@ -19,20 +19,19 @@ def frozen_lake_to_evaluation_row(data: List[Dict[str, Any]]) -> List[Evaluation
     """
     rows = []
 
-    for entry in data:
-        row = EvaluationRow(
-            messages=[Message(role="system", content=entry.get("system_prompt", ""))],
+    for row in data:
+        eval_row = EvaluationRow(
+            messages=[Message(role="system", content=row["system_prompt"])],
             input_metadata=InputMetadata(
-                row_id=entry.get("id"),
-                completion_params=CompletionParams(model="placeholder"), # This gets populated by the rollout processor
+                row_id=row["id"],
                 dataset_info={
-                    "environment_context": entry.get("environment_context", {}),
-                    "user_prompt_template": entry.get("user_prompt_template", ""),
+                    "environment_context": row["environment_context"],
+                    "user_prompt_template": row["user_prompt_template"],
                 }
             )
         )
 
-        rows.append(row)
+        rows.append(eval_row)
 
     return rows
 

tests/pytest/test_lunar_lander.py

@@ -18,20 +18,19 @@ def lunar_lander_to_evaluation_row(data: List[Dict[str, Any]]) -> List[Evaluatio
     """
     rows = []
 
-    for entry in data:
-        row = EvaluationRow(
-            messages=[Message(role="system", content=entry.get("system_prompt", ""))],
+    for row in data:
+        eval_row = EvaluationRow(
+            messages=[Message(role="system", content=row["system_prompt"])],
             input_metadata=InputMetadata(
-                row_id=entry.get("id"),
-                completion_params=CompletionParams(model="placeholder"), # This gets populated by the rollout processor
+                row_id=row["id"],
                 dataset_info={
-                    "environment_context": entry.get("environment_context", {}),
-                    "user_prompt_template": entry.get("user_prompt_template", ""),
+                    "environment_context": row["environment_context"],
+                    "user_prompt_template": row["user_prompt_template"],
                 }
             )
         )
 
-        rows.append(row)
+        rows.append(eval_row)
 
     return rows
 

tests/pytest/test_tau_bench_airline.py

@@ -35,35 +35,28 @@ def tau_bench_airline_to_evaluation_row(data: List[Dict[str, Any]]) -> List[Eval
     rows = []
     test_dir = Path(__file__).parent.parent.parent / "examples" / "tau2_mcp" / "tests"
 
-    for entry in data:
-        # Load system prompt from file so we can change it in one place
-        domain = entry["environment_context"]["domain"]
-        prompt_file = test_dir / f"system_prompts/{domain}_agent_system_prompt.md"
-        
-        with open(prompt_file, "r") as f:
-            system_prompt = f.read().strip()
-        
-        messages = [Message(role="system", content=system_prompt)]
-        
-        evaluation_criteria = entry.get("evaluation_criteria", {})
-        user_simulation = entry.get("user_simulation", {})
-        user_prompt_template = entry.get("user_prompt_template", "")
-        
-        row = EvaluationRow(
-            messages=messages,
+    # Load system prompt from file so we can change it in one place
+    domain = data[0]["environment_context"]["domain"]
+    prompt_file = test_dir / f"system_prompts/{domain}_agent_system_prompt.md"
+    
+    with open(prompt_file, "r") as f:
+        system_prompt = f.read().strip()
+    
+    for row in data:
+        eval_row = EvaluationRow(
+            messages=[Message(role="system", content=system_prompt)],
             input_metadata=InputMetadata(
-                row_id=entry.get("id"),
-                completion_params=CompletionParams(model="placeholder"), # This gets populated by the rollout processor
+                row_id=row["id"],
                 dataset_info={
-                    "environment_context": entry.get("environment_context"),
-                    "user_simulation": user_simulation,
-                    "evaluation_criteria": evaluation_criteria,
-                    "user_prompt_template": user_prompt_template,
+                    "environment_context": row["environment_context"],
+                    "user_simulation": row["user_simulation"],
+                    "evaluation_criteria": row["evaluation_criteria"],
+                    "user_prompt_template": row["user_prompt_template"],
                 }
             ),
         )
 
-        rows.append(row)
+        rows.append(eval_row)
 
     return rows
 
@@ -94,7 +87,7 @@ def save_single_trajectory(trajectory_record: Dict, row_id: str, output_dir: str
     rollout_input_params=[{"temperature": 0.0, "max_tokens": 4096}],
     rollout_processor=default_mcp_gym_rollout_processor,
     threshold_of_success=0.4,
-    num_runs=4,
+    num_runs=1,
     mode="pointwise",
     max_concurrent_rollouts=32,
     server_script_path="examples/tau2_mcp/server.py",