fix: improve separation of judge and test profiles (#6)

Author: zach

mcpx_eval/__main__.py

@@ -208,7 +208,7 @@ async def run():
         "--profile",
         "-p",
         default=None,
-        help="Profile to use for all models",
+        help="Profile to use for judge model",
     )
 
     test_parser.add_argument("--prompt", help="Test prompt")
@@ -318,17 +318,8 @@ async def run():
 
         if hasattr(args, "config") and args.config is not None:
             test = Test.load(args.config)
-            if args.profile:
-                test.profile = args.profile
             for model in args.model:
-                if args.profile:
-                    if "/" in model:
-                        a, _ = model.split("/", maxsplit=1)
-                        test.models.append(f"{a}/{args.profile}")
-                    else:
-                        test.models.append(f"{model}/{args.profile}")
-                else:
-                    test.models.append(model)
+                test.models.append(model)
             if args.name is None or args.name == "":
                 if test.name is not None:
                     name = test.name
@@ -362,7 +353,7 @@ async def run():
 
         judge = Judge(
             models=test.models,
-            profile=test.profile,
+            profile=args.profile,
             db=db,
             judge_model=args.judge_model,
             ignore_tools=test.ignore_tools,

mcpx_eval/judge.py

@@ -264,7 +264,7 @@ async def run(
             result["duration_in_seconds"] = f"{duration_seconds}s"
             result["number_of_tools_used"] = str(tool_analysis.total_tool_calls)
 
-            logger.info(f"Analyzing results of {model.slug}")
+            logger.info(f"Analyzing results of {model.slug} with profile={self.profile}")
             agent = Chat(
                 client=mcp_run.Client(
                     config=mcp_run.ClientConfig(profile=self.profile)

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "mcpx-eval"
-version = "0.1.3"
+version = "0.1.4"
 description = "Open ended tool use evaluation framework"
 readme = "README.md"
 requires-python = ">=3.12"