diff --git a/hud/cli/eval.py b/hud/cli/eval.py index 26db25ed..c2c6c906 100644 --- a/hud/cli/eval.py +++ b/hud/cli/eval.py @@ -714,7 +714,7 @@ async def _run_evaluation(cfg: EvalConfig) -> tuple[list[Any], list[Any]]: for task_obj, task_version_id in zip(tasks_to_create, ids, strict=False): task_obj.id = task_version_id - await submit_rollouts( + trace_ids = await submit_rollouts( tasks=tasks, job_id=job_id, agent_type=cfg.agent_type, @@ -724,6 +724,9 @@ async def _run_evaluation(cfg: EvalConfig) -> tuple[list[Any], list[Any]]: use_byok=cfg.byok, ) + if not trace_ids: + raise ValueError("No tasks were accepted for execution. Check errors above.") + hud_console.success(f"Tasks submitted. View at: https://hud.ai/jobs/{job_id}") return [], tasks diff --git a/hud/eval/manager.py b/hud/eval/manager.py index 1a4635cb..a3670db2 100644 --- a/hud/eval/manager.py +++ b/hud/eval/manager.py @@ -75,7 +75,11 @@ async def _send_job_enter( tasks: list[dict[str, Any]] | None = None, hud_eval_config: dict[str, Any] | None = None, ) -> list[str] | None: - """Send job enter payload (async request before traces start).""" + """Send job enter payload (async request before traces start). + + Returns task_version_ids on success, None if telemetry is disabled. + Raises on any failure (network, bad response, etc). + """ import httpx from hud.eval.types import JobEnterPayload @@ -94,25 +98,20 @@ async def _send_job_enter( hud_eval_config=hud_eval_config, ) - try: - async with httpx.AsyncClient(timeout=10.0) as client: - resp = await client.post( - f"{settings.hud_api_url}/trace/job/{job_id}/enter", - json=payload.model_dump(exclude_none=True), - headers={"Authorization": f"Bearer {api_key}"}, - ) - if resp.is_success: - try: - data = resp.json() - except Exception: - return None - if isinstance(data, dict): - ids = data.get("task_version_ids") - if isinstance(ids, list) and all(isinstance(x, str) for x in ids): - return ids - except Exception as e: - logger.warning("Failed to send job enter: %s", e) - return None + async with httpx.AsyncClient(timeout=10.0) as client: + resp = await client.post( + f"{settings.hud_api_url}/trace/job/{job_id}/enter", + json=payload.model_dump(exclude_none=True), + headers={"Authorization": f"Bearer {api_key}"}, + ) + + resp.raise_for_status() + data = resp.json() + if isinstance(data, dict): + ids = data.get("task_version_ids") + if isinstance(ids, list) and all(isinstance(x, str) for x in ids): + return ids + raise ValueError(f"Job registration failed: unexpected response: {data}") @asynccontextmanager