Skip to content

Commit 2f1bd55

Browse files
committed
Fix flaky CLI job status race under combined worker
1 parent 3c1d2d7 commit 2f1bd55

1 file changed

Lines changed: 49 additions & 2 deletions

File tree

tests/cli/test_cli_marketplace_nodes_jobs.py

Lines changed: 49 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -531,7 +531,7 @@ def test_nodes_register_heartbeat_fabric_scan(live_api: dict[str, Any]) -> None:
531531

532532

533533
def test_jobs_submit_status_list_cancel(live_api: dict[str, Any], tmp_path: Path) -> None:
534-
"""VAL-CLI-008: submit returns id; status matches GET; list/filter; cancel terminals."""
534+
"""VAL-CLI-008: submit returns id; status/API share id + lifecycle status; list/cancel."""
535535

536536
base = live_api["base_url"]
537537
spec_path = tmp_path / "job.json"
@@ -595,9 +595,56 @@ def test_jobs_submit_status_list_cancel(live_api: dict[str, Any], tmp_path: Path
595595
status_body = _json_blob(status.output)
596596
assert status_body.get("id") == job_id or status_body.get("job_id") == job_id
597597

598+
# Combined worker may advance between CLI poll and API GET; require id match
599+
# and a known lifecycle status rather than exact freeze of a single value.
600+
known_statuses = {
601+
"submitted",
602+
"admitted",
603+
"placing",
604+
"provisioning",
605+
"running",
606+
"collecting",
607+
"scoring",
608+
"succeeded",
609+
"failed",
610+
"cancelled",
611+
"timeout",
612+
}
613+
cli_status = status_body.get("status")
614+
assert cli_status in known_statuses, status_body
615+
598616
api = httpx.get(f"{base}/v1/jobs/{job_id}", timeout=5.0)
599617
assert api.status_code == 200
600-
assert api.json().get("status") == status_body.get("status")
618+
api_body = api.json()
619+
assert api_body.get("id") == job_id or api_body.get("job_id") == job_id
620+
api_status = api_body.get("status")
621+
assert api_status in known_statuses, api_body
622+
# If worker advanced between the two reads, only allow monotonic lifecycle
623+
# progression (same status or a later-known stage). Exact equality is not
624+
# required under combined worker.
625+
if cli_status != api_status:
626+
order = [
627+
"submitted",
628+
"admitted",
629+
"placing",
630+
"provisioning",
631+
"running",
632+
"collecting",
633+
"scoring",
634+
"succeeded",
635+
"failed",
636+
"cancelled",
637+
"timeout",
638+
]
639+
# Terminal branches may skip scoring; allow any known→known pair that is
640+
# not a regression to a strictly earlier non-terminal index when both
641+
# are non-terminal. Terminal statuses are always accepted as "advanced".
642+
terminals = {"succeeded", "failed", "cancelled", "timeout"}
643+
if api_status not in terminals and cli_status not in terminals:
644+
assert order.index(api_status) >= order.index(cli_status), (
645+
cli_status,
646+
api_status,
647+
)
601648

602649
listed = _invoke(
603650
[

0 commit comments

Comments
 (0)