Skip to content

Commit f0f36a9

Browse files
committed
fix(ci): normalize train_series wall_s for replica hash
After telemetry-rt, CPU re-exec replica agreement failed because metrics.train_series wall_s (and its sha256 pointers) differed per run. Zero wall_s on normalize and realign digests so two honest replicas share one manifest_sha256 again.
1 parent 5368962 commit f0f36a9

2 files changed

Lines changed: 56 additions & 4 deletions

File tree

src/prism_challenge/evaluator/cpu_test_mode.py

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -147,19 +147,44 @@ def configure_cpu_reexec_test_mode(settings: Any) -> Path:
147147

148148

149149
def normalize_manifest_for_replication(manifest: dict[str, Any]) -> dict[str, Any]:
150-
"""Return a deep copy with volatile ``compute`` timing/memory fields fixed to a constant.
150+
"""Return a deep copy with volatile host-timing fields fixed to a constant.
151151
152-
Two honest CPU re-execs of the SAME submission differ ONLY in these host-timing fields; fixing
153-
them makes the canonical manifest bytes (and thus :func:`compute_manifest_sha256`) identical,
154-
which is what lets two independent worker replicas agree and be accepted (rather than disputed).
152+
Two honest CPU re-execs of the SAME submission differ in host timing: the
153+
classical ``compute`` wall/RSS fields plus (after telemetry-rt) per-step
154+
``metrics.train_series.points[*].wall_s`` and the linked
155+
``train_series_sha256`` digests. Fix timing so canonical manifest bytes
156+
(and thus :func:`compute_manifest_sha256`) match across replicas.
155157
"""
156158

159+
from .train_series import train_series_sha256
160+
157161
normalized = copy.deepcopy(manifest)
158162
compute = normalized.get("compute")
159163
if isinstance(compute, dict):
160164
for field in VOLATILE_COMPUTE_FIELDS:
161165
if field in compute:
162166
compute[field] = 0
167+
168+
metrics = normalized.get("metrics")
169+
series_digest: str | None = None
170+
if isinstance(metrics, dict):
171+
raw_series = metrics.get("train_series")
172+
if isinstance(raw_series, dict):
173+
points = raw_series.get("points")
174+
if isinstance(points, list):
175+
for point in points:
176+
if isinstance(point, dict) and "wall_s" in point:
177+
point["wall_s"] = 0.0
178+
series_digest = train_series_sha256(raw_series)
179+
if "train_series_sha256" in metrics:
180+
metrics["train_series_sha256"] = series_digest
181+
if "sha256" in raw_series:
182+
raw_series["sha256"] = series_digest
183+
184+
artifacts = normalized.get("artifacts")
185+
if isinstance(artifacts, dict) and "train_series_sha256" in artifacts:
186+
artifacts["train_series_sha256"] = series_digest or ("0" * 64)
187+
163188
return normalized
164189

165190

tests/test_cpu_test_mode.py

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,33 @@ def test_normalize_drops_volatile_compute_fields():
9494
assert manifest["compute"]["wall_clock_seconds"] == 12.5
9595

9696

97+
def test_normalize_zeros_train_series_wall_and_realigns_digest():
98+
"""Telemetry wall_s must not keep two honest replicas from sharing one hash."""
99+
series = {
100+
"schema_version": "prism_train_series.v1",
101+
"points": [
102+
{"i": 0, "wall_s": 0.0123, "train_ce_nats": 1.0, "tokens_seen": 8},
103+
{"i": 1, "wall_s": 0.0456, "train_ce_nats": 0.9, "tokens_seen": 16},
104+
],
105+
}
106+
manifest = {
107+
"metrics": {
108+
"train_series": series,
109+
"train_series_sha256": "deadbeef",
110+
},
111+
"artifacts": {"train_series_sha256": "deadbeef"},
112+
}
113+
normalized = normalize_manifest_for_replication(manifest)
114+
points = normalized["metrics"]["train_series"]["points"]
115+
assert all(point["wall_s"] == 0.0 for point in points)
116+
assert points[0]["train_ce_nats"] == 1.0 # non-timing preserved
117+
digest = normalized["metrics"]["train_series_sha256"]
118+
assert digest != "deadbeef" and len(digest) == 64
119+
assert normalized["artifacts"]["train_series_sha256"] == digest
120+
# Input not mutated.
121+
assert series["points"][0]["wall_s"] == 0.0123
122+
123+
97124
def test_two_replicas_of_same_submission_agree_on_hash(tmp_path):
98125
# Two independent worker hosts (distinct artifact roots) re-exec the SAME submission; the
99126
# normalized manifest hash MUST match so the base worker plane accepts (not disputes) them.

0 commit comments

Comments
 (0)