From 49e3f68e6e011c9d96855fced0a2eacd9aef75a0 Mon Sep 17 00:00:00 2001 From: Pigbibi <20649888+Pigbibi@users.noreply.github.com> Date: Thu, 16 Apr 2026 21:44:47 +0800 Subject: [PATCH] Add crypto artifact manifest contract --- README.md | 15 ++-- README.zh-CN.md | 25 +++++- docs/integration_contract.md | 43 ++++++++- docs/operator_runbook.md | 12 +-- docs/validation_status.md | 3 + scripts/build_live_pool.py | 4 +- scripts/run_release_status_summary.py | 7 ++ scripts/validate_release_contract.py | 6 ++ src/export.py | 102 +++++++++++++++++++++ src/pipeline.py | 21 ++++- src/publish.py | 14 +++ src/release_contract.py | 125 +++++++++++++++++++++++++- tests/test_build_live_pool_smoke.py | 4 + tests/test_release_contract.py | 75 ++++++++++++++++ tests/test_release_status_summary.py | 70 ++++++++++++++- 15 files changed, 507 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 769fa7e..1942d1e 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,9 @@ Core upstream artifacts: 2. `data/output/latest_ranking.csv` 3. `data/output/live_pool.json` 4. `data/output/live_pool_legacy.json` -5. `data/output/release_manifest.json` -6. `data/output/release_status_summary.json` +5. `data/output/artifact_manifest.json` +6. `data/output/release_manifest.json` +7. `data/output/release_status_summary.json` ## Upstream Boundary @@ -28,7 +29,7 @@ Core upstream artifacts: In practice, that means: -- upstream publishes and explains `latest_universe`, `latest_ranking`, `live_pool`, `release_manifest`, and release-status summaries +- upstream publishes and explains `latest_universe`, `latest_ranking`, `live_pool`, `artifact_manifest`, `release_manifest`, and release-status summaries - downstream consumes the official live-pool contract plus publish metadata and emits only runtime/execution status - research CSVs, shadow-track diagnostics, and monthly review outputs stay upstream and are not part of the minimum downstream execution contract @@ -40,7 +41,7 @@ The repository is now intentionally split into two tracks: - data source: `Binance Spot only` - universe mode: `core_major` - publish cadence: `monthly` - - default outputs: `latest_universe.json`, `latest_ranking.csv`, `live_pool.json`, `live_pool_legacy.json` + - default outputs: `latest_universe.json`, `latest_ranking.csv`, `live_pool.json`, `live_pool_legacy.json`, `artifact_manifest.json` - `Experimental external-data track` - used for research, comparison, and validation only - not enabled by default @@ -245,10 +246,10 @@ Validate the local production artifacts before publish or rollback: .venv/bin/python scripts/validate_release_contract.py --mode core_major --expected-pool-size 5 ``` -Require a generated manifest as part of the check: +Require generated release and artifact manifests as part of the production check: ```bash -.venv/bin/python scripts/validate_release_contract.py --mode core_major --expected-pool-size 5 --require-manifest +.venv/bin/python scripts/validate_release_contract.py --mode core_major --expected-pool-size 5 --require-manifest --require-artifact-manifest ``` Operator workflow details, rollback steps, and research-vs-production boundaries are documented in `docs/operator_runbook.md`. @@ -370,7 +371,7 @@ Downstream consumers should rely on these core fields in `data/output/live_pool. - `symbol_map` - `source_project` -Publish-time pointer fields such as `storage_prefix`, `current_prefix`, `live_pool_uri`, `live_pool_legacy_uri`, `latest_universe_uri`, and `latest_ranking_uri` are stable when present in the published Firestore payload, but they are release/distribution metadata rather than research features. +Publish-time pointer fields such as `storage_prefix`, `current_prefix`, `live_pool_uri`, `live_pool_legacy_uri`, `artifact_manifest_uri`, `latest_universe_uri`, and `latest_ranking_uri` are stable when present in the published Firestore payload, but they are release/distribution metadata rather than research features. Optional additive research extensions: diff --git a/README.zh-CN.md b/README.zh-CN.md index ef6d0db..4c86cd1 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -9,6 +9,10 @@ 1. `data/output/latest_universe.json` 2. `data/output/latest_ranking.csv` 3. `data/output/live_pool.json` +4. `data/output/live_pool_legacy.json` +5. `data/output/artifact_manifest.json` +6. `data/output/release_manifest.json` +7. `data/output/release_status_summary.json` ## 当前状态 @@ -18,7 +22,7 @@ - 数据源：仅 `Binance Spot` - universe mode：`core_major` - 发布频率：`monthly` - - 默认输出：`latest_universe.json`、`latest_ranking.csv`、`live_pool.json`、`live_pool_legacy.json` + - 默认输出：`latest_universe.json`、`latest_ranking.csv`、`live_pool.json`、`live_pool_legacy.json`、`artifact_manifest.json` - `Experimental external-data track` - 仅用于研究、比较和验证 - 默认不启用 @@ -147,6 +151,20 @@ pip install -r requirements.txt - 输出设置 - GCS / Firestore 发布设置 +## 发布契约检查 + +发布或回滚前，先校验本地生产产物： + +```bash +.venv/bin/python scripts/validate_release_contract.py --mode core_major --expected-pool-size 5 +``` + +生产发布链应同时要求 release manifest 和 profile-aware artifact manifest： + +```bash +.venv/bin/python scripts/validate_release_contract.py --mode core_major --expected-pool-size 5 --require-manifest --require-artifact-manifest +``` + ## 最小可运行流程 1. 下载历史数据 @@ -215,12 +233,15 @@ pip install -r requirements.txt - `data/output/live_pool_legacy.json` - Firestore summary document +`data/output/artifact_manifest.json` 是 profile-aware wrapper，负责声明 artifact contract version、主 artifact、相关文件路径和校验和；它不是 `live_pool.json` 的字段复制。 + 一些发布期辅助字段，例如： - `storage_prefix` - `current_prefix` - `live_pool_uri` - `live_pool_legacy_uri` +- `artifact_manifest_uri` - `latest_universe_uri` - `latest_ranking_uri` @@ -290,6 +311,7 @@ make monthly-shadow-build - official baseline - `data/output/live_pool.json` - `data/output/live_pool_legacy.json` + - `data/output/artifact_manifest.json` - `data/output/release_manifest.json` - shadow candidate tracks - `data/output/shadow_candidate_tracks/track_summary.csv` @@ -322,6 +344,7 @@ make monthly-build-telegram - 只发送简短的 monthly build/publish health summary - 使用已有的 `monthly_shadow_build_summary.json`、`live_pool.json`、`release_manifest.json`、`track_summary.csv` +- 生产发布链还会检查 `artifact_manifest.json`，但 Telegram 文本只展示摘要状态 - 如果 Telegram 凭证缺失，会跳过而不是报错中断 - 不改变 monthly build 行为，也不是 review 包生成器 diff --git a/docs/integration_contract.md b/docs/integration_contract.md index 1424daf..30edfa0 100644 --- a/docs/integration_contract.md +++ b/docs/integration_contract.md @@ -10,6 +10,41 @@ The upstream project publishes a monthly `core_major` live pool and exposes it t ## Canonical Downstream Files +### `artifact_manifest.json` + +This profile-aware manifest is the canonical contract wrapper for downstream +runtimes. It lets the execution platform validate the artifact family without +binding to `CryptoLeaderRotation` internals. + +Required stable fields: + +- `manifest_type = strategy_artifact` +- `contract_version = crypto_leader_rotation.live_pool.v1` +- `strategy_profile = crypto_leader_rotation` +- `artifact_type = live_pool` +- `artifact_name = crypto_leader_rotation_live_pool` +- `as_of_date` +- `snapshot_as_of` +- `version` +- `mode` +- `symbol_count` +- `symbols` +- `source_project` +- `generated_at` +- `primary_artifact = live_pool` +- `artifacts` + +The `artifacts` mapping includes relative file paths and SHA-256 checksums for: + +- `latest_universe` +- `latest_ranking` +- `live_pool` +- `live_pool_legacy` + +Downstream platforms should treat this manifest as the strategy artifact +contract and keep legacy `live_pool_legacy.json` parsing as a compatibility +path, not as the only contract shape. + ### `live_pool_legacy.json` This is the most convenient file for older downstream scripts that expect a direct symbol mapping. @@ -123,9 +158,12 @@ Payload example: "current_prefix": "gs://example-bucket/crypto-leader-rotation/current", "live_pool_legacy_uri": "gs://example-bucket/crypto-leader-rotation/current/live_pool_legacy.json", "live_pool_uri": "gs://example-bucket/crypto-leader-rotation/current/live_pool.json", + "artifact_manifest_uri": "gs://example-bucket/crypto-leader-rotation/current/artifact_manifest.json", "latest_universe_uri": "gs://example-bucket/crypto-leader-rotation/current/latest_universe.json", "latest_ranking_uri": "gs://example-bucket/crypto-leader-rotation/current/latest_ranking.csv", "versioned_live_pool_legacy_uri": "gs://example-bucket/crypto-leader-rotation/releases/2026-03-13-core_major/live_pool_legacy.json", + "versioned_artifact_manifest_uri": "gs://example-bucket/crypto-leader-rotation/releases/2026-03-13-core_major/artifact_manifest.json", + "artifact_contract_version": "crypto_leader_rotation.live_pool.v1", "generated_at": "2026-03-13T13:00:00+00:00", "source_project": "crypto-leader-rotation" } @@ -136,8 +174,9 @@ The Firestore document intentionally excludes the full ranking CSV. Downstream r Stable vs additive fields: - stable core fields: `as_of_date`, `version`, `mode`, `pool_size`, `symbols`, `symbol_map`, `source_project` -- publish-only pointer fields: `storage_prefix`, `current_prefix`, `live_pool_uri`, `live_pool_legacy_uri`, `latest_universe_uri`, `latest_ranking_uri`, `versioned_live_pool_legacy_uri` +- publish-only pointer fields: `storage_prefix`, `current_prefix`, `live_pool_uri`, `live_pool_legacy_uri`, `artifact_manifest_uri`, `latest_universe_uri`, `latest_ranking_uri`, `versioned_live_pool_legacy_uri`, `versioned_artifact_manifest_uri` - additive observability field: `generated_at` +- artifact wrapper field: `artifact_contract_version` ## GCS Path Layout @@ -148,6 +187,7 @@ gs:///crypto-leader-rotation/releases//latest_universe. gs:///crypto-leader-rotation/releases//latest_ranking.csv gs:///crypto-leader-rotation/releases//live_pool.json gs:///crypto-leader-rotation/releases//live_pool_legacy.json +gs:///crypto-leader-rotation/releases//artifact_manifest.json ``` Current pointers: @@ -157,6 +197,7 @@ gs:///crypto-leader-rotation/current/latest_universe.json gs:///crypto-leader-rotation/current/latest_ranking.csv gs:///crypto-leader-rotation/current/live_pool.json gs:///crypto-leader-rotation/current/live_pool_legacy.json +gs:///crypto-leader-rotation/current/artifact_manifest.json ``` ## Local Shadow Release History diff --git a/docs/operator_runbook.md b/docs/operator_runbook.md index 31075be..1363994 100644 --- a/docs/operator_runbook.md +++ b/docs/operator_runbook.md @@ -10,6 +10,7 @@ Primary production outputs: - `data/output/latest_ranking.csv` - `data/output/live_pool.json` - `data/output/live_pool_legacy.json` +- `data/output/artifact_manifest.json` - `data/output/release_manifest.json` - `data/output/release_status_summary.json` - `data/output/release_status_summary.md` @@ -86,7 +87,7 @@ Boundary rules: 3. Run explicit contract validation: ```bash -.venv/bin/python scripts/validate_release_contract.py --mode core_major --expected-pool-size 5 +.venv/bin/python scripts/validate_release_contract.py --mode core_major --expected-pool-size 5 --require-manifest --require-artifact-manifest ``` 4. Run publish preflight without external writes: @@ -125,7 +126,7 @@ Boundary rules: - `requirements-lock.txt` is present and matches the intended release dependency set. - Local artifacts are non-empty and pass `scripts/validate_release_contract.py`. -- `live_pool.json`, `live_pool_legacy.json`, and `release_manifest.json` agree on `as_of_date`, `version`, `mode`, `pool_size`, and `source_project`. +- `live_pool.json`, `live_pool_legacy.json`, and `release_manifest.json` agree on `as_of_date`, `version`, `mode`, `pool_size`, and `source_project`; `artifact_manifest.json` agrees on `as_of_date`, `version`, `mode`, `source_project`, and `symbol_count`. - `GCP_PROJECT_ID`, `GCS_BUCKET`, `FIRESTORE_COLLECTION`, and `FIRESTORE_DOCUMENT` are set correctly for real publish. - Historical backfills use `--allow-stale` explicitly; do not silently publish stale artifacts. @@ -141,7 +142,7 @@ Symptoms: Actions: - Re-run `scripts/build_live_pool.py` -- Inspect `data/output/latest_universe.json`, `live_pool.json`, `live_pool_legacy.json`, and `latest_ranking.csv` +- Inspect `data/output/latest_universe.json`, `latest_ranking.csv`, `live_pool.json`, `live_pool_legacy.json`, and `artifact_manifest.json` - Confirm `pool_size`, `symbols`, `symbol_map`, `version`, and `source_project` are present and aligned ### Stale artifacts @@ -192,17 +193,18 @@ Use rollback only when the newest publish is clearly bad or malformed. - GCS `crypto-leader-rotation/releases//` - the last good `data/output/release_manifest.json` -2. Restore the four canonical artifacts from that version into `data/output/`: +2. Restore the five canonical artifacts from that version into `data/output/`: - `latest_universe.json` - `latest_ranking.csv` - `live_pool.json` - `live_pool_legacy.json` +- `artifact_manifest.json` 3. Validate the restored payload locally: ```bash -.venv/bin/python scripts/validate_release_contract.py --mode core_major --expected-pool-size 5 +.venv/bin/python scripts/validate_release_contract.py --mode core_major --expected-pool-size 5 --require-artifact-manifest ``` 4. Regenerate the manifest and verify publish preflight: diff --git a/docs/validation_status.md b/docs/validation_status.md index ea01654..45c26ac 100644 --- a/docs/validation_status.md +++ b/docs/validation_status.md @@ -15,6 +15,7 @@ The repository is now frozen around this production default: - `latest_ranking.csv` - `live_pool.json` - `live_pool_legacy.json` + - `artifact_manifest.json` This is the only path that should be treated as the formal production baseline. @@ -125,6 +126,7 @@ Validated in-repo: - `scripts/build_live_pool.py` produces the default `Production v1` live output - `scripts/publish_release.py --dry-run` builds a correct production release manifest +- `scripts/validate_release_contract.py --require-artifact-manifest` validates the profile-aware artifact contract - `scripts/write_release_heartbeat.py` writes a small logs-branch heartbeat file - GitHub Actions workflow YAML parses correctly - release versioning, GCS object keys, and Firestore payload layout are consistent @@ -136,6 +138,7 @@ Validated artifacts: - `data/output/latest_ranking.csv` - `data/output/live_pool.json` - `data/output/live_pool_legacy.json` +- `data/output/artifact_manifest.json` - `data/output/release_manifest.json` - `data/output/heartbeat/monthly/.json` diff --git a/scripts/build_live_pool.py b/scripts/build_live_pool.py index 6de71ca..cbbd10a 100644 --- a/scripts/build_live_pool.py +++ b/scripts/build_live_pool.py @@ -62,13 +62,15 @@ def main() -> None: ), expected_pool_size=int(config["export"]["live_pool_size"]), max_age_days=args.contract_max_age_days, + require_artifact_manifest=True, require_freshness=not bool(args.as_of_date or args.allow_stale), ) logger.info( - "Release contract validated | version=%s | pool_size=%s | manifest_present=%s", + "Release contract validated | version=%s | pool_size=%s | release_manifest_present=%s | artifact_manifest_present=%s", validation["version"], validation["pool_size"], validation["manifest_present"], + validation["artifact_manifest_present"], ) logger.info("Export payload:\n%s", result["live_payload"]) diff --git a/scripts/run_release_status_summary.py b/scripts/run_release_status_summary.py index 5d505f9..947a8f5 100644 --- a/scripts/run_release_status_summary.py +++ b/scripts/run_release_status_summary.py @@ -85,6 +85,7 @@ def build_release_status_payload( universe = load_json(root / "latest_universe.json") live_pool = load_json(root / "live_pool.json") manifest = load_optional_json(root / "release_manifest.json") or {} + artifact_manifest = load_optional_json(root / "artifact_manifest.json") or {} ranking = pd.read_csv(root / "latest_ranking.csv") validation = validate_release_outputs( @@ -94,6 +95,7 @@ def build_release_status_payload( expected_pool_size=live_pool.get("pool_size"), max_age_days=max_age_days, require_manifest=True, + require_artifact_manifest=True, require_freshness=require_freshness, ) @@ -134,6 +136,7 @@ def build_release_status_payload( "latest_universe_symbol_count": len(list(universe.get("symbols", []))), "latest_ranking_row_count": int(len(ranking)), "latest_ranking_selected_count": int(selected_mask.sum()) if not selected_mask.empty else 0, + "artifact_contract_version": str(artifact_manifest.get("contract_version", "")).strip(), "ranking_preview": ranking_preview_rows, }, "publish_summary": { @@ -147,6 +150,7 @@ def build_release_status_payload( "validation": { "ok": bool(validation.get("ok")), "manifest_present": bool(validation.get("manifest_present")), + "artifact_manifest_present": bool(validation.get("artifact_manifest_present")), "age_days": validation.get("age_days"), "errors": list(validation.get("errors", [])), "warnings": list(validation.get("warnings", [])), @@ -156,6 +160,7 @@ def build_release_status_payload( "latest_ranking": str(root / "latest_ranking.csv"), "live_pool": str(root / "live_pool.json"), "release_manifest": str(root / "release_manifest.json"), + "artifact_manifest": str(root / "artifact_manifest.json"), }, } @@ -190,6 +195,7 @@ def render_markdown(payload: dict[str, Any]) -> str: - latest_universe symbol count: {artifact['latest_universe_symbol_count']} - latest_ranking row count: {artifact['latest_ranking_row_count']} - latest_ranking selected count: {artifact['latest_ranking_selected_count']} +- artifact contract version: {artifact['artifact_contract_version'] or 'n/a'} ### Ranking preview @@ -207,6 +213,7 @@ def render_markdown(payload: dict[str, Any]) -> str: - ok: {validation['ok']} - manifest_present: {validation['manifest_present']} +- artifact_manifest_present: {validation['artifact_manifest_present']} - age_days: {validation['age_days']} ### Errors diff --git a/scripts/validate_release_contract.py b/scripts/validate_release_contract.py index 105e8db..704fecb 100644 --- a/scripts/validate_release_contract.py +++ b/scripts/validate_release_contract.py @@ -37,6 +37,11 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Require release_manifest.json and validate it against the live pool contract.", ) + parser.add_argument( + "--require-artifact-manifest", + action="store_true", + help="Require artifact_manifest.json and validate the profile-aware artifact contract.", + ) parser.add_argument( "--reference-date", default=None, @@ -55,6 +60,7 @@ def main() -> None: reference_date=args.reference_date, max_age_days=args.max_age_days, require_manifest=args.require_manifest, + require_artifact_manifest=args.require_artifact_manifest, require_freshness=not args.allow_stale, ) print(json.dumps(validation, ensure_ascii=False, indent=2)) diff --git a/src/export.py b/src/export.py index a51a92c..9bfd108 100644 --- a/src/export.py +++ b/src/export.py @@ -1,5 +1,7 @@ from __future__ import annotations +import hashlib +from pathlib import Path from typing import Any import pandas as pd @@ -8,6 +10,19 @@ from .utils import date_to_str, write_json +DEFAULT_STRATEGY_PROFILE = "crypto_leader_rotation" +DEFAULT_ARTIFACT_TYPE = "live_pool" +DEFAULT_ARTIFACT_CONTRACT_VERSION = "crypto_leader_rotation.live_pool.v1" + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + def export_latest_universe(panel: pd.DataFrame, output_dir: str | Any, as_of_date: pd.Timestamp) -> dict[str, Any]: """Export the latest dynamic universe to JSON.""" snapshot = panel.xs(as_of_date, level="date") @@ -141,3 +156,90 @@ def export_live_pool( if save_legacy: write_json(output_dir / "live_pool_legacy.json", legacy_payload) return payload + + +def build_strategy_artifact_manifest( + *, + output_dir: str | Any, + live_pool: dict[str, Any], + strategy_profile: str = DEFAULT_STRATEGY_PROFILE, + artifact_type: str = DEFAULT_ARTIFACT_TYPE, + contract_version: str = DEFAULT_ARTIFACT_CONTRACT_VERSION, + source_project: str = "crypto-leader-rotation", + generated_at: Any | None = None, +) -> dict[str, Any]: + """Build the profile-aware artifact manifest consumed by downstream runtimes.""" + output_path = Path(output_dir) + generated_at_value = generated_at if generated_at is not None else pd.Timestamp.now(tz="UTC") + generated_at_text = ( + generated_at_value.isoformat() + if hasattr(generated_at_value, "isoformat") + else str(generated_at_value) + ) + + artifact_files = { + "live_pool": "live_pool.json", + "live_pool_legacy": "live_pool_legacy.json", + "latest_ranking": "latest_ranking.csv", + "latest_universe": "latest_universe.json", + } + artifacts = {} + for artifact_name, filename in artifact_files.items(): + path = output_path / filename + if not path.exists(): + continue + artifacts[artifact_name] = { + "path": filename, + "sha256": _sha256_file(path), + } + + symbols = live_pool.get("symbols", ()) + if isinstance(symbols, dict): + symbols = tuple(symbols) + elif isinstance(symbols, list): + symbols = tuple(str(symbol) for symbol in symbols) + else: + symbols = () + + as_of_date = str(live_pool.get("as_of_date", "")).strip() + mode = str(live_pool.get("mode", "")).strip() + version = str(live_pool.get("version", "")).strip() + source_project_text = str(live_pool.get("source_project") or source_project) + return { + "manifest_type": "strategy_artifact", + "contract_version": str(contract_version), + "strategy_profile": str(strategy_profile), + "artifact_type": str(artifact_type), + "artifact_name": f"{strategy_profile}_{artifact_type}", + "as_of_date": as_of_date, + "snapshot_as_of": as_of_date, + "version": version, + "mode": mode, + "symbol_count": len(symbols), + "symbols": list(symbols), + "source_project": source_project_text, + "generated_at": generated_at_text, + "primary_artifact": "live_pool", + "artifacts": artifacts, + } + + +def export_strategy_artifact_manifest( + *, + output_dir: str | Any, + live_pool: dict[str, Any], + strategy_profile: str = DEFAULT_STRATEGY_PROFILE, + artifact_type: str = DEFAULT_ARTIFACT_TYPE, + contract_version: str = DEFAULT_ARTIFACT_CONTRACT_VERSION, + source_project: str = "crypto-leader-rotation", +) -> dict[str, Any]: + manifest = build_strategy_artifact_manifest( + output_dir=output_dir, + live_pool=live_pool, + strategy_profile=strategy_profile, + artifact_type=artifact_type, + contract_version=contract_version, + source_project=source_project, + ) + write_json(Path(output_dir) / "artifact_manifest.json", manifest) + return manifest diff --git a/src/pipeline.py b/src/pipeline.py index 56af6cf..a721643 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -8,7 +8,12 @@ from .backtest import run_backtest_suite, run_walkforward_scoring from .evaluation import evaluate_leader_selection, leader_metrics_to_frame from .external_data import load_optional_market_cap_metadata, merge_histories_with_external -from .export import export_latest_ranking, export_latest_universe, export_live_pool +from .export import ( + export_latest_ranking, + export_latest_universe, + export_live_pool, + export_strategy_artifact_manifest, +) from .features import MODEL_FEATURE_COLUMNS, add_market_context_features, build_feature_panel from .labels import build_labels from .models import fit_predict_models @@ -386,6 +391,12 @@ def build_live_pool_outputs( export_latest_universe(panel, output_dir, latest_date) export_latest_ranking(panel, output_dir, latest_date) latest_snapshot = latest_ranking_snapshot(panel, latest_date) + source_project = str( + config.get("publish", {}).get( + "source_project", + config.get("project", {}).get("name", "crypto-leader-rotation"), + ) + ) live_payload = export_live_pool( ranking_snapshot=latest_snapshot.loc[latest_snapshot["selected_flag"] | latest_snapshot["in_universe"]], metadata=metadata, @@ -393,7 +404,7 @@ def build_live_pool_outputs( as_of_date=latest_date, pool_size=int(config["export"]["live_pool_size"]), mode=str(resolved_mode), - source_project=str(config.get("publish", {}).get("source_project", config.get("project", {}).get("name", "crypto-leader-rotation"))), + source_project=source_project, selection_meta_fields=( list(config["export"].get("selection_meta_fields", [])) if config["export"].get("include_selection_meta", False) @@ -401,12 +412,18 @@ def build_live_pool_outputs( ), save_legacy=bool(config["export"]["save_legacy_live_pool"]), ) + artifact_manifest = export_strategy_artifact_manifest( + output_dir=output_dir, + live_pool=live_payload, + source_project=source_project, + ) logger.info("Live pool exports saved into %s for %s.", output_dir, latest_date.date()) return { "panel": panel, "metadata": metadata, "live_payload": live_payload, + "artifact_manifest": artifact_manifest, "as_of_date": latest_date, "train_start_date": train_start_date, "train_end_date": train_end_date, diff --git a/src/publish.py b/src/publish.py index aab7737..c6df6ec 100644 --- a/src/publish.py +++ b/src/publish.py @@ -16,6 +16,7 @@ "latest_ranking.csv", "live_pool.json", "live_pool_legacy.json", + "artifact_manifest.json", ) @@ -42,10 +43,12 @@ class ReleaseArtifacts: latest_ranking_path: Path live_pool_path: Path live_pool_legacy_path: Path + artifact_manifest_path: Path latest_universe: dict[str, Any] latest_ranking: pd.DataFrame live_pool: dict[str, Any] live_pool_legacy: dict[str, Any] + artifact_manifest: dict[str, Any] def parse_bool(value: Any, default: bool = False) -> bool: @@ -119,6 +122,7 @@ def load_release_artifacts(output_dir: Path | str, mode: str) -> ReleaseArtifact latest_universe = read_json(paths["latest_universe.json"]) live_pool = read_json(paths["live_pool.json"]) live_pool_legacy = read_json(paths["live_pool_legacy.json"]) + artifact_manifest = read_json(paths["artifact_manifest.json"]) latest_ranking = pd.read_csv(paths["latest_ranking.csv"]) as_of_values = { @@ -147,10 +151,12 @@ def load_release_artifacts(output_dir: Path | str, mode: str) -> ReleaseArtifact latest_ranking_path=paths["latest_ranking.csv"], live_pool_path=paths["live_pool.json"], live_pool_legacy_path=paths["live_pool_legacy.json"], + artifact_manifest_path=paths["artifact_manifest.json"], latest_universe=latest_universe, latest_ranking=latest_ranking, live_pool=live_pool, live_pool_legacy=live_pool_legacy, + artifact_manifest=artifact_manifest, ) @@ -171,6 +177,7 @@ def ensure_publish_preflight( reference_date=reference_date, max_age_days=max_age_days, require_manifest=False, + require_artifact_manifest=True, require_freshness=require_freshness, ) if settings.dry_run: @@ -199,6 +206,7 @@ def build_storage_layout(settings: PublishSettings, artifacts: ReleaseArtifacts) "latest_ranking.csv": artifacts.latest_ranking_path, "live_pool.json": artifacts.live_pool_path, "live_pool_legacy.json": artifacts.live_pool_legacy_path, + "artifact_manifest.json": artifacts.artifact_manifest_path, } objects: dict[str, dict[str, str]] = {} @@ -243,6 +251,9 @@ def build_firestore_payload( "latest_universe_uri": storage_layout["objects"]["latest_universe.json"]["current_uri"], "latest_ranking_uri": storage_layout["objects"]["latest_ranking.csv"]["current_uri"], "versioned_live_pool_legacy_uri": storage_layout["objects"]["live_pool_legacy.json"]["release_uri"], + "artifact_manifest_uri": storage_layout["objects"]["artifact_manifest.json"]["current_uri"], + "versioned_artifact_manifest_uri": storage_layout["objects"]["artifact_manifest.json"]["release_uri"], + "artifact_contract_version": str(artifacts.artifact_manifest.get("contract_version", "")), "generated_at": generated_at, "source_project": settings.source_project, } @@ -267,6 +278,7 @@ def build_release_manifest( "latest_ranking": storage_layout["objects"]["latest_ranking.csv"], "live_pool": storage_layout["objects"]["live_pool.json"], "live_pool_legacy": storage_layout["objects"]["live_pool_legacy.json"], + "artifact_manifest": storage_layout["objects"]["artifact_manifest.json"], }, "firestore": { "collection": settings.firestore_collection, @@ -306,6 +318,7 @@ def upload_release_artifacts( "latest_ranking.csv": artifacts.latest_ranking_path, "live_pool.json": artifacts.live_pool_path, "live_pool_legacy.json": artifacts.live_pool_legacy_path, + "artifact_manifest.json": artifacts.artifact_manifest_path, } for filename, local_path in files.items(): object_info = storage_layout["objects"][filename] @@ -371,6 +384,7 @@ def run_release_publish( expected_pool_size=int(config["export"]["live_pool_size"]), max_age_days=max_age_days, require_manifest=True, + require_artifact_manifest=True, require_freshness=require_freshness, ) diff --git a/src/release_contract.py b/src/release_contract.py index 55ee3c3..6e29e53 100644 --- a/src/release_contract.py +++ b/src/release_contract.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib from pathlib import Path from typing import Any @@ -15,6 +16,7 @@ "live_pool_legacy.json", ) REQUIRED_MANIFEST_FILE = "release_manifest.json" +REQUIRED_ARTIFACT_MANIFEST_FILE = "artifact_manifest.json" REQUIRED_RANKING_COLUMNS = ( "as_of_date", "symbol", @@ -36,6 +38,38 @@ "symbol_map", "source_project", ) +REQUIRED_ARTIFACT_MANIFEST_FIELDS = ( + "manifest_type", + "contract_version", + "strategy_profile", + "artifact_type", + "artifact_name", + "as_of_date", + "snapshot_as_of", + "version", + "mode", + "symbol_count", + "symbols", + "source_project", + "generated_at", + "primary_artifact", + "artifacts", +) +REQUIRED_ARTIFACT_MANIFEST_ARTIFACTS = { + "latest_universe": "latest_universe.json", + "latest_ranking": "latest_ranking.csv", + "live_pool": "live_pool.json", + "live_pool_legacy": "live_pool_legacy.json", +} +EXPECTED_ARTIFACT_CONTRACT_VERSION = "crypto_leader_rotation.live_pool.v1" + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() def build_release_version(as_of_date: str, mode: str) -> str: @@ -149,6 +183,7 @@ def validate_release_outputs( reference_date: Any = None, max_age_days: int | None = None, require_manifest: bool = False, + require_artifact_manifest: bool = False, require_freshness: bool = False, ) -> dict[str, Any]: output_path = Path(output_dir) @@ -170,6 +205,13 @@ def validate_release_outputs( elif manifest_present: loaded_files[REQUIRED_MANIFEST_FILE] = manifest_path + artifact_manifest_path = output_path / REQUIRED_ARTIFACT_MANIFEST_FILE + artifact_manifest_present = artifact_manifest_path.exists() + if require_artifact_manifest and not artifact_manifest_present: + errors.append(f"missing required output: {artifact_manifest_path}") + elif artifact_manifest_present: + loaded_files[REQUIRED_ARTIFACT_MANIFEST_FILE] = artifact_manifest_path + if errors: return { "ok": False, @@ -177,12 +219,14 @@ def validate_release_outputs( "warnings": warnings, "output_dir": str(output_path), "manifest_present": manifest_present, + "artifact_manifest_present": artifact_manifest_present, } latest_universe = read_json(loaded_files["latest_universe.json"], default={}) or {} live_pool = read_json(loaded_files["live_pool.json"], default={}) or {} live_pool_legacy = read_json(loaded_files["live_pool_legacy.json"], default={}) or {} manifest = read_json(manifest_path, default={}) or {} + artifact_manifest = read_json(artifact_manifest_path, default={}) or {} ranking_path = loaded_files["latest_ranking.csv"] try: @@ -203,6 +247,9 @@ def validate_release_outputs( if manifest_present and not isinstance(manifest, dict): errors.append("release_manifest.json must contain an object") manifest = {} + if artifact_manifest_present and not isinstance(artifact_manifest, dict): + errors.append("artifact_manifest.json must contain an object") + artifact_manifest = {} _append_missing_fields(live_pool, REQUIRED_LIVE_POOL_FIELDS, errors, "live_pool.json") _append_missing_fields(live_pool_legacy, REQUIRED_LIVE_POOL_FIELDS, errors, "live_pool_legacy.json") @@ -305,6 +352,10 @@ def validate_release_outputs( manifest_as_of_date, _ = _parse_as_of_date(manifest.get("as_of_date")) if manifest_as_of_date: as_of_values.add(manifest_as_of_date) + if artifact_manifest_present: + artifact_manifest_as_of_date, _ = _parse_as_of_date(artifact_manifest.get("as_of_date")) + if artifact_manifest_as_of_date: + as_of_values.add(artifact_manifest_as_of_date) if len(as_of_values) > 1: errors.append(f"release outputs have inconsistent as_of_date values: {sorted(as_of_values)}") @@ -399,10 +450,78 @@ def validate_release_outputs( "release_manifest.json firestore.payload source_project does not match live_pool.json" ) + if artifact_manifest_present: + _append_missing_fields( + artifact_manifest, + REQUIRED_ARTIFACT_MANIFEST_FIELDS, + errors, + "artifact_manifest.json", + ) + if str(artifact_manifest.get("manifest_type", "")).strip() != "strategy_artifact": + errors.append("artifact_manifest.json manifest_type must be strategy_artifact") + if str(artifact_manifest.get("contract_version", "")).strip() != EXPECTED_ARTIFACT_CONTRACT_VERSION: + errors.append( + f"artifact_manifest.json contract_version must be {EXPECTED_ARTIFACT_CONTRACT_VERSION}" + ) + if str(artifact_manifest.get("strategy_profile", "")).strip() != "crypto_leader_rotation": + errors.append("artifact_manifest.json strategy_profile must be crypto_leader_rotation") + if str(artifact_manifest.get("artifact_type", "")).strip() != "live_pool": + errors.append("artifact_manifest.json artifact_type must be live_pool") + if str(artifact_manifest.get("primary_artifact", "")).strip() != "live_pool": + errors.append("artifact_manifest.json primary_artifact must be live_pool") + if str(artifact_manifest.get("version", "")).strip() != live_pool_version: + errors.append("artifact_manifest.json version does not match live_pool.json version") + if str(artifact_manifest.get("mode", "")).strip() != live_pool_mode: + errors.append("artifact_manifest.json mode does not match live_pool.json mode") + artifact_as_of_date, _ = _parse_as_of_date(artifact_manifest.get("as_of_date")) + if artifact_as_of_date and live_pool_as_of_date and artifact_as_of_date != live_pool_as_of_date: + errors.append("artifact_manifest.json as_of_date does not match live_pool.json as_of_date") + artifact_snapshot_as_of, _ = _parse_as_of_date(artifact_manifest.get("snapshot_as_of")) + if artifact_snapshot_as_of and live_pool_as_of_date and artifact_snapshot_as_of != live_pool_as_of_date: + errors.append("artifact_manifest.json snapshot_as_of does not match live_pool.json as_of_date") + if str(artifact_manifest.get("source_project", "")).strip() != live_pool_source_project: + errors.append("artifact_manifest.json source_project does not match live_pool.json") + artifact_symbols = artifact_manifest.get("symbols") + if artifact_symbols != live_pool_symbols: + errors.append("artifact_manifest.json symbols do not match live_pool.json symbols") + try: + artifact_symbol_count = int(artifact_manifest.get("symbol_count", -1)) + except Exception: + artifact_symbol_count = -1 + if artifact_symbol_count != len(live_pool_symbols): + errors.append("artifact_manifest.json symbol_count does not match live_pool.json") + artifacts = artifact_manifest.get("artifacts", {}) + if not isinstance(artifacts, dict): + errors.append("artifact_manifest.json artifacts must be an object") + artifacts = {} + for artifact_name, expected_filename in REQUIRED_ARTIFACT_MANIFEST_ARTIFACTS.items(): + artifact_entry = artifacts.get(artifact_name) + if not isinstance(artifact_entry, dict): + errors.append(f"artifact_manifest.json artifacts.{artifact_name} must be an object") + continue + artifact_path = str(artifact_entry.get("path", "")).strip() + if not artifact_path: + errors.append(f"artifact_manifest.json artifacts.{artifact_name}.path must be non-empty") + continue + if Path(artifact_path).name != expected_filename: + errors.append( + f"artifact_manifest.json artifacts.{artifact_name}.path must point to {expected_filename}" + ) + continue + resolved_path = output_path / artifact_path + if not resolved_path.exists(): + errors.append(f"artifact_manifest.json artifacts.{artifact_name}.path does not exist: {artifact_path}") + continue + expected_sha = str(artifact_entry.get("sha256", "")).strip() + if not expected_sha: + errors.append(f"artifact_manifest.json artifacts.{artifact_name}.sha256 must be non-empty") + elif expected_sha != _sha256_file(resolved_path): + errors.append(f"artifact_manifest.json artifacts.{artifact_name}.sha256 does not match file content") + age_days: int | None = None if live_pool_as_of_ts is not None: if reference_date is None: - reference_ts = pd.Timestamp.utcnow().normalize() + reference_ts = pd.Timestamp.now(tz="UTC").normalize() else: reference_ts = pd.Timestamp(reference_date).normalize() age_days = (reference_ts.date() - live_pool_as_of_ts.date()).days @@ -424,6 +543,10 @@ def validate_release_outputs( "warnings": warnings, "output_dir": str(output_path), "manifest_present": manifest_present, + "artifact_manifest_present": artifact_manifest_present, + "artifact_contract_version": str(artifact_manifest.get("contract_version", "")).strip() + if artifact_manifest_present + else None, "as_of_date": live_pool_as_of_date, "version": live_pool_version, "mode": live_pool_mode, diff --git a/tests/test_build_live_pool_smoke.py b/tests/test_build_live_pool_smoke.py index 1b55c86..f7699d0 100644 --- a/tests/test_build_live_pool_smoke.py +++ b/tests/test_build_live_pool_smoke.py @@ -10,6 +10,7 @@ from unittest.mock import patch from src.release_contract import validate_release_outputs +from src.export import export_strategy_artifact_manifest PROJECT_ROOT = Path(__file__).resolve().parents[1] @@ -61,6 +62,7 @@ def fake_build_live_pool_outputs(config, as_of_date=None, universe_mode=None): for fixture_file in FIXTURE_ROOT.iterdir(): shutil.copy2(fixture_file, output_path / fixture_file.name) live_payload = json.loads((output_path / "live_pool.json").read_text(encoding="utf-8")) + export_strategy_artifact_manifest(output_dir=output_path, live_pool=live_payload) return { "as_of_date": MODULE.pd.Timestamp("2026-03-13"), "train_start_date": MODULE.pd.Timestamp("2024-01-01"), @@ -90,10 +92,12 @@ def fake_build_live_pool_outputs(config, as_of_date=None, universe_mode=None): expected_mode="core_major", expected_source_project="crypto-leader-rotation", expected_pool_size=5, + require_artifact_manifest=True, require_freshness=False, ) self.assertTrue(validation["ok"]) + self.assertTrue(validation["artifact_manifest_present"]) self.assertEqual(validation["version"], "2026-03-13-core_major") self.assertEqual(validation["pool_size"], 5) diff --git a/tests/test_release_contract.py b/tests/test_release_contract.py index 0a2db96..74c92c1 100644 --- a/tests/test_release_contract.py +++ b/tests/test_release_contract.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import json import tempfile import unittest @@ -15,6 +16,14 @@ def write_json(path: Path, payload: dict[str, object]) -> None: path.write_text(json.dumps(payload, indent=2), encoding="utf-8") +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + class ReleaseContractValidationTests(unittest.TestCase): def build_outputs( self, @@ -76,6 +85,44 @@ def build_outputs( ] ).to_csv(output_dir / "latest_ranking.csv", index=False) + write_json( + output_dir / "artifact_manifest.json", + { + "manifest_type": "strategy_artifact", + "contract_version": "crypto_leader_rotation.live_pool.v1", + "strategy_profile": "crypto_leader_rotation", + "artifact_type": "live_pool", + "artifact_name": "crypto_leader_rotation_live_pool", + "as_of_date": as_of_date, + "snapshot_as_of": as_of_date, + "version": version, + "mode": mode, + "symbol_count": len(symbols), + "symbols": symbols, + "source_project": source_project, + "generated_at": "2026-03-13T00:00:00+00:00", + "primary_artifact": "live_pool", + "artifacts": { + "latest_universe": { + "path": "latest_universe.json", + "sha256": sha256_file(output_dir / "latest_universe.json"), + }, + "latest_ranking": { + "path": "latest_ranking.csv", + "sha256": sha256_file(output_dir / "latest_ranking.csv"), + }, + "live_pool": { + "path": "live_pool.json", + "sha256": sha256_file(output_dir / "live_pool.json"), + }, + "live_pool_legacy": { + "path": "live_pool_legacy.json", + "sha256": sha256_file(output_dir / "live_pool_legacy.json"), + }, + }, + }, + ) + if include_manifest: write_json( output_dir / "release_manifest.json", @@ -117,14 +164,42 @@ def test_validate_release_outputs_accepts_consistent_contract(self) -> None: reference_date="2026-03-14", max_age_days=45, require_manifest=True, + require_artifact_manifest=True, require_freshness=True, ) self.assertTrue(validation["ok"]) + self.assertTrue(validation["artifact_manifest_present"]) + self.assertEqual(validation["artifact_contract_version"], "crypto_leader_rotation.live_pool.v1") self.assertEqual(validation["version"], "2026-03-13-core_major") self.assertEqual(validation["pool_size"], 5) self.assertEqual(validation["age_days"], 1) + def test_validate_release_outputs_rejects_mismatched_artifact_manifest(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + root = Path(tmp_dir) + self.build_outputs(root) + manifest_path = root / "data" / "output" / "artifact_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + manifest["contract_version"] = "crypto_leader_rotation.live_pool.v0" + manifest["artifacts"]["live_pool"]["sha256"] = "wrong" + write_json(manifest_path, manifest) + + validation = validate_release_outputs( + root / "data" / "output", + require_artifact_manifest=True, + ) + + self.assertFalse(validation["ok"]) + self.assertIn( + "artifact_manifest.json contract_version must be crypto_leader_rotation.live_pool.v1", + validation["errors"], + ) + self.assertIn( + "artifact_manifest.json artifacts.live_pool.sha256 does not match file content", + validation["errors"], + ) + def test_validate_release_outputs_rejects_mismatched_manifest_payload(self) -> None: with tempfile.TemporaryDirectory() as tmp_dir: root = Path(tmp_dir) diff --git a/tests/test_release_status_summary.py b/tests/test_release_status_summary.py index 9169b16..9b346b5 100644 --- a/tests/test_release_status_summary.py +++ b/tests/test_release_status_summary.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import importlib.util import json import tempfile @@ -21,8 +22,22 @@ def write_json(path: Path, payload: dict[str, object]) -> None: path.write_text(json.dumps(payload, indent=2), encoding="utf-8") +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + class ReleaseStatusSummaryTests(unittest.TestCase): - def write_outputs(self, root: Path, *, include_manifest: bool = True) -> Path: + def write_outputs( + self, + root: Path, + *, + include_manifest: bool = True, + include_artifact_manifest: bool = True, + ) -> Path: output_dir = root / "data" / "output" output_dir.mkdir(parents=True, exist_ok=True) symbols = ["TRXUSDT", "ETHUSDT", "BCHUSDT", "NEARUSDT", "SOLUSDT"] @@ -72,6 +87,45 @@ def write_outputs(self, root: Path, *, include_manifest: bool = True) -> Path: ] ).to_csv(output_dir / "latest_ranking.csv", index=False) + if include_artifact_manifest: + write_json( + output_dir / "artifact_manifest.json", + { + "manifest_type": "strategy_artifact", + "contract_version": "crypto_leader_rotation.live_pool.v1", + "strategy_profile": "crypto_leader_rotation", + "artifact_type": "live_pool", + "artifact_name": "crypto_leader_rotation_live_pool", + "as_of_date": "2026-03-13", + "snapshot_as_of": "2026-03-13", + "version": version, + "mode": "core_major", + "symbol_count": len(symbols), + "symbols": symbols, + "source_project": "crypto-leader-rotation", + "generated_at": "2026-03-13T00:00:00+00:00", + "primary_artifact": "live_pool", + "artifacts": { + "latest_universe": { + "path": "latest_universe.json", + "sha256": sha256_file(output_dir / "latest_universe.json"), + }, + "latest_ranking": { + "path": "latest_ranking.csv", + "sha256": sha256_file(output_dir / "latest_ranking.csv"), + }, + "live_pool": { + "path": "live_pool.json", + "sha256": sha256_file(output_dir / "live_pool.json"), + }, + "live_pool_legacy": { + "path": "live_pool_legacy.json", + "sha256": sha256_file(output_dir / "live_pool_legacy.json"), + }, + }, + }, + ) + if include_manifest: write_json( output_dir / "release_manifest.json", @@ -117,6 +171,7 @@ def test_build_release_status_payload_reports_ok_for_consistent_release(self) -> self.assertEqual(payload["status"], "ok") self.assertEqual(payload["official_release"]["version"], "2026-03-13-core_major") self.assertEqual(payload["artifact_summary"]["latest_universe_symbol_count"], 6) + self.assertEqual(payload["artifact_summary"]["artifact_contract_version"], "crypto_leader_rotation.live_pool.v1") self.assertEqual(len(payload["artifact_summary"]["ranking_preview"]), 3) self.assertTrue(payload["validation"]["ok"]) @@ -135,6 +190,19 @@ def test_build_release_status_payload_reports_error_when_manifest_missing(self) self.assertTrue(any("release_manifest.json" in item for item in payload["validation"]["errors"])) self.assertIn("## Validation", markdown) + def test_build_release_status_payload_reports_error_when_artifact_manifest_missing(self) -> None: + with tempfile.TemporaryDirectory() as tmp_dir: + output_dir = self.write_outputs(Path(tmp_dir), include_artifact_manifest=False) + payload = MODULE.build_release_status_payload( + output_dir, + max_age_days=45, + require_freshness=False, + ) + + self.assertEqual(payload["status"], "error") + self.assertFalse(payload["validation"]["artifact_manifest_present"]) + self.assertTrue(any("artifact_manifest.json" in item for item in payload["validation"]["errors"])) + if __name__ == "__main__": unittest.main()