Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/monthly_publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ jobs:
python-version: "3.11"

- name: Authenticate to Google Cloud
uses: google-github-actions/auth@v2
uses: google-github-actions/auth@v3

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Re-authenticate closer to the publish step

With the new Workload Identity Federation setup, the google-github-actions/auth docs warn that the GitHub OIDC token and derived credentials expire after about 5 minutes; in this workflow the only Google Cloud work happens later in Publish Production v1 Release, after dependency install, history download, and the monthly shadow build. When those pre-publish steps take more than a few minutes, the publish script will try to upload to GCS/Firestore with expired ADC credentials, so move this auth step immediately before publishing or refresh credentials there.

Useful? React with 👍 / 👎.

with:
workload_identity_provider: "projects/677468735457/locations/global/workloadIdentityPools/github-actions/providers/github-main"
service_account: "codex-gcp-operator@binancequant.iam.gserviceaccount.com"
Expand Down
1 change: 1 addition & 0 deletions requirements-lock.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@d786c1140967f0e96e35599d057f0655e5a9ba25
pandas==3.0.3
numpy==2.4.6
requests==2.34.2
Expand Down
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
quant-platform-kit @ git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@d786c1140967f0e96e35599d057f0655e5a9ba25
pandas>=3.0.3
numpy>=2.4.6,<2.5
requests>=2.34.2
Expand Down
100 changes: 68 additions & 32 deletions scripts/gate_codex_app_review.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,12 +33,18 @@ def env(name: str, default: str = "") -> str:


def env_int(name: str, default: int) -> int:
try: return int(env(name, str(default)))
except ValueError: return default
try:
return int(env(name, str(default)))
except ValueError:
return default


def github_request(token: str, method: str, path: str,
payload: dict[str, Any] | None = None) -> Any:
def github_request(
token: str,
method: str,
path: str,
payload: dict[str, Any] | None = None,
) -> Any:
url = f"{API_BASE}{path}" if not path.startswith("https://") else path
data = json.dumps(payload).encode() if payload else None
headers = {
Expand All @@ -47,7 +53,8 @@ def github_request(token: str, method: str, path: str,
"X-GitHub-Api-Version": "2022-11-28",
"User-Agent": "codex-review-gate",
}
if payload: headers["Content-Type"] = "application/json"
if payload:
headers["Content-Type"] = "application/json"
req = urllib.request.Request(url, data=data, method=method, headers=headers)
try:
with urllib.request.urlopen(req, timeout=30) as resp:
Expand All @@ -69,8 +76,10 @@ def step_summary(text: str) -> None:

def load_policy() -> dict[str, Any]:
if POLICY_PATH.exists():
try: return json.loads(POLICY_PATH.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError): pass
try:
return json.loads(POLICY_PATH.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
pass
return {
"version": 1,
"blocked_path_patterns": [
Expand All @@ -85,8 +94,10 @@ def compile_patterns(policy: dict[str, Any]) -> list[re.Pattern[str]]:
pp: list[re.Pattern[str]] = []
for p in policy.get("blocked_path_patterns", []):
if isinstance(p, str) and p.strip():
try: pp.append(re.compile(p, re.IGNORECASE))
except re.error: pass
try:
pp.append(re.compile(p, re.IGNORECASE))
except re.error:
pass
return pp


Expand All @@ -111,8 +122,11 @@ def scan_diff(diff_text: str, path_patterns: list[re.Pattern[str]]) -> list[str]
violations.append(f"**Blocked file**: `{current}` matches `{pat.pattern}`")
break
continue
if line.startswith("+++ b/"): current = line[6:]; continue
if not line.startswith("+") or line.startswith("+++"): continue
if line.startswith("+++ b/"):
current = line[6:]
continue
if not line.startswith("+") or line.startswith("+++"):
continue
m = _SENSITIVE.search(line[1:])
if m:
violations.append(f"**Hardcoded secret** in `{current}`: `{m.group(0)[:100]}`")
Expand All @@ -128,8 +142,10 @@ def check_metadata(files: list[dict[str, Any]], policy: dict[str, Any]) -> list[
for f in files:
fn = f.get("filename", "?")
st = (f.get("status") or "").lower().strip()
if st == "removed": issues.append(f"**File deleted**: `{fn}` — verify intentional")
elif st == "renamed": issues.append(f"**File renamed**: `{f.get('previous_filename', '?')}` → `{fn}`")
if st == "removed":
issues.append(f"**File deleted**: `{fn}` — verify intentional")
elif st == "renamed":
issues.append(f"**File renamed**: `{f.get('previous_filename', '?')}` → `{fn}`")
if len(files) > mx_f:
issues.append(f"**Too many files**: {len(files)} changed (limit {mx_f})")
if ta + td > mx_l:
Expand All @@ -144,12 +160,18 @@ def run_static_guard(token: str, repo: str, pr_number: int) -> int:
page = 1
while True:
try:
batch = github_request(token, "GET",
f"/repos/{repo}/pulls/{pr_number}/files?per_page=100&page={page}")
except RuntimeError: break
if not isinstance(batch, list) or not batch: break
batch = github_request(
token,
"GET",
f"/repos/{repo}/pulls/{pr_number}/files?per_page=100&page={page}",
)
except RuntimeError:
break
if not isinstance(batch, list) or not batch:
break
files.extend(batch)
if len(batch) < 100: break
if len(batch) < 100:
break
page += 1

diff_text = ""
Expand All @@ -165,23 +187,29 @@ def run_static_guard(token: str, repo: str, pr_number: int) -> int:
)
with urllib.request.urlopen(req, timeout=30) as resp:
diff_text = resp.read().decode("utf-8", errors="replace")
except Exception: pass
except Exception:
pass

issues = check_metadata(files, policy) + scan_diff(diff_text, compile_patterns(policy))
if not issues: return 0
if not issues:
return 0

print(f"STATIC → BLOCKED: {len(issues)} issue(s)")
for i in issues: print(f" • {i}")
step_summary(f"## Merge blocked: {len(issues)} static issue(s)\n\n" +
"\n".join(f"- {i}" for i in issues))
for i in issues:
print(f" • {i}")
step_summary(
f"## Merge blocked: {len(issues)} static issue(s)\n\n"
+ "\n".join(f"- {i}" for i in issues)
)
return 1


# ─── app review ──────────────────────────────────────────────────────────────

def get_codex_review(token: str, repo: str, pr_number: int) -> dict[str, Any] | None:
reviews = github_request(token, "GET", f"/repos/{repo}/pulls/{pr_number}/reviews?per_page=100")
if not isinstance(reviews, list): return None
if not isinstance(reviews, list):
return None
for r in reversed(reviews):
if isinstance(r, dict) and (r.get("user") or {}).get("login") == BOT_LOGIN:
return r
Expand Down Expand Up @@ -228,16 +256,20 @@ def main() -> int:
pr_number = pr.get("number")
head_sha = (pr.get("head") or {}).get("sha")
if not pr_number or not head_sha:
print(f"::warning::Cannot resolve PR context"); return 0
print("::warning::Cannot resolve PR context")
return 0

print(f"PR #{pr_number} sha={head_sha[:12]} event={event_name}")

# ── Phase 1: Static guard (skip on review-only events) ────────────
if event_name != "pull_request_review":
try: rc = run_static_guard(token, repo, pr_number)
try:
rc = run_static_guard(token, repo, pr_number)
except RuntimeError as exc:
print(f"::warning::Static guard error: {exc}"); rc = 0
if rc != 0: return 1
print(f"::warning::Static guard error: {exc}")
rc = 0
if rc != 0:
return 1
print("STATIC → clean")

# ── Phase 2: App review ───────────────────────────────────────────
Expand All @@ -250,8 +282,10 @@ def main() -> int:
return rc

# WAIT: poll for existing or upcoming review
try: existing = get_codex_review(token, repo, pr_number)
except RuntimeError: existing = None
try:
existing = get_codex_review(token, repo, pr_number)
except RuntimeError:
existing = None

if existing is not None:
rc, title, summary = app_decision(existing)
Expand All @@ -266,8 +300,10 @@ def main() -> int:

while time.time() < deadline:
time.sleep(poll_s)
try: review = get_codex_review(token, repo, pr_number)
except RuntimeError: continue
try:
review = get_codex_review(token, repo, pr_number)
except RuntimeError:
continue
if review is not None:
rc, title, summary = app_decision(review)
print(f"WAIT → found review → exit={rc}: {title}")
Expand Down
2 changes: 1 addition & 1 deletion src/indicators.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import annotations

import math
from typing import Any

import numpy as np
import pandas as pd
Expand Down Expand Up @@ -188,4 +189,3 @@ def rolling_beta(asset_returns: pd.Series, benchmark_returns: pd.Series, window:

def rolling_correlation(asset_returns: pd.Series, benchmark_returns: pd.Series, window: int = 60) -> pd.Series:
return asset_returns.rolling(window, min_periods=window).corr(benchmark_returns)

2 changes: 1 addition & 1 deletion src/publish.py
Original file line number Diff line number Diff line change
Expand Up @@ -187,7 +187,7 @@ def ensure_publish_preflight(
if settings.dry_run:
return validation
if not settings.project_id:
raise ValueError("Publish preflight failed: CLOUD_PROJECT_ID is required for a real publish.")
raise ValueError("Publish preflight failed: CLOUD_PROJECT_ID or GCP_PROJECT_ID is required for a real publish.")
if not settings.cloud_bucket:
raise ValueError("Publish preflight failed: CLOUD_BUCKET is required for a real publish.")
if not str(settings.firestore_collection).strip():
Expand Down
16 changes: 15 additions & 1 deletion tests/test_monthly_publish_workflow_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@
PROJECT_ROOT = Path(__file__).resolve().parents[1]
WORKFLOW_PATH = PROJECT_ROOT / ".github" / "workflows" / "monthly_publish.yml"
README_ZH_PATH = PROJECT_ROOT / "README.zh-CN.md"
QPK_DEPENDENCY = (
"quant-platform-kit @ "
"git+https://github.com/QuantStrategyLab/QuantPlatformKit.git@d786c1140967f0e96e35599d057f0655e5a9ba25"
)


class MonthlyPublishWorkflowConfigTests(unittest.TestCase):
Expand All @@ -19,10 +23,13 @@ def test_publish_targets_use_vars_only(self) -> None:
self.assertIn("actions/upload-artifact@v7", workflow)
self.assertIn("GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}", workflow)
self.assertIn("GCS_BUCKET: ${{ vars.GCS_BUCKET }}", workflow)
self.assertIn("credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}", workflow)
self.assertIn("workload_identity_provider:", workflow)
self.assertIn("service_account:", workflow)
self.assertIn("issues: write", workflow)
self.assertNotIn("secrets.GCP_PROJECT_ID", workflow)
self.assertNotIn("secrets.GCS_BUCKET", workflow)
self.assertNotIn("credentials_json:", workflow)
self.assertNotIn("GCP_SERVICE_ACCOUNT_KEY", workflow)

def test_monthly_review_issue_creation_does_not_require_gh_cli(self) -> None:
workflow = WORKFLOW_PATH.read_text(encoding="utf-8")
Expand Down Expand Up @@ -64,6 +71,13 @@ def test_monthly_review_issue_creation_does_not_require_gh_cli(self) -> None:
self.assertNotIn("LEGACY_API_REVIEW_ENABLED", workflow)
self.assertNotIn("/actions/workflows/ai_review.yml/dispatches", workflow)

def test_real_publish_dependency_is_locked(self) -> None:
requirements = (PROJECT_ROOT / "requirements.txt").read_text(encoding="utf-8")
requirements_lock = (PROJECT_ROOT / "requirements-lock.txt").read_text(encoding="utf-8")

self.assertIn(QPK_DEPENDENCY, requirements)
self.assertIn(QPK_DEPENDENCY, requirements_lock)

def test_source_local_legacy_ai_workflows_are_removed(self) -> None:
workflow_dir = PROJECT_ROOT / ".github" / "workflows"

Expand Down
Loading