Experiment Validation #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Experiment Validation | |
| "on": | |
| issues: | |
| types: [opened, edited, reopened, labeled] | |
| workflow_dispatch: | |
| inputs: | |
| issue_number: | |
| description: "Monthly optimization task issue number" | |
| required: true | |
| jobs: | |
| validate: | |
| if: contains(github.event.issue.labels.*.name, 'monthly-optimization-task') || inputs.issue_number != '' | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| issues: write | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v6 | |
| - name: Setup Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| run: | | |
| set -euo pipefail | |
| REQ_FILE="requirements-lock.txt" | |
| if [ ! -f "$REQ_FILE" ]; then REQ_FILE="requirements.txt"; fi | |
| python -m pip install --upgrade pip | |
| python -m pip install -r "$REQ_FILE" | |
| - name: Load issue context | |
| id: issue_context | |
| run: | | |
| python3 - <<'PY' | |
| import json | |
| import os | |
| import urllib.request | |
| from pathlib import Path | |
| repo = os.environ["GITHUB_REPOSITORY"] | |
| issue_number = os.environ["ISSUE_NUMBER"] | |
| token = os.environ["GITHUB_TOKEN"] | |
| api_url = f"https://api.github.com/repos/{repo}/issues/{issue_number}" | |
| request = urllib.request.Request( | |
| api_url, | |
| headers={ | |
| "Accept": "application/vnd.github+json", | |
| "Authorization": f"Bearer {token}", | |
| "X-GitHub-Api-Version": "2022-11-28", | |
| "User-Agent": "experiment-validation", | |
| }, | |
| ) | |
| with urllib.request.urlopen(request) as response: | |
| issue = json.load(response) | |
| issue_context = { | |
| "number": issue["number"], | |
| "title": issue["title"], | |
| "body": issue.get("body", ""), | |
| } | |
| output_dir = Path("data/output/experiment_validation") | |
| output_dir.mkdir(parents=True, exist_ok=True) | |
| (output_dir / "issue_context.json").write_text( | |
| json.dumps(issue_context, ensure_ascii=False, indent=2) + "\n", | |
| encoding="utf-8", | |
| ) | |
| PY | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| ISSUE_NUMBER: ${{ inputs.issue_number || github.event.issue.number }} | |
| - name: Prepare experiment validation payload | |
| id: experiment_payload | |
| run: | | |
| python3 scripts/prepare_experiment_validation.py \ | |
| --issue-context-file data/output/experiment_validation/issue_context.json \ | |
| --output-dir data/output/experiment_validation >> "$GITHUB_OUTPUT" | |
| - name: Append task summary | |
| run: cat data/output/experiment_validation/task_summary.md >> "$GITHUB_STEP_SUMMARY" | |
| - name: Append skip reason | |
| if: steps.experiment_payload.outputs.should_run != 'true' | |
| run: | | |
| if [ -f data/output/experiment_validation/skip_reason.txt ]; then | |
| cat data/output/experiment_validation/skip_reason.txt >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| - name: Run monthly shadow build | |
| if: steps.experiment_payload.outputs.should_run == 'true' && steps.experiment_payload.outputs.run_shadow_build == 'true' | |
| run: | | |
| python3 scripts/run_monthly_shadow_build.py --skip-publish-dry-run | |
| - name: Run walk-forward validation | |
| if: steps.experiment_payload.outputs.should_run == 'true' && steps.experiment_payload.outputs.run_walkforward_validation == 'true' | |
| run: | | |
| python3 scripts/run_walkforward_validation.py | |
| - name: Render validation summary | |
| if: steps.experiment_payload.outputs.should_run == 'true' | |
| run: | | |
| python3 scripts/render_experiment_validation_summary.py \ | |
| --payload-file data/output/experiment_validation/payload.json \ | |
| --shadow-summary-file data/output/monthly_shadow_build_summary.json \ | |
| --output-file data/output/experiment_validation/validation_summary.md | |
| - name: Append validation summary | |
| if: steps.experiment_payload.outputs.should_run == 'true' | |
| run: cat data/output/experiment_validation/validation_summary.md >> "$GITHUB_STEP_SUMMARY" | |
| - name: Post validation comment | |
| if: steps.experiment_payload.outputs.should_run == 'true' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| python3 scripts/post_experiment_validation_comment.py \ | |
| --repo "${GITHUB_REPOSITORY}" \ | |
| --issue-number "${{ inputs.issue_number || github.event.issue.number }}" \ | |
| --review-file data/output/experiment_validation/validation_summary.md \ | |
| --run-url "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" | |
| - name: Upload validation artifact | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: experiment-validation-${{ inputs.issue_number || github.event.issue.number }} | |
| path: data/output/experiment_validation/ |