diff --git a/deep_quoridor/src/run_benchmarks_v2.py b/deep_quoridor/src/run_benchmarks_v2.py new file mode 100644 index 00000000..a12fdd8b --- /dev/null +++ b/deep_quoridor/src/run_benchmarks_v2.py @@ -0,0 +1,119 @@ +"""Run just the benchmark schedules from an existing run's config.yaml. + +Usage: + python deep_quoridor/src/run_benchmarks_v2.py [-o key=val ...] + +Spawns one process per `config.benchmarks` schedule and waits until Ctrl-C. +Reuses `benchmarks.create_benchmark_processes` from the v2 package; does not +train, run self-play, or generate AI reports. +""" + +import argparse +import multiprocessing as mp +import os +import time +from pathlib import Path + +from v2 import benchmarks +from v2.common import ShutdownSignal +from v2.config import Config, load_user_config + +# Match train_v2.py: suppress wandb's "install weave" log spam. +os.environ["WANDB_DISABLE_WEAVE"] = "true" + + +def _derive_base_dir(run_dir: Path) -> str: + """Given a run dir laid out as `base_dir/runs//`, return `base_dir`. + + The run-dir convention used by `train_v2.py`'s `load_config_and_setup_run` + places each run under `/runs//`, so the parent of `runs/` + is the base_dir the rest of the v2 machinery expects. + """ + return str(run_dir.parent.parent) + + +def _load_config(run_dir: Path, overrides: list[str] | None) -> Config: + """Load `/config.yaml` and build a Config without touching disk. + + Uses `Config.from_user(..., create_dirs=False)` so the existing run directory + isn't disturbed and no `config.yaml` snapshot is rewritten. Raises + `FileNotFoundError` if the config file is missing. + """ + config_yaml = run_dir / "config.yaml" + if not config_yaml.is_file(): + raise FileNotFoundError(f"No config.yaml in {run_dir}") + user_config = load_user_config(str(config_yaml), overrides=overrides) + return Config.from_user(user_config, _derive_base_dir(run_dir), create_dirs=False) + + +def _check_run_dir(run_dir: Path) -> None: + """Verify the run directory has the layout we need before spawning processes. + + Aborts early on a missing `latest.yaml` so the benchmark processes don't enter + `LatestModel.wait_for_creation`'s blocking wait (no training is producing + models in this script). + """ + if not run_dir.is_dir(): + raise FileNotFoundError(f"Run directory not found: {run_dir}") + if not (run_dir / "config.yaml").is_file(): + raise FileNotFoundError(f"No config.yaml in {run_dir}") + latest_yaml = run_dir / "models" / "latest.yaml" + if not latest_yaml.is_file(): + raise FileNotFoundError(f"No models/latest.yaml in {run_dir}; the run has no trained model to benchmark.") + + +def main(args) -> int: + """Entry point. Returns the exit code.""" + run_dir = Path(args.run_dir).resolve() + _check_run_dir(run_dir) + config = _load_config(run_dir, args.overrides) + + if not config.benchmarks: + print(f"No benchmarks configured in {run_dir}/config.yaml; nothing to run.") + return 0 + + mp.set_start_method("spawn", force=True) + ShutdownSignal.clear(config) + + benchmark_processes = benchmarks.create_benchmark_processes(config) + for p in benchmark_processes: + p.start() + print(f"Started {len(benchmark_processes)} benchmark processes") + + try: + b_count_prev = -1 + while True: + b_count = sum(p.is_alive() for p in benchmark_processes) + if b_count != b_count_prev: + print(f"Waiting for {b_count} benchmark processes") + b_count_prev = b_count + if b_count == 0: + break + time.sleep(1) + except KeyboardInterrupt: + print("\nCaught Ctrl-C; signaling shutdown...") + ShutdownSignal.signal(config) + for p in benchmark_processes: + p.join() + + ShutdownSignal.clear(config) + return 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run just the benchmark schedules from an existing run's config.yaml.", + ) + parser.add_argument( + "run_dir", + type=str, + help="Path to an existing run directory (e.g. /path/to/runs//).", + ) + parser.add_argument( + "-o", + "--overrides", + nargs="*", + help="Configuration overrides (e.g., benchmarks.0.every=2 minutes).", + ) + args = parser.parse_args() + raise SystemExit(main(args)) diff --git a/deep_quoridor/src/train_v2.py b/deep_quoridor/src/train_v2.py index 9e5512be..6defb3fd 100644 --- a/deep_quoridor/src/train_v2.py +++ b/deep_quoridor/src/train_v2.py @@ -10,6 +10,7 @@ check_ai_available, load_config_and_setup_run, metrics_dir_for, + preload_symlinks, run_ai_reporter, run_selfplay_metrics, self_play, @@ -85,7 +86,17 @@ def _selfplay_subprocess_env(): # Make sure we don't have the shutdown signal from a previous run ShutdownSignal.clear(config) - train_process = mp.Process(target=train, args=[config]) + games_already_trained_on = 0 + if config.training.initial_replay_buffer is not None: + n_loaded = preload_symlinks( + source_run=Path(config.training.initial_replay_buffer.run), + dest_ready=config.paths.replay_buffers_ready, + buffer_size=config.training.replay_buffer_size, + ) + print(f"Preloaded {n_loaded} games from {config.training.initial_replay_buffer.run}") + games_already_trained_on = n_loaded + + train_process = mp.Process(target=train, args=[config, games_already_trained_on]) train_process.start() benchmark_processes = benchmarks.create_benchmark_processes(config) @@ -99,40 +110,41 @@ def _selfplay_subprocess_env(): self_play_processes = [] rust_subprocesses = [] - if config.self_play.program == "rust": - # Spawn Rust self-play processes in continuous mode - selfplay_env = _selfplay_subprocess_env() - if selfplay_env is not None: - print(f"Self-play GPU env: ORT_DYLIB_PATH={selfplay_env['ORT_DYLIB_PATH']}") - metrics_dir = metrics_dir_for(config) - os.makedirs(metrics_dir, exist_ok=True) - config_file_path = str(config.paths.config_file) - for i in range(config.self_play.num_processes): - cmd = [ - config.self_play.rust_selfplay_binary, - "--config", - config_file_path, - "--output-dir", - str(config.paths.replay_buffers_ready), - "--continuous", - "--latest-model-yaml", - str(config.paths.latest_model_yaml), - "--shutdown-file", - str(ShutdownSignal.file_path(config)), - "--metrics-dir", - metrics_dir, - ] - proc = subprocess.Popen(cmd, env=selfplay_env) - rust_subprocesses.append(proc) - print(f"Started Rust self-play process {proc.pid}") - selfplay_metrics_process = mp.Process(target=run_selfplay_metrics, args=[config]) - selfplay_metrics_process.start() - self_play_processes.append(selfplay_metrics_process) - else: - for i in range(config.self_play.num_processes): - p = mp.Process(target=self_play, args=[config]) - p.start() - self_play_processes.append(p) + if config.self_play.enabled: + if config.self_play.program == "rust": + # Spawn Rust self-play processes in continuous mode + selfplay_env = _selfplay_subprocess_env() + if selfplay_env is not None: + print(f"Self-play GPU env: ORT_DYLIB_PATH={selfplay_env['ORT_DYLIB_PATH']}") + metrics_dir = metrics_dir_for(config) + os.makedirs(metrics_dir, exist_ok=True) + config_file_path = str(config.paths.config_file) + for i in range(config.self_play.num_processes): + cmd = [ + config.self_play.rust_selfplay_binary, + "--config", + config_file_path, + "--output-dir", + str(config.paths.replay_buffers_ready), + "--continuous", + "--latest-model-yaml", + str(config.paths.latest_model_yaml), + "--shutdown-file", + str(ShutdownSignal.file_path(config)), + "--metrics-dir", + metrics_dir, + ] + proc = subprocess.Popen(cmd, env=selfplay_env) + rust_subprocesses.append(proc) + print(f"Started Rust self-play process {proc.pid}") + selfplay_metrics_process = mp.Process(target=run_selfplay_metrics, args=[config]) + selfplay_metrics_process.start() + self_play_processes.append(selfplay_metrics_process) + else: + for i in range(config.self_play.num_processes): + p = mp.Process(target=self_play, args=[config]) + p.start() + self_play_processes.append(p) train_process.join() ShutdownSignal.signal(config) diff --git a/deep_quoridor/src/v2/__init__.py b/deep_quoridor/src/v2/__init__.py index 2d52b649..cdf5aef1 100644 --- a/deep_quoridor/src/v2/__init__.py +++ b/deep_quoridor/src/v2/__init__.py @@ -15,6 +15,7 @@ "generate_on_demand_report", "metrics_dir_for", "run_selfplay_metrics", + "preload_symlinks", ] from v2.ai_report import check_ai_available, generate_on_demand_report, run_ai_reporter @@ -22,6 +23,7 @@ from v2.selfplay_metrics import metrics_dir_for, run_selfplay_metrics from v2.common import JobTrigger, MockWandb, ShutdownSignal, create_alphazero, upload_model from v2.config import load_config_and_setup_run +from v2.offline_preload import preload_symlinks from v2.self_play import self_play from v2.trainer import train from v2.yaml_models import GameInfo, LatestModel diff --git a/deep_quoridor/src/v2/common.py b/deep_quoridor/src/v2/common.py index 271e2193..4d154a8b 100644 --- a/deep_quoridor/src/v2/common.py +++ b/deep_quoridor/src/v2/common.py @@ -1,10 +1,12 @@ import re import time from abc import abstractmethod +from pathlib import Path from typing import Any, Callable, Optional import wandb from agents.alphazero import AlphaZeroAgent, AlphaZeroParams +from pydantic_yaml import parse_yaml_file_as from v2.config import AlphaZeroPlayConfig, AlphaZeroSelfPlayConfig, Config from v2.yaml_models import LatestModel @@ -135,11 +137,15 @@ def alphazero_params_dict_from_config( im = config.training.initial_model if im.file: params_dict["model_filename"] = im.file - if im.wandb_alias: + elif im.wandb_alias: params_dict["wandb_alias"] = im.wandb_alias params_dict["wandb_project"] = im.wandb_project or ( config.wandb.project if config.wandb else "deep_quoridor" ) + elif im.run: + latest_yaml = Path(im.run) / "models" / "latest.yaml" + latest = parse_yaml_file_as(LatestModel, latest_yaml) + params_dict["model_filename"] = latest.filename # Add network config if config.alphazero.network.type == "mlp": diff --git a/deep_quoridor/src/v2/config.py b/deep_quoridor/src/v2/config.py index e66b1b17..e6fc89ca 100644 --- a/deep_quoridor/src/v2/config.py +++ b/deep_quoridor/src/v2/config.py @@ -3,7 +3,7 @@ from typing import Annotated, Literal, Optional, Union import yaml -from pydantic import BaseModel, ConfigDict, Field, field_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator class StrictBaseModel(BaseModel): @@ -64,6 +64,7 @@ class AlphaZeroSelfPlayConfig(StrictBaseModel): class SelfPlayConfig(StrictBaseModel): + enabled: bool = True num_processes: int games_per_process: int # Leaf-parallel MCTS knobs (Rust self-play only). @@ -83,13 +84,29 @@ class InitialModel(StrictBaseModel): file: Optional[str] = None wandb_project: Optional[str] = None wandb_alias: Optional[str] = None + run: Optional[str] = None + + @model_validator(mode="after") + def at_most_one_source(self) -> "InitialModel": + sources = [ + ("file", self.file), + ("wandb_alias", self.wandb_alias), + ("run", self.run), + ] + set_sources = [name for name, val in sources if val is not None] + if len(set_sources) > 1: + raise ValueError(f"At most one of file, wandb_alias, run may be set in initial_model; got: {set_sources}") + return self + + +class InitialReplayBuffer(StrictBaseModel): + """Configures preloading the replay buffer from a previous run. + + `run` points at a run directory (parent of `replay_buffers/`), mirroring + `InitialModel.run`. At preload time the loader reads `/replay_buffers/`. + """ - @field_validator("wandb_alias") - @classmethod - def file_and_wandb_mutually_exclusive(cls, v, info): - if v is not None and info.data.get("file") is not None: - raise ValueError("Cannot specify both 'file' and 'wandb_alias' in initial_model") - return v + run: str class CosineWarmRestartsSchedulerConfig(StrictBaseModel): @@ -113,6 +130,7 @@ class TrainingConfig(StrictBaseModel): save_onnx: bool = False finish_after: Optional[str] = None initial_model: Optional[InitialModel] = None + initial_replay_buffer: Optional[InitialReplayBuffer] = None lr_scheduler: Optional[LRSchedulerConfig] = None @@ -196,6 +214,15 @@ def replace_datetime_placeholder(cls, v: str) -> str: return v.replace("$DATETIME", current_datetime) return v + @model_validator(mode="after") + def selfplay_off_requires_replay_buffer(self) -> "UserConfig": + if not self.self_play.enabled and self.training.initial_replay_buffer is None: + raise ValueError( + "When self_play.enabled is False, training.initial_replay_buffer must be set " + "(otherwise the trainer would hang forever waiting for games)." + ) + return self + class PathsConfig(StrictBaseModel): run_dir: Path @@ -372,7 +399,7 @@ def load_config_and_setup_run( with config_filename.open(mode="w") as f: f.write(to_yaml_str_ordered(user_config)) - use_rust = config.self_play.program == "rust" + use_rust = config.self_play.enabled and config.self_play.program == "rust" if use_rust: # Apply default Rust binary path if not specified in config if config.self_play.rust_selfplay_binary is None: diff --git a/deep_quoridor/src/v2/offline_preload.py b/deep_quoridor/src/v2/offline_preload.py new file mode 100644 index 00000000..35066123 --- /dev/null +++ b/deep_quoridor/src/v2/offline_preload.py @@ -0,0 +1,56 @@ +"""Preload selected games from a previous run's replay_buffers into a new run's ready/ dir. + +Used by `train_v2.py` when `config.training.initial_replay_buffer` is set, to seed the +replay buffer from a previous run's games (typically when training a new architecture +on the same lineage of self-play data). +""" + +from __future__ import annotations + +from pathlib import Path + + +def select_games(filenames: list[str], buffer_size: int) -> list[str]: + """Return the newest `buffer_size` filenames in chronological (ascending) order. + + Source replay-buffer filenames are monotonically numbered (`game_NNNNNNN.npz`), so + sorting ascending is chronological. If `filenames` has fewer than `buffer_size` + entries, returns them all. + """ + return sorted(filenames)[-buffer_size:] + + +def preload_symlinks(source_run: Path, dest_ready: Path, buffer_size: int) -> int: + """Symlink the newest source games (.npz + .yaml each) into `dest_ready`. + + Reads `/replay_buffers/` for `.npz` files, picks the newest + `buffer_size` games by filename (which the trainer trims by count, not by + move total), and creates symlinks (preserving source basenames) for both + files in `dest_ready`. + + Returns the number of games linked. Raises: + - FileNotFoundError if `/replay_buffers/` does not exist, or if any + selected `.npz` lacks its `.yaml` sidecar. + - ValueError if the source replay_buffers dir contains no `.npz` files. + """ + source_replay = Path(source_run) / "replay_buffers" + if not source_replay.is_dir(): + raise FileNotFoundError(f"Source replay_buffers dir not found: {source_replay}") + + npz_paths = sorted(source_replay.glob("*.npz")) + if not npz_paths: + raise ValueError(f"Source dir contains no .npz files: {source_replay}") + + selected = select_games([p.name for p in npz_paths], buffer_size) + + for name in selected: + npz_src = source_replay / name + yaml_src = npz_src.with_suffix(".yaml") + if not yaml_src.is_file(): + raise FileNotFoundError(f"Missing yaml sidecar: {yaml_src}") + npz_dst = Path(dest_ready) / name + yaml_dst = npz_dst.with_suffix(".yaml") + npz_dst.symlink_to(npz_src.resolve()) + yaml_dst.symlink_to(yaml_src.resolve()) + + return len(selected) diff --git a/deep_quoridor/src/v2/trainer.py b/deep_quoridor/src/v2/trainer.py index d36dff56..f701b2fc 100644 --- a/deep_quoridor/src/v2/trainer.py +++ b/deep_quoridor/src/v2/trainer.py @@ -53,6 +53,52 @@ def sample(self, game_filename: str, n: int): ] +def _should_skip_iteration( + total_moves: int, + batch_size: int, + games_needed_to_train: int, + last_game: int, + selfplay_disabled: bool, +) -> bool: + """Decide whether to skip this iteration of the trainer's main loop. + + Always skip when the buffer holds fewer moves than one batch. When self-play is + enabled, also skip when the trainer is ahead of self-play (the + `games_needed_to_train` gate). When self-play is disabled the buffer is static + and there is no production cadence to wait on, so we train every iteration once + enough moves are available. + """ + if total_moves < batch_size: + return True + if selfplay_disabled: + return False + return games_needed_to_train > last_game + + +def _build_game_log( + game_info, + model_version: int, + last_game: int, + omit_model_lag: bool, +) -> dict: + """Per-game wandb log payload emitted when a game is ingested from ready/. + + `model_lag` is meaningful only when games arrive from live self-play of the + current run, since it compares the trainer's current model version against the + version that *produced* the game. When games come from a preloaded buffer + (different lineage), the subtraction is nonsense, so the caller asks us to + omit the key. + """ + log = { + "game_length": game_info.game_length, + "Game num": last_game, + "Model version": model_version, + } + if not omit_model_lag: + log["model_lag"] = model_version - 1 - game_info.model_version + return log + + def model_uploader(config: Config, every: str, model_id: str, wandb_run, shutdown_event: threading.Event): LatestModel.wait_for_creation(config) @@ -70,8 +116,18 @@ def model_uploader(config: Config, every: str, model_id: str, wandb_run, shutdow trigger.wait(lambda: shutdown_event.is_set()) -def train(config: Config): +def train(config: Config, games_already_trained_on: int = 0): + """ + Main training loop. + + Args: + config: Config object + games_already_trained_on: used to determine when to train next. This is used when preloading a replay buffer from a + previous run, so that the trainer doesn't train on the same games multiple times. + """ batch_size = config.training.batch_size + selfplay_disabled = not config.self_play.enabled + omit_model_lag = config.training.initial_replay_buffer is not None alphazero_agent = create_alphazero(config, config.self_play.alphazero, overrides={"training_mode": True}) alphazero_agent.evaluator.setup_lr_scheduler(config.training.lr_scheduler) @@ -120,7 +176,7 @@ def train(config: Config): if config.training.finish_after: finish_condition = JobTrigger.from_string(config, config.training.finish_after) - training_steps = 0 + games_needed_to_train = games_already_trained_on + config.training.games_per_training_step last_game = 0 total_moves_played = 0 model_version = 1 @@ -155,14 +211,7 @@ def train(config: Config): moves_per_game.append(game_info.game_length) total_moves_played += game_info.game_length game_filename.append(new_name.name) - wandb_run.log( - { - "game_length": game_info.game_length, - "model_lag": model_version - 1 - game_info.model_version, - "Game num": last_game, - "Model version": model_version, - } - ) + wandb_run.log(_build_game_log(game_info, model_version, last_game, omit_model_lag)) # Trim oldest games to stay within the replay buffer size limit while len(moves_per_game) > config.training.replay_buffer_size: @@ -172,9 +221,13 @@ def train(config: Config): total_moves = sum(moves_per_game) - games_needed_to_train = config.training.games_per_training_step * (training_steps + 1) - - if total_moves < batch_size or games_needed_to_train > last_game: + if _should_skip_iteration( + total_moves=total_moves, + batch_size=batch_size, + games_needed_to_train=games_needed_to_train, + last_game=last_game, + selfplay_disabled=selfplay_disabled, + ): time.sleep(1) continue @@ -195,7 +248,7 @@ def train(config: Config): # Train the network for one step using the samples Timer.start("train") policy_loss, value_loss, total_loss = alphazero_agent.evaluator.train_iteration_v2(samples) - training_steps += 1 + games_needed_to_train += config.training.games_per_training_step time_train = Timer.finish("train") wandb_run.log( diff --git a/deep_quoridor/test/config_test.py b/deep_quoridor/test/config_test.py index 960ce2d5..952b5f1c 100644 --- a/deep_quoridor/test/config_test.py +++ b/deep_quoridor/test/config_test.py @@ -103,3 +103,116 @@ def test_invalid_override_format(config_file): def test_invalid_key_rejected_by_pydantic(config_file): with pytest.raises(Exception): load_user_config(config_file, overrides=["nonexistent_key=value"]) + + +def test_initial_model_run_accepted(config_file): + config = load_user_config(config_file, overrides=["training.initial_model.run=/some/old/run"]) + assert config.training.initial_model is not None + assert config.training.initial_model.run == "/some/old/run" + assert config.training.initial_model.file is None + assert config.training.initial_model.wandb_alias is None + + +def test_initial_model_rejects_file_plus_run(config_file): + with pytest.raises(Exception, match="initial_model"): + load_user_config( + config_file, + overrides=[ + "training.initial_model.file=/a.pt", + "training.initial_model.run=/some/old/run", + ], + ) + + +def test_initial_model_rejects_wandb_alias_plus_run(config_file): + with pytest.raises(Exception, match="initial_model"): + load_user_config( + config_file, + overrides=[ + "training.initial_model.wandb_alias=m1", + "training.initial_model.run=/some/old/run", + ], + ) + + +def test_initial_model_rejects_file_plus_wandb_alias(config_file): + # Existing behavior; restated under the new model_validator. + with pytest.raises(Exception, match="initial_model"): + load_user_config( + config_file, + overrides=[ + "training.initial_model.file=/a.pt", + "training.initial_model.wandb_alias=m1", + ], + ) + + +def test_initial_replay_buffer_accepted(config_file): + config = load_user_config(config_file, overrides=["training.initial_replay_buffer.run=/some/old/run"]) + assert config.training.initial_replay_buffer is not None + assert config.training.initial_replay_buffer.run == "/some/old/run" + + +def test_initial_replay_buffer_defaults_to_none(config_file): + config = load_user_config(config_file) + assert config.training.initial_replay_buffer is None + + +def test_source_run_field_no_longer_exists(config_file): + # Removed in favor of training.initial_replay_buffer. + with pytest.raises(Exception, match="source_run|extra"): + load_user_config(config_file, overrides=["training.source_run=/some/old/run"]) + + +def test_self_play_enabled_defaults_true(config_file): + config = load_user_config(config_file) + assert config.self_play.enabled is True + + +def test_self_play_enabled_can_be_false(config_file): + config = load_user_config( + config_file, + overrides=[ + "self_play.enabled=False", + "training.initial_replay_buffer.run=/some/old/run", + ], + ) + assert config.self_play.enabled is False + + +def test_selfplay_off_without_replay_buffer_is_rejected(config_file): + with pytest.raises(Exception, match="initial_replay_buffer"): + load_user_config(config_file, overrides=["self_play.enabled=False"]) + + +def test_initial_model_run_resolves_to_latest_filename(tmp_path): + """alphazero_params_dict_from_config translates initial_model.run into the + .pt filename recorded in /models/latest.yaml.""" + from pydantic_yaml import to_yaml_file + from v2.common import alphazero_params_dict_from_config + from v2.config import Config, load_user_config + from v2.yaml_models import LatestModel + + # Build a fake "old run" with a latest.yaml pointing at a model file. + old_run = tmp_path / "old_run" + models_dir = old_run / "models" + models_dir.mkdir(parents=True) + to_yaml_file( + models_dir / "latest.yaml", + LatestModel(filename=str(old_run / "models" / "checkpoints" / "model_42.pt"), version=42), + ) + + # Build a config that points initial_model.run at the fake run. + cfg_data = dict(EXAMPLE_CONFIG) + cfg_data["training"] = { + **EXAMPLE_CONFIG["training"], + "initial_model": {"run": str(old_run)}, + } + cfg_path = tmp_path / "config.yaml" + cfg_path.write_text(yaml.safe_dump(cfg_data, sort_keys=False)) + + user = load_user_config(str(cfg_path)) + config = Config.from_user(user, str(tmp_path), create_dirs=False) + + params = alphazero_params_dict_from_config(config) + assert params["model_filename"] == str(old_run / "models" / "checkpoints" / "model_42.pt") diff --git a/deep_quoridor/test/test_offline_preload.py b/deep_quoridor/test/test_offline_preload.py new file mode 100644 index 00000000..cd2fabc2 --- /dev/null +++ b/deep_quoridor/test/test_offline_preload.py @@ -0,0 +1,162 @@ +from pathlib import Path + +import numpy as np +import pytest +from pydantic_yaml import to_yaml_file + +from v2.offline_preload import preload_symlinks, select_games +from v2.yaml_models import GameInfo + + +def test_select_games_source_larger_than_buffer(): + # 5 games, buffer holds 3 games. Take the newest 3 in ascending order. + filenames = [ + "game_0000001.npz", + "game_0000002.npz", + "game_0000003.npz", + "game_0000004.npz", + "game_0000005.npz", + ] + result = select_games(filenames, buffer_size=3) + assert result == ["game_0000003.npz", "game_0000004.npz", "game_0000005.npz"] + + +def test_select_games_source_smaller_than_buffer(): + filenames = ["game_0000001.npz", "game_0000002.npz"] + result = select_games(filenames, buffer_size=100) + assert result == ["game_0000001.npz", "game_0000002.npz"] + + +def test_select_games_empty_source(): + assert select_games([], buffer_size=100) == [] + + +def test_select_games_buffer_equals_source_size(): + filenames = ["game_0000001.npz", "game_0000002.npz"] + result = select_games(filenames, buffer_size=2) + assert result == ["game_0000001.npz", "game_0000002.npz"] + + +def test_select_games_input_order_does_not_matter(): + # The function sorts internally, so any input order yields the same chronological result. + filenames = [ + "game_0000005.npz", + "game_0000001.npz", + "game_0000003.npz", + "game_0000002.npz", + "game_0000004.npz", + ] + result = select_games(filenames, buffer_size=3) + assert result == ["game_0000003.npz", "game_0000004.npz", "game_0000005.npz"] + + +def _make_source_game(source_replay_dir: Path, name: str, game_length: int, model_version: int = 0) -> None: + """Create a tiny .npz + .yaml sidecar pair, the same shape the real trainer writes.""" + npz_path = source_replay_dir / f"{name}.npz" + np.savez( + npz_path, + input_arrays=np.zeros((game_length, 1), dtype=np.float32), + policies=np.zeros((game_length, 1), dtype=np.float32), + action_masks=np.zeros((game_length, 1), dtype=np.float32), + values=np.zeros(game_length, dtype=np.float32), + players=np.zeros(game_length, dtype=np.int32), + ) + to_yaml_file( + source_replay_dir / f"{name}.yaml", + GameInfo(model_version=model_version, game_length=game_length, creator="test"), + ) + + +def _make_source_run(tmp_path: Path, num_games: int, moves_per_game: int) -> Path: + """Build a fake source run directory with `replay_buffers/` populated.""" + source_run = tmp_path / "source_run" + replay_dir = source_run / "replay_buffers" + replay_dir.mkdir(parents=True) + for i in range(1, num_games + 1): + _make_source_game(replay_dir, f"game_{i:07d}", moves_per_game, model_version=i) + return source_run + + +def test_preload_symlinks_creates_npz_and_yaml_symlinks(tmp_path): + source_run = _make_source_run(tmp_path, num_games=5, moves_per_game=10) + dest_ready = tmp_path / "new_run" / "ready" + dest_ready.mkdir(parents=True) + + count = preload_symlinks(source_run, dest_ready, buffer_size=3) + + # Newest 3 games are selected. + assert count == 3 + expected = {"game_0000003", "game_0000004", "game_0000005"} + npz_links = {p.stem for p in dest_ready.glob("*.npz")} + yaml_links = {p.stem for p in dest_ready.glob("*.yaml")} + assert npz_links == expected + assert yaml_links == expected + # All entries in dest_ready are symlinks, not copies. + for p in dest_ready.iterdir(): + assert p.is_symlink(), f"{p} is not a symlink" + + +def test_preload_symlinks_target_resolves_via_np_load(tmp_path): + source_run = _make_source_run(tmp_path, num_games=2, moves_per_game=5) + dest_ready = tmp_path / "new_run" / "ready" + dest_ready.mkdir(parents=True) + + preload_symlinks(source_run, dest_ready, buffer_size=100) + + # np.load through the symlink should yield the same arrays as the source. + link = dest_ready / "game_0000001.npz" + with np.load(link) as npz: + assert npz["values"].shape == (5,) + + +def test_preload_symlinks_source_smaller_than_buffer_takes_all(tmp_path): + source_run = _make_source_run(tmp_path, num_games=2, moves_per_game=3) + dest_ready = tmp_path / "new_run" / "ready" + dest_ready.mkdir(parents=True) + + count = preload_symlinks(source_run, dest_ready, buffer_size=1_000_000) + + assert count == 2 + + +def test_preload_symlinks_aborts_when_replay_buffers_missing(tmp_path): + source_run = tmp_path / "empty_run" + source_run.mkdir() + dest_ready = tmp_path / "new_run" / "ready" + dest_ready.mkdir(parents=True) + + with pytest.raises(FileNotFoundError, match="replay_buffers"): + preload_symlinks(source_run, dest_ready, buffer_size=10) + + +def test_preload_symlinks_aborts_when_replay_buffers_empty(tmp_path): + source_run = tmp_path / "empty_run" + (source_run / "replay_buffers").mkdir(parents=True) + dest_ready = tmp_path / "new_run" / "ready" + dest_ready.mkdir(parents=True) + + with pytest.raises(ValueError, match="no .npz files"): + preload_symlinks(source_run, dest_ready, buffer_size=10) + + +def test_preload_symlinks_aborts_when_yaml_sidecar_missing(tmp_path): + source_run = _make_source_run(tmp_path, num_games=2, moves_per_game=5) + # Delete one yaml sidecar to simulate corruption. + (source_run / "replay_buffers" / "game_0000002.yaml").unlink() + dest_ready = tmp_path / "new_run" / "ready" + dest_ready.mkdir(parents=True) + + with pytest.raises(FileNotFoundError, match="game_0000002.yaml"): + preload_symlinks(source_run, dest_ready, buffer_size=10) + + +def test_preload_symlinks_ignores_missing_yaml_for_non_selected_game(tmp_path): + # 5 source games, but buffer_size=2 selects only the newest 2. + # Delete the yaml for an oldest game (not selected) and confirm preload still succeeds. + source_run = _make_source_run(tmp_path, num_games=5, moves_per_game=10) + (source_run / "replay_buffers" / "game_0000001.yaml").unlink() + dest_ready = tmp_path / "new_run" / "ready" + dest_ready.mkdir(parents=True) + + count = preload_symlinks(source_run, dest_ready, buffer_size=2) + assert count == 2 diff --git a/deep_quoridor/test/test_run_benchmarks_v2.py b/deep_quoridor/test/test_run_benchmarks_v2.py new file mode 100644 index 00000000..494b32d0 --- /dev/null +++ b/deep_quoridor/test/test_run_benchmarks_v2.py @@ -0,0 +1,118 @@ +from argparse import Namespace +from pathlib import Path + +import pytest +import yaml + +from run_benchmarks_v2 import _check_run_dir, _derive_base_dir, _load_config, main + + +EXAMPLE_CONFIG = { + "run_id": "test-run", + "quoridor": {"board_size": 5, "max_walls": 3, "max_steps": 50}, + "alphazero": {"network": {"type": "mlp"}, "mcts_n": 300, "mcts_c_puct": 1.2}, + "self_play": {"num_processes": 2, "games_per_process": 16, "alphazero": {"mcts_noise_epsilon": 0.25}}, + "training": { + "games_per_training_step": 25.0, + "learning_rate": 0.001, + "batch_size": 256, + "weight_decay": 0.0001, + "replay_buffer_size": 1000000, + }, + "benchmarks": [ + { + "every": "10 models", + "jobs": [ + {"type": "tournament", "prefix": "raw", "times": 10, "opponents": ["random", "greedy"]}, + ], + }, + ], +} + + +def _make_run_dir(tmp_path: Path, run_id: str = "test-run") -> Path: + """Create a runs// structure with a valid config.yaml inside.""" + run_dir = tmp_path / "runs" / run_id + run_dir.mkdir(parents=True) + cfg = dict(EXAMPLE_CONFIG) + cfg["run_id"] = run_id + (run_dir / "config.yaml").write_text(yaml.safe_dump(cfg, sort_keys=False)) + return run_dir + + +def test_derive_base_dir_uses_grandparent(tmp_path): + run_dir = tmp_path / "runs" / "my-run" + assert _derive_base_dir(run_dir) == str(tmp_path) + + +def test_load_config_returns_full_config(tmp_path): + run_dir = _make_run_dir(tmp_path, run_id="my-run") + config = _load_config(run_dir, overrides=None) + assert config.run_id == "my-run" + assert config.training.learning_rate == 0.001 + assert len(config.benchmarks) == 1 + # paths derived from the run dir + assert config.paths.run_dir == run_dir + + +def test_load_config_applies_overrides(tmp_path): + run_dir = _make_run_dir(tmp_path) + config = _load_config(run_dir, overrides=["training.learning_rate=0.05"]) + assert config.training.learning_rate == 0.05 + + +def test_load_config_does_not_create_dirs(tmp_path): + """Config.from_user(..., create_dirs=False) — no replay_buffers/, etc. spawned.""" + run_dir = _make_run_dir(tmp_path) + _load_config(run_dir, overrides=None) + assert not (run_dir / "replay_buffers").exists() + assert not (run_dir / "models").exists() + + +def test_load_config_raises_when_config_yaml_missing(tmp_path): + run_dir = tmp_path / "runs" / "my-run" + run_dir.mkdir(parents=True) + with pytest.raises(FileNotFoundError, match="config.yaml"): + _load_config(run_dir, overrides=None) + + +def test_check_run_dir_raises_when_run_dir_missing(tmp_path): + with pytest.raises(FileNotFoundError, match="Run directory not found"): + _check_run_dir(tmp_path / "does-not-exist") + + +def test_check_run_dir_raises_when_config_yaml_missing(tmp_path): + run_dir = tmp_path / "runs" / "my-run" + run_dir.mkdir(parents=True) + with pytest.raises(FileNotFoundError, match="config.yaml"): + _check_run_dir(run_dir) + + +def test_check_run_dir_raises_when_latest_yaml_missing(tmp_path): + run_dir = _make_run_dir(tmp_path) + with pytest.raises(FileNotFoundError, match="models/latest.yaml"): + _check_run_dir(run_dir) + + +def test_check_run_dir_passes_when_all_present(tmp_path): + run_dir = _make_run_dir(tmp_path) + (run_dir / "models").mkdir() + (run_dir / "models" / "latest.yaml").write_text("filename: /tmp/m.pt\nversion: 0\n") + _check_run_dir(run_dir) # no exception + + +def test_main_exits_zero_when_no_benchmarks(tmp_path, capsys): + run_dir = _make_run_dir(tmp_path) + (run_dir / "models").mkdir() + (run_dir / "models" / "latest.yaml").write_text("filename: /tmp/m.pt\nversion: 0\n") + + # Strip the benchmarks section from the config.yaml. + cfg = yaml.safe_load((run_dir / "config.yaml").read_text()) + cfg["benchmarks"] = [] + (run_dir / "config.yaml").write_text(yaml.safe_dump(cfg, sort_keys=False)) + + args = Namespace(run_dir=str(run_dir), overrides=None) + exit_code = main(args) + assert exit_code == 0 + captured = capsys.readouterr() + assert "No benchmarks configured" in captured.out diff --git a/deep_quoridor/test/test_trainer_helpers.py b/deep_quoridor/test/test_trainer_helpers.py new file mode 100644 index 00000000..290cface --- /dev/null +++ b/deep_quoridor/test/test_trainer_helpers.py @@ -0,0 +1,108 @@ +from v2.trainer import _build_game_log, _should_skip_iteration +from v2.yaml_models import GameInfo + + +def test_should_skip_when_not_enough_moves(): + # Below batch_size: must skip regardless of mode. + assert ( + _should_skip_iteration( + total_moves=10, + batch_size=64, + games_needed_to_train=100, + last_game=100, + selfplay_disabled=False, + ) + is True + ) + assert ( + _should_skip_iteration( + total_moves=10, + batch_size=64, + games_needed_to_train=100, + last_game=100, + selfplay_disabled=True, + ) + is True + ) + + +def test_selfplay_on_honors_games_needed_gate(): + # Enough moves, but games_needed_to_train > last_game: skip. + assert ( + _should_skip_iteration( + total_moves=1000, + batch_size=64, + games_needed_to_train=100, + last_game=100, + selfplay_disabled=False, + ) + is False + ) # 100 == last_game; not greater, so train. + assert ( + _should_skip_iteration( + total_moves=1000, + batch_size=64, + games_needed_to_train=101, + last_game=100, + selfplay_disabled=False, + ) + is True + ) # 101 > 100; throttle. + + +def test_selfplay_off_skips_games_needed_gate(): + # Same parameters that would throttle now train. + assert ( + _should_skip_iteration( + total_moves=1000, + batch_size=64, + games_needed_to_train=101, + last_game=100, + selfplay_disabled=True, + ) + is False + ) + assert ( + _should_skip_iteration( + total_moves=1000, + batch_size=64, + games_needed_to_train=1_000_000, + last_game=100, + selfplay_disabled=True, + ) + is False + ) + + +def _gi(model_version: int, game_length: int) -> GameInfo: + return GameInfo(model_version=model_version, game_length=game_length, creator="test") + + +def test_build_game_log_includes_model_lag_by_default(): + log = _build_game_log( + game_info=_gi(model_version=5, game_length=42), + model_version=8, + last_game=123, + omit_model_lag=False, + ) + assert log == { + "game_length": 42, + "model_lag": 8 - 1 - 5, + "Game num": 123, + "Model version": 8, + } + + +def test_build_game_log_omits_model_lag_when_requested(): + log = _build_game_log( + game_info=_gi(model_version=5, game_length=42), + model_version=8, + last_game=123, + omit_model_lag=True, + ) + assert log == { + "game_length": 42, + "Game num": 123, + "Model version": 8, + } + assert "model_lag" not in log diff --git a/docs/superpowers/plans/2026-06-04-train-on-existing-selfplay-games.md b/docs/superpowers/plans/2026-06-04-train-on-existing-selfplay-games.md new file mode 100644 index 00000000..ef030dac --- /dev/null +++ b/docs/superpowers/plans/2026-06-04-train-on-existing-selfplay-games.md @@ -0,0 +1,1037 @@ +# Train on Existing Self-play Games — Implementation Plan + +> **AMENDMENT (2026-06-04, post-completion):** Tasks 2 and 3 of this plan implemented +> `replay_buffer_size` as a **moves** budget (cumulative `game_length` ≥ `buffer_size`). +> That was wrong — the trainer's existing trim at `trainer.py:208` (`len(moves_per_game) > +> replay_buffer_size`) treats it as a **count of games**. The follow-up commit +> "vibe: fix preload to use replay_buffer_size as games count" corrects the implementation +> and tests. See the (updated) spec at `docs/superpowers/specs/2026-06-04-train-on-existing-selfplay-design.md` +> for the authoritative semantics. The task descriptions below still show the original +> (incorrect) wording — preserved as a historical record. + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +I'm using AGENTS.md + +**Goal:** Add an "offline" mode to `train_v2.py` that consumes a previous run's stored self-play games as a fixed replay buffer, trains a new (fresh-weights) model on it, and runs benchmarks normally — no self-play subprocess, no new games arriving. + +**Architecture:** A new CLI flag `--source-run ` on `train_v2.py` is sugar for two config overrides — `training.source_run=` and `self_play.program=python`. When `config.training.source_run` is set, `train_v2.py` symlinks the newest source games into the new run's `replay_buffers/ready/` and skips spawning self-play workers; the trainer's existing pickup loop ingests them indistinguishably from live games. Two tiny conditionals in the trainer drop the `games_per_training_step` pacing gate and suppress the meaningless `model_lag` metric in offline mode. + +**Tech Stack:** Python 3.12, pydantic, pytest, numpy, pydantic-yaml. + +**Spec:** `docs/superpowers/specs/2026-06-04-train-on-existing-selfplay-design.md` + +--- + +## File structure + +- `deep_quoridor/src/v2/config.py` — add `source_run` field to `TrainingConfig`. (modify) +- `deep_quoridor/src/v2/offline_preload.py` — `select_games()` + `preload_symlinks()`. (create) +- `deep_quoridor/src/v2/__init__.py` — export `preload_symlinks`. (modify) +- `deep_quoridor/src/v2/trainer.py` — extract `_should_skip_iteration` and `_build_game_log` helpers; wire them in `train()`. (modify) +- `deep_quoridor/src/train_v2.py` — `--source-run` flag, override injection, preload call, skip self-play spawning. (modify) +- `deep_quoridor/test/config_test.py` — add `source_run` override test. (modify) +- `deep_quoridor/test/test_offline_preload.py` — unit tests for `select_games` + `preload_symlinks`. (create) +- `deep_quoridor/test/test_trainer_helpers.py` — unit tests for `_should_skip_iteration` + `_build_game_log`. (create) +- `deep_quoridor/test/test_train_v2_args.py` — unit tests for the override-injection helper. (create) + +**Run all Python tests with:** +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/ -v +``` + +**Commit style (AGENTS.md):** `vibe: ` imperative subject ≤ 50 chars. Keep functional changes in one commit and formatting/lint in a separate commit. Activate the venv when running Python. + +--- + +## Task 1: Add `source_run` field to TrainingConfig + +**Files:** +- Modify: `deep_quoridor/src/v2/config.py:105-117` (the `TrainingConfig` class) +- Modify: `deep_quoridor/test/config_test.py` (add one test) + +- [ ] **Step 1: Write the failing test** + +Append to `deep_quoridor/test/config_test.py`: + +```python +def test_override_source_run(config_file): + config = load_user_config(config_file, overrides=["training.source_run=/path/to/old/run"]) + assert config.training.source_run == "/path/to/old/run" + + +def test_source_run_defaults_to_none(config_file): + config = load_user_config(config_file) + assert config.training.source_run is None +``` + +- [ ] **Step 2: Run tests to confirm they fail** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/config_test.py::test_override_source_run deep_quoridor/test/config_test.py::test_source_run_defaults_to_none -v +``` + +Expected: `test_override_source_run` fails with a pydantic `extra="forbid"` validation error (unknown field `source_run`); `test_source_run_defaults_to_none` fails with `AttributeError: 'TrainingConfig' object has no attribute 'source_run'`. + +- [ ] **Step 3: Add the field to TrainingConfig** + +In `deep_quoridor/src/v2/config.py`, modify the `TrainingConfig` class (around line 105). Add `source_run` after `initial_model`: + +```python +class TrainingConfig(StrictBaseModel): + games_per_training_step: float + learning_rate: float + batch_size: int + weight_decay: float + replay_buffer_size: int + max_cached_games: int = 100000 + model_save_timing: bool = False + save_onnx: bool = False + finish_after: Optional[str] = None + initial_model: Optional[InitialModel] = None + source_run: Optional[str] = None + lr_scheduler: Optional[LRSchedulerConfig] = None +``` + +- [ ] **Step 4: Run tests to confirm they pass** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/config_test.py -v +``` + +Expected: all tests pass (the two new ones plus the existing 11). + +- [ ] **Step 5: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/v2/config.py deep_quoridor/test/config_test.py +git commit -m "vibe: add training.source_run config field" +``` + +--- + +## Task 2: `select_games()` — pure selector function + +**Files:** +- Create: `deep_quoridor/src/v2/offline_preload.py` +- Create: `deep_quoridor/test/test_offline_preload.py` + +- [ ] **Step 1: Write the failing tests** + +Create `deep_quoridor/test/test_offline_preload.py`: + +```python +import pytest + +from v2.offline_preload import select_games + + +def test_select_games_source_larger_than_buffer(): + # Source has 5 games totaling 50 moves; buffer holds 25 moves. + # Newest games are at the end; take from the end until cumulative >= 25. + entries = [ + ("game_0000001.npz", 10), + ("game_0000002.npz", 10), + ("game_0000003.npz", 10), + ("game_0000004.npz", 10), + ("game_0000005.npz", 10), + ] + result = select_games(entries, buffer_size=25) + # Newest 3 games (4, 5 wouldn't be enough; need 3 to reach >= 25). + # Returned in ascending (chronological) order. + assert result == ["game_0000003.npz", "game_0000004.npz", "game_0000005.npz"] + + +def test_select_games_source_smaller_than_buffer(): + entries = [ + ("game_0000001.npz", 10), + ("game_0000002.npz", 10), + ] + result = select_games(entries, buffer_size=100) + assert result == ["game_0000001.npz", "game_0000002.npz"] + + +def test_select_games_empty_source(): + assert select_games([], buffer_size=100) == [] + + +def test_select_games_exact_equal_cumulative(): + entries = [ + ("game_0000001.npz", 10), + ("game_0000002.npz", 10), + ] + # Newest one alone has exactly 10 moves; buffer wants >= 10. + result = select_games(entries, buffer_size=10) + assert result == ["game_0000002.npz"] + + +def test_select_games_input_order_does_not_matter(): + # The function sorts by filename internally, so any input order yields the + # same chronological result. + entries = [ + ("game_0000005.npz", 10), + ("game_0000001.npz", 10), + ("game_0000003.npz", 10), + ("game_0000002.npz", 10), + ("game_0000004.npz", 10), + ] + result = select_games(entries, buffer_size=25) + assert result == ["game_0000003.npz", "game_0000004.npz", "game_0000005.npz"] +``` + +- [ ] **Step 2: Run tests to confirm they fail** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_offline_preload.py -v +``` + +Expected: `ModuleNotFoundError: No module named 'v2.offline_preload'`. + +- [ ] **Step 3: Implement `select_games`** + +Create `deep_quoridor/src/v2/offline_preload.py`: + +```python +"""Preload selected games from a previous run's replay_buffers into a new run's ready/ dir. + +Used by `train_v2.py` when `--source-run` (config.training.source_run) is set, to seed +the replay buffer for a fresh-architecture training run without spawning self-play. +""" + +from __future__ import annotations + +from pathlib import Path + +from pydantic_yaml import parse_yaml_file_as + +from v2.yaml_models import GameInfo + + +def select_games(entries: list[tuple[str, int]], buffer_size: int) -> list[str]: + """Pick newest games (by filename) whose cumulative game_length covers `buffer_size`. + + `entries` is a list of (filename, game_length) pairs. The source numbers games + monotonically (`game_NNNNNNN.npz`), so sorting filenames ascending is chronological. + + Returns the selected filenames in ascending (chronological) order. If the source has + fewer total moves than `buffer_size`, returns every entry. + """ + sorted_asc = sorted(entries, key=lambda e: e[0]) + # Walk newest-first (descending), collect names until cumulative >= buffer_size. + selected: list[str] = [] + cumulative = 0 + for name, length in reversed(sorted_asc): + selected.append(name) + cumulative += length + if cumulative >= buffer_size: + break + # Return in ascending (chronological) order to match the trainer's ready/-sort. + selected.reverse() + return selected +``` + +- [ ] **Step 4: Run tests to confirm they pass** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_offline_preload.py -v +``` + +Expected: all 5 tests pass. + +- [ ] **Step 5: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/v2/offline_preload.py deep_quoridor/test/test_offline_preload.py +git commit -m "vibe: add select_games offline-preload helper" +``` + +--- + +## Task 3: `preload_symlinks()` — I/O function + +**Files:** +- Modify: `deep_quoridor/src/v2/offline_preload.py` +- Modify: `deep_quoridor/test/test_offline_preload.py` + +- [ ] **Step 1: Write the failing tests** + +Append to `deep_quoridor/test/test_offline_preload.py`: + +```python +import numpy as np +from pydantic_yaml import to_yaml_file + +from v2.offline_preload import preload_symlinks +from v2.yaml_models import GameInfo + + +def _make_source_game(source_replay_dir: Path, name: str, game_length: int, model_version: int = 0) -> None: + """Create a tiny .npz + .yaml sidecar pair, the same shape the real trainer writes.""" + npz_path = source_replay_dir / f"{name}.npz" + np.savez( + npz_path, + input_arrays=np.zeros((game_length, 1), dtype=np.float32), + policies=np.zeros((game_length, 1), dtype=np.float32), + action_masks=np.zeros((game_length, 1), dtype=np.float32), + values=np.zeros(game_length, dtype=np.float32), + players=np.zeros(game_length, dtype=np.int32), + ) + to_yaml_file( + source_replay_dir / f"{name}.yaml", + GameInfo(model_version=model_version, game_length=game_length, creator="test"), + ) + + +def _make_source_run(tmp_path: Path, num_games: int, moves_per_game: int) -> Path: + """Build a fake source run directory with `replay_buffers/` populated.""" + source_run = tmp_path / "source_run" + replay_dir = source_run / "replay_buffers" + replay_dir.mkdir(parents=True) + for i in range(1, num_games + 1): + _make_source_game(replay_dir, f"game_{i:07d}", moves_per_game, model_version=i) + return source_run + + +def test_preload_symlinks_creates_npz_and_yaml_symlinks(tmp_path): + source_run = _make_source_run(tmp_path, num_games=5, moves_per_game=10) + dest_ready = tmp_path / "new_run" / "ready" + dest_ready.mkdir(parents=True) + + count = preload_symlinks(source_run, dest_ready, buffer_size=25) + + # Newest 3 games (totaling 30 >= 25) are selected. + assert count == 3 + expected = {"game_0000003", "game_0000004", "game_0000005"} + npz_links = {p.stem for p in dest_ready.glob("*.npz")} + yaml_links = {p.stem for p in dest_ready.glob("*.yaml")} + assert npz_links == expected + assert yaml_links == expected + # All entries in dest_ready are symlinks, not copies. + for p in dest_ready.iterdir(): + assert p.is_symlink(), f"{p} is not a symlink" + + +def test_preload_symlinks_target_resolves_via_np_load(tmp_path): + source_run = _make_source_run(tmp_path, num_games=2, moves_per_game=5) + dest_ready = tmp_path / "new_run" / "ready" + dest_ready.mkdir(parents=True) + + preload_symlinks(source_run, dest_ready, buffer_size=100) + + # np.load through the symlink should yield the same arrays as the source. + link = dest_ready / "game_0000001.npz" + with np.load(link) as npz: + assert npz["values"].shape == (5,) + + +def test_preload_symlinks_source_smaller_than_buffer_takes_all(tmp_path): + source_run = _make_source_run(tmp_path, num_games=2, moves_per_game=3) + dest_ready = tmp_path / "new_run" / "ready" + dest_ready.mkdir(parents=True) + + count = preload_symlinks(source_run, dest_ready, buffer_size=1_000_000) + + assert count == 2 + + +def test_preload_symlinks_aborts_when_replay_buffers_missing(tmp_path): + source_run = tmp_path / "empty_run" + source_run.mkdir() + dest_ready = tmp_path / "new_run" / "ready" + dest_ready.mkdir(parents=True) + + with pytest.raises(FileNotFoundError, match="replay_buffers"): + preload_symlinks(source_run, dest_ready, buffer_size=10) + + +def test_preload_symlinks_aborts_when_replay_buffers_empty(tmp_path): + source_run = tmp_path / "empty_run" + (source_run / "replay_buffers").mkdir(parents=True) + dest_ready = tmp_path / "new_run" / "ready" + dest_ready.mkdir(parents=True) + + with pytest.raises(ValueError, match="no .npz files"): + preload_symlinks(source_run, dest_ready, buffer_size=10) + + +def test_preload_symlinks_aborts_when_yaml_sidecar_missing(tmp_path): + source_run = _make_source_run(tmp_path, num_games=2, moves_per_game=5) + # Delete one yaml sidecar to simulate corruption. + (source_run / "replay_buffers" / "game_0000002.yaml").unlink() + dest_ready = tmp_path / "new_run" / "ready" + dest_ready.mkdir(parents=True) + + with pytest.raises(FileNotFoundError, match="game_0000002.yaml"): + preload_symlinks(source_run, dest_ready, buffer_size=10) +``` + +- [ ] **Step 2: Run tests to confirm they fail** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_offline_preload.py -v +``` + +Expected: the 5 new tests fail with `ImportError` on `preload_symlinks` (the older 5 `select_games` tests still pass). + +- [ ] **Step 3: Implement `preload_symlinks`** + +Append to `deep_quoridor/src/v2/offline_preload.py`: + +```python +def preload_symlinks(source_run: Path, dest_ready: Path, buffer_size: int) -> int: + """Symlink the newest source games (.npz + .yaml each) into `dest_ready`. + + Reads `/replay_buffers/` for `.npz` files, parses each sibling `.yaml` for + its `game_length`, picks games newest-first until cumulative >= `buffer_size`, and + creates symlinks (preserving source basenames) for both files in `dest_ready`. + + Returns the number of games linked. Raises: + - FileNotFoundError if `/replay_buffers/` does not exist, or if any + selected `.npz` lacks its `.yaml` sidecar. + - ValueError if the source replay_buffers dir contains no `.npz` files. + """ + source_replay = Path(source_run) / "replay_buffers" + if not source_replay.is_dir(): + raise FileNotFoundError(f"Source replay_buffers dir not found: {source_replay}") + + npz_paths = sorted(source_replay.glob("*.npz")) + if not npz_paths: + raise ValueError(f"Source dir contains no .npz files: {source_replay}") + + # Build (name, game_length) entries; abort if any yaml sidecar is missing. + entries: list[tuple[str, int]] = [] + for npz_path in npz_paths: + yaml_path = npz_path.with_suffix(".yaml") + if not yaml_path.is_file(): + raise FileNotFoundError(f"Missing yaml sidecar: {yaml_path}") + info = parse_yaml_file_as(GameInfo, yaml_path) + entries.append((npz_path.name, info.game_length)) + + selected = select_games(entries, buffer_size) + + for name in selected: + npz_src = source_replay / name + yaml_src = npz_src.with_suffix(".yaml") + npz_dst = Path(dest_ready) / name + yaml_dst = npz_dst.with_suffix(".yaml") + npz_dst.symlink_to(npz_src.resolve()) + yaml_dst.symlink_to(yaml_src.resolve()) + + return len(selected) +``` + +- [ ] **Step 4: Run tests to confirm they pass** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_offline_preload.py -v +``` + +Expected: all 11 tests pass. + +- [ ] **Step 5: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/v2/offline_preload.py deep_quoridor/test/test_offline_preload.py +git commit -m "vibe: add preload_symlinks offline-preload helper" +``` + +--- + +## Task 4: Export `preload_symlinks` from `v2/__init__.py` + +**Files:** +- Modify: `deep_quoridor/src/v2/__init__.py` + +- [ ] **Step 1: Add the export** + +In `deep_quoridor/src/v2/__init__.py`, add `"preload_symlinks"` to `__all__` and add the import. The full file should be: + +```python +__all__ = [ + "load_config_and_setup_run", + "create_benchmark_processes", + "create_alphazero", + "LatestModel", + "JobTrigger", + "MockWandb", + "self_play", + "train", + "GameInfo", + "ShutdownSignal", + "upload_model", + "check_ai_available", + "run_ai_reporter", + "generate_on_demand_report", + "metrics_dir_for", + "run_selfplay_metrics", + "preload_symlinks", +] + +from v2.ai_report import check_ai_available, generate_on_demand_report, run_ai_reporter +from v2.benchmarks import create_benchmark_processes +from v2.selfplay_metrics import metrics_dir_for, run_selfplay_metrics +from v2.common import JobTrigger, MockWandb, ShutdownSignal, create_alphazero, upload_model +from v2.config import load_config_and_setup_run +from v2.offline_preload import preload_symlinks +from v2.self_play import self_play +from v2.trainer import train +from v2.yaml_models import GameInfo, LatestModel +``` + +- [ ] **Step 2: Sanity-check the import works** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/python -c "from v2 import preload_symlinks; print(preload_symlinks)" +``` + +Expected: prints ``. + +- [ ] **Step 3: Re-run the offline_preload tests** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_offline_preload.py -v +``` + +Expected: all 11 tests pass (this confirms the export change didn't break anything). + +- [ ] **Step 4: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/v2/__init__.py +git commit -m "vibe: export preload_symlinks from v2" +``` + +--- + +## Task 5: Trainer changes — extract helpers, wire offline mode + +**Files:** +- Modify: `deep_quoridor/src/v2/trainer.py` +- Create: `deep_quoridor/test/test_trainer_helpers.py` + +This task does TDD for two small extracted helpers (`_should_skip_iteration`, `_build_game_log`), then wires them into the trainer's loop. + +- [ ] **Step 1: Write failing tests for the helpers** + +Create `deep_quoridor/test/test_trainer_helpers.py`: + +```python +from v2.trainer import _build_game_log, _should_skip_iteration +from v2.yaml_models import GameInfo + + +def test_should_skip_when_not_enough_moves(): + # Below batch_size: must skip regardless of mode. + assert _should_skip_iteration( + total_moves=10, batch_size=64, games_per_training_step=1.0, + training_steps=0, last_game=100, offline_mode=False, + ) is True + assert _should_skip_iteration( + total_moves=10, batch_size=64, games_per_training_step=1.0, + training_steps=0, last_game=100, offline_mode=True, + ) is True + + +def test_online_mode_honors_games_per_step_gate(): + # Enough moves, but games_per_training_step * (steps+1) > last_game: skip. + assert _should_skip_iteration( + total_moves=1000, batch_size=64, games_per_training_step=1.0, + training_steps=99, last_game=100, offline_mode=False, + ) is False # 1.0 * 100 == last_game; not greater, so train. + assert _should_skip_iteration( + total_moves=1000, batch_size=64, games_per_training_step=1.0, + training_steps=100, last_game=100, offline_mode=False, + ) is True # 1.0 * 101 > 100; throttle. + + +def test_offline_mode_skips_games_per_step_gate(): + # Same parameters that would throttle in online mode now train. + assert _should_skip_iteration( + total_moves=1000, batch_size=64, games_per_training_step=1.0, + training_steps=100, last_game=100, offline_mode=True, + ) is False + # And keeps training even at very high step counts. + assert _should_skip_iteration( + total_moves=1000, batch_size=64, games_per_training_step=1.0, + training_steps=10_000, last_game=100, offline_mode=True, + ) is False + + +def _gi(model_version: int, game_length: int) -> GameInfo: + return GameInfo(model_version=model_version, game_length=game_length, creator="test") + + +def test_build_game_log_online_includes_model_lag(): + log = _build_game_log( + game_info=_gi(model_version=5, game_length=42), + model_version=8, last_game=123, offline_mode=False, + ) + assert log == { + "game_length": 42, + "model_lag": 8 - 1 - 5, + "Game num": 123, + "Model version": 8, + } + + +def test_build_game_log_offline_omits_model_lag(): + log = _build_game_log( + game_info=_gi(model_version=5, game_length=42), + model_version=8, last_game=123, offline_mode=True, + ) + assert log == { + "game_length": 42, + "Game num": 123, + "Model version": 8, + } + assert "model_lag" not in log +``` + +- [ ] **Step 2: Run tests to confirm they fail** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_trainer_helpers.py -v +``` + +Expected: ImportError on `_should_skip_iteration` and `_build_game_log`. + +- [ ] **Step 3: Add the helpers to trainer.py** + +In `deep_quoridor/src/v2/trainer.py`, just below the `Sampler` class and above `def model_uploader(...)`, add: + +```python +def _should_skip_iteration( + total_moves: int, + batch_size: int, + games_per_training_step: float, + training_steps: int, + last_game: int, + offline_mode: bool, +) -> bool: + """Decide whether to skip this iteration of the trainer's main loop. + + Always skip when the buffer holds fewer moves than one batch. In online mode also + skip when the trainer is ahead of self-play (the `games_per_training_step` gate). + In offline mode the buffer is static and there is no production cadence to wait on, + so we train every iteration once enough moves are available. + """ + if total_moves < batch_size: + return True + if offline_mode: + return False + games_needed_to_train = games_per_training_step * (training_steps + 1) + return games_needed_to_train > last_game + + +def _build_game_log( + game_info, + model_version: int, + last_game: int, + offline_mode: bool, +) -> dict: + """Per-game wandb log payload emitted when a game is ingested from ready/. + + `model_lag` is meaningful only when games arrive from live self-play, since it + compares the trainer's current model version against the version that *produced* + the game. In offline mode the source's `game_info.model_version` came from a + different training run and the subtraction is nonsense, so the key is omitted. + """ + log = { + "game_length": game_info.game_length, + "Game num": last_game, + "Model version": model_version, + } + if not offline_mode: + log["model_lag"] = model_version - 1 - game_info.model_version + return log +``` + +- [ ] **Step 4: Run helper tests to confirm they pass** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_trainer_helpers.py -v +``` + +Expected: all 6 tests pass. + +- [ ] **Step 5: Wire the helpers into `train()`** + +In `deep_quoridor/src/v2/trainer.py`, modify `train(config)`. Near the top of the function (after `batch_size = config.training.batch_size`), add: + +```python + offline_mode = config.training.source_run is not None +``` + +Replace the per-game log construction. The current code (around lines 158-165) reads: + +```python + wandb_run.log( + { + "game_length": game_info.game_length, + "model_lag": model_version - 1 - game_info.model_version, + "Game num": last_game, + "Model version": model_version, + } + ) +``` + +Change it to: + +```python + wandb_run.log(_build_game_log(game_info, model_version, last_game, offline_mode)) +``` + +Replace the gate. The current code (around lines 175-179) reads: + +```python + games_needed_to_train = config.training.games_per_training_step * (training_steps + 1) + + if total_moves < batch_size or games_needed_to_train > last_game: + time.sleep(1) + continue +``` + +Change it to: + +```python + if _should_skip_iteration( + total_moves=total_moves, + batch_size=batch_size, + games_per_training_step=config.training.games_per_training_step, + training_steps=training_steps, + last_game=last_game, + offline_mode=offline_mode, + ): + time.sleep(1) + continue +``` + +- [ ] **Step 6: Re-run helper tests and the full test directory to confirm nothing regressed** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_trainer_helpers.py deep_quoridor/test/config_test.py deep_quoridor/test/test_offline_preload.py deep_quoridor/test/test_selfplay_metrics.py -v +``` + +Expected: all tests pass. + +- [ ] **Step 7: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/v2/trainer.py deep_quoridor/test/test_trainer_helpers.py +git commit -m "vibe: wire offline mode into trainer loop" +``` + +--- + +## Task 6: `train_v2.py` — extract argument helper, add `--source-run` + +**Files:** +- Modify: `deep_quoridor/src/train_v2.py` +- Create: `deep_quoridor/test/test_train_v2_args.py` + +- [ ] **Step 1: Write the failing tests** + +Create `deep_quoridor/test/test_train_v2_args.py`: + +```python +import importlib + +# train_v2 is a top-level module under src/; import the helper directly. +train_v2 = importlib.import_module("train_v2") + + +def test_source_run_overrides_when_unset(): + assert train_v2.source_run_overrides(None) == [] + + +def test_source_run_overrides_when_set(): + result = train_v2.source_run_overrides("/path/to/old/run") + assert result == [ + "training.source_run=/path/to/old/run", + "self_play.program=python", + ] +``` + +- [ ] **Step 2: Run tests to confirm they fail** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_train_v2_args.py -v +``` + +Expected: `AttributeError: module 'train_v2' has no attribute 'source_run_overrides'`. + +- [ ] **Step 3: Add the helper to `train_v2.py`** + +In `deep_quoridor/src/train_v2.py`, just below the `_selfplay_subprocess_env` function (above `if __name__ == "__main__":`), add: + +```python +def source_run_overrides(source_run: str | None) -> list[str]: + """Build the config overrides implied by ``--source-run ``. + + When ``--source-run`` is set, the run executes in offline mode: no self-play + workers are spawned. We inject two overrides: + - ``training.source_run=`` (the single source of truth for "offline mode") + - ``self_play.program=python`` so ``load_config_and_setup_run`` doesn't reject the + run when the source's old config has ``program=rust`` but no rust binary is + available locally. + + Returns the empty list when ``source_run`` is None. + """ + if source_run is None: + return [] + return [ + f"training.source_run={source_run}", + "self_play.program=python", + ] +``` + +- [ ] **Step 4: Run tests to confirm they pass** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_train_v2_args.py -v +``` + +Expected: 2 tests pass. + +- [ ] **Step 5: Wire `--source-run` into the argparse and main flow** + +In `deep_quoridor/src/train_v2.py`, modify the `if __name__ == "__main__":` block. + +Add the new argparse flag (after the existing `--overrides` argument, around line 66): + +```python + parser.add_argument( + "--source-run", + type=str, + default=None, + help=( + "Run in offline mode: symlink the newest games from /replay_buffers/ " + "into this run's replay_buffers/ready/ and skip spawning self-play. " + "Use when training a new network architecture on a previous run's games." + ), + ) +``` + +Just after `args = parser.parse_args()` (around line 68), merge the source-run overrides: + +```python + extra_overrides = source_run_overrides(args.source_run) + if extra_overrides: + print(f"Offline mode: injecting overrides {extra_overrides}") + overrides = (args.overrides or []) + extra_overrides +``` + +Change the next line from `config = load_config_and_setup_run(args.config_file, runs_dir, overrides=args.overrides)` to: + +```python + config = load_config_and_setup_run(args.config_file, runs_dir, overrides=overrides) +``` + +Update the top of file imports — add `preload_symlinks` to the `from v2 import (...)` block: + +```python +from v2 import ( + benchmarks, + check_ai_available, + load_config_and_setup_run, + metrics_dir_for, + preload_symlinks, + run_ai_reporter, + run_selfplay_metrics, + self_play, + train, +) +``` + +After the `ShutdownSignal.clear(config)` call and **before** `train_process = mp.Process(target=train, args=[config])`, add the preload + skip-self-play decision: + +```python + offline_mode = config.training.source_run is not None + if offline_mode: + n_loaded = preload_symlinks( + source_run=Path(config.training.source_run), + dest_ready=config.paths.replay_buffers_ready, + buffer_size=config.training.replay_buffer_size, + ) + print(f"Offline mode: linked {n_loaded} games from {config.training.source_run}") +``` + +Then wrap the entire `if config.self_play.program == "rust":` / `else:` block (currently lines 102-135) so it is **skipped** when `offline_mode` is true: + +```python + if not offline_mode: + if config.self_play.program == "rust": + # ... existing rust branch unchanged ... + else: + # ... existing python branch unchanged ... +``` + +(`self_play_processes` and `rust_subprocesses` are initialized to `[]` immediately above this block in the existing code — those initializations stay outside the `if not offline_mode:` so the shutdown wait loop below still works.) + +- [ ] **Step 6: Sanity-check the script still parses** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/python -c "import train_v2; print('ok')" +``` + +Expected: prints `ok`. + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/python deep_quoridor/src/train_v2.py --help 2>&1 | head -30 +``` + +Expected: usage text including the `--source-run` flag. + +- [ ] **Step 7: Re-run the affected tests** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_train_v2_args.py deep_quoridor/test/test_offline_preload.py deep_quoridor/test/test_trainer_helpers.py deep_quoridor/test/config_test.py -v +``` + +Expected: all tests pass. + +- [ ] **Step 8: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/train_v2.py deep_quoridor/test/test_train_v2_args.py +git commit -m "vibe: add --source-run offline mode to train_v2" +``` + +--- + +## Task 7: End-to-end smoke test + +This task verifies the wired-together behavior against a real (tiny) source run. It does not add a test file — it is a manual procedure with explicit success criteria, because the end-to-end path involves `multiprocessing.Process` spawning and a real neural network init that's expensive to mock cleanly. Use any existing small training config in `deep_quoridor/experiments/` as the base, or use the snippet below. + +**Files:** +- No code changes. Verification only. + +- [ ] **Step 1: Prepare a fake "source" run dir** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +PYTHONPATH=deep_quoridor/src .venv/bin/python - <<'PY' +from pathlib import Path +import numpy as np +from pydantic_yaml import to_yaml_file +from v2.yaml_models import GameInfo + +source = Path("/tmp/smoke_source_run/replay_buffers") +source.mkdir(parents=True, exist_ok=True) + +# Five small games matching board_size=5, max_walls=3 shape would normally be +# required for real training. For this smoke test we won't actually train -- +# we'll let train_v2 boot, preload, then Ctrl-C. So array shapes don't matter +# beyond what the trainer's ingestion needs (game_length from yaml; npz can be +# minimal). +for i in range(1, 6): + game_length = 8 + np.savez( + source / f"game_{i:07d}.npz", + input_arrays=np.zeros((game_length, 1), dtype=np.float32), + policies=np.zeros((game_length, 1), dtype=np.float32), + action_masks=np.zeros((game_length, 1), dtype=np.float32), + values=np.zeros(game_length, dtype=np.float32), + players=np.zeros(game_length, dtype=np.int32), + ) + to_yaml_file( + source / f"game_{i:07d}.yaml", + GameInfo(model_version=i, game_length=game_length, creator="smoke"), + ) +print("source_run prepared at /tmp/smoke_source_run") +PY +``` + +- [ ] **Step 2: Create a minimal config** + +Save this to `/tmp/smoke_config.yaml`: + +```yaml +run_id: smoke-offline-$DATETIME +quoridor: + board_size: 5 + max_walls: 3 + max_steps: 50 +alphazero: + network: + type: mlp + mcts_n: 25 + mcts_c_puct: 1.2 +self_play: + num_processes: 1 + games_per_process: 4 + alphazero: + mcts_noise_epsilon: 0.25 +training: + games_per_training_step: 1.0 + learning_rate: 0.001 + batch_size: 32 + weight_decay: 0.0001 + replay_buffer_size: 30 + finish_after: "3 models" +``` + +- [ ] **Step 3: Run train_v2 in offline mode** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/python deep_quoridor/src/train_v2.py /tmp/smoke_config.yaml -r /tmp/smoke_runs --source-run /tmp/smoke_source_run 2>&1 | tee /tmp/smoke_run.log +``` + +(Stop after a few seconds with Ctrl-C if it doesn't reach `finish_after`; the array shapes in the fake .npz won't satisfy the real network, but startup logging will show whether offline mode wired up correctly.) + +- [ ] **Step 4: Verify the smoke test output** + +Check the log: +```bash +grep -E "Offline mode|Started Rust self-play|linked .* games from" /tmp/smoke_run.log +``` + +Expected to see: +- `Offline mode: injecting overrides ['training.source_run=/tmp/smoke_source_run', 'self_play.program=python']` +- `Offline mode: linked N games from /tmp/smoke_source_run` (N between 3 and 5 — newest until cumulative ≥ 30 moves) +- No `Started Rust self-play process` lines. + +Verify the symlinks landed in the new run: +```bash +ls -la /tmp/smoke_runs/runs/smoke-offline-*/replay_buffers/ready/ 2>/dev/null || \ + ls -la /tmp/smoke_runs/runs/smoke-offline-*/replay_buffers/ +``` + +Expected: symlinks (shown by `ls -la` as `->`) targeting `/tmp/smoke_source_run/replay_buffers/game_*.npz` and `.yaml` files. Files may have been already moved into `replay_buffers/` (from `ready/`) by the trainer's pickup loop. + +- [ ] **Step 5: Cleanup** + +```bash +rm -rf /tmp/smoke_source_run /tmp/smoke_runs /tmp/smoke_config.yaml /tmp/smoke_run.log +``` + +- [ ] **Step 6: Commit (formatting / lint pass if any)** + +Run formatters across the touched files: + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && .venv/bin/ruff format deep_quoridor/src/v2/config.py deep_quoridor/src/v2/offline_preload.py deep_quoridor/src/v2/__init__.py deep_quoridor/src/v2/trainer.py deep_quoridor/src/train_v2.py deep_quoridor/test/test_offline_preload.py deep_quoridor/test/test_trainer_helpers.py deep_quoridor/test/test_train_v2_args.py deep_quoridor/test/config_test.py +``` + +If anything reformatted, commit it separately per AGENTS.md: + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add -u +git diff --cached --quiet || git commit -m "vibe: ruff format" +``` + +--- + +## Done criteria + +- All unit tests in `test_offline_preload.py`, `test_trainer_helpers.py`, `test_train_v2_args.py`, and the new `config_test.py` cases pass. +- `train_v2.py --source-run ` boots, logs the override injection and symlink count, does not spawn any rust self-play subprocess, and proceeds into the training loop. +- Existing tests in `config_test.py`, `test_selfplay_metrics.py` still pass. +- Files left clean by `ruff format`. diff --git a/docs/superpowers/plans/2026-06-05-train-from-previous-run.md b/docs/superpowers/plans/2026-06-05-train-from-previous-run.md new file mode 100644 index 00000000..a233b06f --- /dev/null +++ b/docs/superpowers/plans/2026-06-05-train-from-previous-run.md @@ -0,0 +1,1139 @@ +# Train from a Previous Run Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +I'm using AGENTS.md + +**Goal:** Replace the single `--source-run` mechanism (shipped yesterday) with three independent config knobs — `training.initial_model.run`, `training.initial_replay_buffer.run`, and `self_play.enabled` — so each concern can be toggled separately and mixed-mode (preload + self-play on) becomes a legitimate combination. + +**Architecture:** Three additive schema changes (one new `InitialReplayBuffer` sub-model, a new `run` option on the existing `InitialModel`, and an `enabled` flag on `SelfPlayConfig`). Two cross-field validators (three-way mutual exclusion on `InitialModel` sources; refuse `enabled=False` with no replay buffer source). The trainer's two existing helpers (`_should_skip_iteration`, `_build_game_log`) keep their shapes but their bool params rename to reflect the split concerns. `train_v2.py` loses its `--source-run` CLI flag and the `source_run_overrides` helper; everything drives off config. + +**Tech Stack:** Python 3.12, pydantic, pytest, pydantic-yaml. + +**Spec:** `docs/superpowers/specs/2026-06-05-train-from-previous-run-design.md` + +--- + +## File structure + +- `deep_quoridor/src/v2/config.py` — `InitialModel.run` field + 3-way validator; new `InitialReplayBuffer` sub-model; remove `TrainingConfig.source_run`; add `TrainingConfig.initial_replay_buffer`; add `SelfPlayConfig.enabled`; UserConfig-level validator; rust-binary check gating. (modify) +- `deep_quoridor/src/v2/common.py` — `alphazero_params_dict_from_config`: resolve `initial_model.run` to the latest checkpoint via `LatestModel`. (modify) +- `deep_quoridor/src/v2/trainer.py` — rename helper bool params (`offline_mode` → `selfplay_disabled` / `omit_model_lag`); wire from new config fields. (modify) +- `deep_quoridor/src/train_v2.py` — remove `--source-run` flag and `source_run_overrides`; derive two booleans from config; wrap self-play block accordingly. (modify) +- `deep_quoridor/test/config_test.py` — replace `source_run` tests with `initial_replay_buffer` tests; add `initial_model.run` + 3-way exclusion tests; add `self_play.enabled` tests + the no-source-of-games validator test. (modify) +- `deep_quoridor/test/test_trainer_helpers.py` — rename bool params in the 5 existing test cases. (modify) +- `deep_quoridor/test/test_train_v2_args.py` — **delete** (its only purpose was the removed `source_run_overrides` helper). (delete) + +**Run Python tests with:** +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/ -v +``` + +**Commit style (AGENTS.md):** `vibe: ` imperative subject ≤ 50 chars. Functional vs formatting changes in separate commits. Activate the venv when running Python. + +**Branch state:** This continues on `jdb/train-on-existing-selfplay-games`. The just-shipped `source_run` mechanism is on the same branch (commit `31ff54e` and follow-ups); we replace it in place — no main has ever seen it. + +--- + +## Task 1: `InitialModel.run` + 3-way mutual exclusion validator + +**Files:** +- Modify: `deep_quoridor/src/v2/config.py:82-92` (the `InitialModel` class) +- Modify: `deep_quoridor/test/config_test.py` + +- [ ] **Step 1: Write the failing tests** + +Append to `deep_quoridor/test/config_test.py`. The override parser supports dotted keys with auto-created intermediate dicts (see `_apply_overrides` → `_ensure_and_navigate`), so we can target `training.initial_model.run=...` directly without needing a dict literal. + +```python +def test_initial_model_run_accepted(config_file): + config = load_user_config( + config_file, overrides=["training.initial_model.run=/some/old/run"] + ) + assert config.training.initial_model is not None + assert config.training.initial_model.run == "/some/old/run" + assert config.training.initial_model.file is None + assert config.training.initial_model.wandb_alias is None + + +def test_initial_model_rejects_file_plus_run(config_file): + with pytest.raises(Exception, match="initial_model"): + load_user_config( + config_file, + overrides=[ + "training.initial_model.file=/a.pt", + "training.initial_model.run=/some/old/run", + ], + ) + + +def test_initial_model_rejects_wandb_alias_plus_run(config_file): + with pytest.raises(Exception, match="initial_model"): + load_user_config( + config_file, + overrides=[ + "training.initial_model.wandb_alias=m1", + "training.initial_model.run=/some/old/run", + ], + ) + + +def test_initial_model_rejects_file_plus_wandb_alias(config_file): + # Existing behavior; restated under the new model_validator. + with pytest.raises(Exception, match="initial_model"): + load_user_config( + config_file, + overrides=[ + "training.initial_model.file=/a.pt", + "training.initial_model.wandb_alias=m1", + ], + ) +``` + +`pytest` is already imported in the file. + +- [ ] **Step 2: Run the new tests to confirm they fail** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/config_test.py::test_initial_model_run_accepted deep_quoridor/test/config_test.py::test_initial_model_rejects_file_plus_run deep_quoridor/test/config_test.py::test_initial_model_rejects_wandb_alias_plus_run -v +``` + +Expected: `test_initial_model_run_accepted` fails with `extra_forbidden` (unknown field `run`); the two `rejects_*` tests with `run` may fail or pass depending on whether the field exists. Either way they will be correct after Step 3. + +- [ ] **Step 3: Replace `InitialModel`'s validator with a 3-way `model_validator`** + +In `deep_quoridor/src/v2/config.py`, ensure `model_validator` is imported from pydantic at the top: + +```python +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +``` + +Replace the `InitialModel` class (lines 82-92) with: + +```python +class InitialModel(StrictBaseModel): + file: Optional[str] = None + wandb_project: Optional[str] = None + wandb_alias: Optional[str] = None + run: Optional[str] = None + + @model_validator(mode="after") + def at_most_one_source(self) -> "InitialModel": + sources = [ + ("file", self.file), + ("wandb_alias", self.wandb_alias), + ("run", self.run), + ] + set_sources = [name for name, val in sources if val is not None] + if len(set_sources) > 1: + raise ValueError( + "At most one of file, wandb_alias, run may be set in initial_model; " + f"got: {set_sources}" + ) + return self +``` + +This replaces the old `file_and_wandb_mutually_exclusive` field_validator. The new validator covers the same 2-way case and the two new pair cases involving `run`. + +- [ ] **Step 4: Run the full `config_test.py` to confirm everything passes** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/config_test.py -v +``` + +Expected: all tests pass (the 4 new ones plus the pre-existing 15). + +- [ ] **Step 5: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/v2/config.py deep_quoridor/test/config_test.py +git commit -m "vibe: add initial_model.run + 3-way exclusion" +``` + +End the commit message with the Co-Authored-By trailer. + +--- + +## Task 2: `InitialReplayBuffer` sub-model + replace `source_run` + +**Files:** +- Modify: `deep_quoridor/src/v2/config.py` (TrainingConfig) +- Modify: `deep_quoridor/test/config_test.py` + +- [ ] **Step 1: Write the failing test (and remove the old `source_run` tests)** + +In `deep_quoridor/test/config_test.py`, **delete** these two tests (lines around 108-115): +```python +def test_override_source_run(config_file): + ... +def test_source_run_defaults_to_none(config_file): + ... +``` + +Append these new tests: + +```python +def test_initial_replay_buffer_accepted(config_file): + config = load_user_config( + config_file, overrides=["training.initial_replay_buffer.run=/some/old/run"] + ) + assert config.training.initial_replay_buffer is not None + assert config.training.initial_replay_buffer.run == "/some/old/run" + + +def test_initial_replay_buffer_defaults_to_none(config_file): + config = load_user_config(config_file) + assert config.training.initial_replay_buffer is None + + +def test_source_run_field_no_longer_exists(config_file): + # Removed in favor of training.initial_replay_buffer. + with pytest.raises(Exception, match="source_run|extra"): + load_user_config(config_file, overrides=["training.source_run=/some/old/run"]) +``` + +- [ ] **Step 2: Run new tests to confirm they fail** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/config_test.py::test_initial_replay_buffer_accepted deep_quoridor/test/config_test.py::test_initial_replay_buffer_defaults_to_none deep_quoridor/test/config_test.py::test_source_run_field_no_longer_exists -v +``` + +Expected: the first two fail with `extra_forbidden` for `initial_replay_buffer`; the third passes today (source_run still exists) but will need to start failing after Step 3 — so it confirms the schema change actually took effect. + +- [ ] **Step 3: Add `InitialReplayBuffer` and replace `source_run` in `TrainingConfig`** + +In `deep_quoridor/src/v2/config.py`, just above the `CosineWarmRestartsSchedulerConfig` class (around line 95), add: + +```python +class InitialReplayBuffer(StrictBaseModel): + """Configures preloading the replay buffer from a previous run. + + `run` points at a run directory (parent of `replay_buffers/`), mirroring + `InitialModel.run`. At preload time the loader reads `/replay_buffers/`. + """ + + run: str +``` + +In `TrainingConfig` (around line 105-117), replace the line `source_run: Optional[str] = None` with: + +```python + initial_replay_buffer: Optional[InitialReplayBuffer] = None +``` + +Final `TrainingConfig`: + +```python +class TrainingConfig(StrictBaseModel): + games_per_training_step: float + learning_rate: float + batch_size: int + weight_decay: float + replay_buffer_size: int + max_cached_games: int = 100000 + model_save_timing: bool = False + save_onnx: bool = False + finish_after: Optional[str] = None + initial_model: Optional[InitialModel] = None + initial_replay_buffer: Optional[InitialReplayBuffer] = None + lr_scheduler: Optional[LRSchedulerConfig] = None +``` + +- [ ] **Step 4: Run `config_test.py` to confirm everything passes** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/config_test.py -v +``` + +Expected: all tests pass. The two old `source_run` tests are gone; three new tests in place. + +- [ ] **Step 5: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/v2/config.py deep_quoridor/test/config_test.py +git commit -m "vibe: replace source_run with initial_replay_buffer" +``` + +End with the Co-Authored-By trailer. + +--- + +## Task 3: `self_play.enabled` field + Config validators + rust-binary gate + +**Files:** +- Modify: `deep_quoridor/src/v2/config.py` (SelfPlayConfig, UserConfig, `load_config_and_setup_run`) +- Modify: `deep_quoridor/test/config_test.py` + +- [ ] **Step 1: Write the failing tests** + +Append to `deep_quoridor/test/config_test.py`: + +```python +def test_self_play_enabled_defaults_true(config_file): + config = load_user_config(config_file) + assert config.self_play.enabled is True + + +def test_self_play_enabled_can_be_false(config_file): + config = load_user_config( + config_file, + overrides=[ + "self_play.enabled=False", + "training.initial_replay_buffer.run=/some/old/run", + ], + ) + assert config.self_play.enabled is False + + +def test_selfplay_off_without_replay_buffer_is_rejected(config_file): + with pytest.raises(Exception, match="initial_replay_buffer"): + load_user_config(config_file, overrides=["self_play.enabled=False"]) +``` + +- [ ] **Step 2: Run new tests to confirm they fail** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/config_test.py::test_self_play_enabled_defaults_true deep_quoridor/test/config_test.py::test_self_play_enabled_can_be_false deep_quoridor/test/config_test.py::test_selfplay_off_without_replay_buffer_is_rejected -v +``` + +Expected: all three fail with `extra_forbidden` for the unknown `enabled` field. + +- [ ] **Step 3: Add `enabled` to `SelfPlayConfig` and the Config-level validator** + +In `deep_quoridor/src/v2/config.py`, modify the `SelfPlayConfig` class (around lines 66-79). Add `enabled: bool = True` near the top of the field list: + +```python +class SelfPlayConfig(StrictBaseModel): + enabled: bool = True + num_processes: int + games_per_process: int + # Leaf-parallel MCTS knobs (Rust self-play only). + leaf_parallelism: int = 16 + virtual_loss: int = 3 + enable_tree_reuse: bool = True + mcts_worker_threads: Optional[int] = None + eval_batch_size: int = 2048 + eval_max_wait_ms: int = 0 + eval_cache_max_size: int = 100000 + alphazero: Optional[AlphaZeroSelfPlayConfig] = None + program: Literal["python", "rust"] = "python" + rust_selfplay_binary: Optional[str] = None +``` + +In the `UserConfig` class (around lines 178-198), add a `model_validator(mode="after")` after the existing `replace_datetime_placeholder` field_validator. The full class becomes: + +```python +class UserConfig(StrictBaseModel): + """A normal pydantic model that can be used as an inner class.""" + + run_id: str + quoridor: QuoridorConfig + alphazero: AlphaZeroBaseConfig + wandb: Optional[WandbConfig] = None + self_play: SelfPlayConfig + training: TrainingConfig + benchmarks: list[BenchmarkScheduleConfig] = [] + ai_report: Optional[AIReportConfig] = None + + @field_validator("run_id") + @classmethod + def replace_datetime_placeholder(cls, v: str) -> str: + """Replace $DATETIME with current datetime in format YYYYMMDD-HHMM.""" + if "$DATETIME" in v: + current_datetime = datetime.now().strftime("%Y%m%d-%H%M") + return v.replace("$DATETIME", current_datetime) + return v + + @model_validator(mode="after") + def selfplay_off_requires_replay_buffer(self) -> "UserConfig": + if not self.self_play.enabled and self.training.initial_replay_buffer is None: + raise ValueError( + "When self_play.enabled is False, training.initial_replay_buffer must be set " + "(otherwise the trainer would hang forever waiting for games)." + ) + return self +``` + +- [ ] **Step 4: Gate the rust-binary check on `enabled`** + +In `deep_quoridor/src/v2/config.py`, find `load_config_and_setup_run` (around line 362-390). The current code reads: + +```python + use_rust = config.self_play.program == "rust" + if use_rust: + # Apply default Rust binary path if not specified in config + if config.self_play.rust_selfplay_binary is None: + config.self_play.rust_selfplay_binary = str( + Path(__file__).parent.parent.parent / "rust" / "target" / "release" / "selfplay" + ) + rust_binary = config.self_play.rust_selfplay_binary + if not Path(rust_binary).exists(): + print(f"ERROR: Rust self-play binary not found at {rust_binary}") + print("Build it with: cd deep_quoridor/rust && cargo build --release --features binary --bin selfplay") + exit(1) + # Rust self-play requires ONNX model exports + config.training.save_onnx = True +``` + +Change the first line to: + +```python + use_rust = config.self_play.enabled and config.self_play.program == "rust" +``` + +Rest unchanged. When self-play is disabled, no rust binary is needed. + +- [ ] **Step 5: Run `config_test.py` to confirm everything passes** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/config_test.py -v +``` + +Expected: all tests pass. + +- [ ] **Step 6: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/v2/config.py deep_quoridor/test/config_test.py +git commit -m "vibe: add self_play.enabled + cross-field guards" +``` + +End with the Co-Authored-By trailer. + +--- + +## Task 4: Resolve `initial_model.run` in `alphazero_params_dict_from_config` + +**Files:** +- Modify: `deep_quoridor/src/v2/common.py:134-142` (the initial_model branch in `alphazero_params_dict_from_config`) +- Test: covered by Task 7 smoke verification (a unit test for this would require mocking the AlphaZero model load, which is heavyweight and low-value; the smoke run with `initial_model.run` set covers it end-to-end). Add one targeted unit test of the resolution helper alone — see Step 1. + +- [ ] **Step 1: Write the failing test** + +Append to `deep_quoridor/test/config_test.py` (kept in the same test file since it's a small assertion on config-derived behavior): + +```python +def test_initial_model_run_resolves_to_latest_filename(tmp_path): + """alphazero_params_dict_from_config translates initial_model.run into the + .pt filename recorded in /models/latest.yaml.""" + from pydantic_yaml import to_yaml_file + from v2.common import alphazero_params_dict_from_config + from v2.config import Config, load_user_config + from v2.yaml_models import LatestModel + + # Build a fake "old run" with a latest.yaml pointing at a model file. + old_run = tmp_path / "old_run" + models_dir = old_run / "models" + models_dir.mkdir(parents=True) + to_yaml_file( + models_dir / "latest.yaml", + LatestModel(filename=str(old_run / "models" / "checkpoints" / "model_42.pt"), version=42), + ) + + # Build a config that points initial_model.run at the fake run. + cfg_data = dict(EXAMPLE_CONFIG) + cfg_data["training"] = { + **EXAMPLE_CONFIG["training"], + "initial_model": {"run": str(old_run)}, + } + cfg_path = tmp_path / "config.yaml" + cfg_path.write_text(yaml.safe_dump(cfg_data, sort_keys=False)) + + user = load_user_config(str(cfg_path)) + config = Config.from_user(user, str(tmp_path), create_dirs=False) + + params = alphazero_params_dict_from_config(config) + assert params["model_filename"] == str(old_run / "models" / "checkpoints" / "model_42.pt") +``` + +(Imports inside the test keep them local; `EXAMPLE_CONFIG` and `yaml` are already imported at the top of `config_test.py`.) + +- [ ] **Step 2: Run the new test to confirm it fails** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/config_test.py::test_initial_model_run_resolves_to_latest_filename -v +``` + +Expected: passes the config validation (Task 1 added `run`) but `params["model_filename"]` is not set — KeyError or missing key. The current `if im.file: ...; if im.wandb_alias: ...` branches don't handle `run`. + +- [ ] **Step 3: Wire `run` in `alphazero_params_dict_from_config`** + +In `deep_quoridor/src/v2/common.py`, the file already imports `LatestModel` (line 9). Add `parse_yaml_file_as` and `Path` if not present. Current imports at the top of `common.py`: + +```python +import re +import time +from abc import abstractmethod +from typing import Any, Callable, Optional + +import wandb +from agents.alphazero import AlphaZeroAgent, AlphaZeroParams +from v2.config import AlphaZeroPlayConfig, AlphaZeroSelfPlayConfig, Config +from v2.yaml_models import LatestModel +``` + +Add: + +```python +from pathlib import Path + +from pydantic_yaml import parse_yaml_file_as +``` + +Then modify the `initial_model` block inside `alphazero_params_dict_from_config` (currently lines 134-142): + +```python + if config.training.initial_model: + im = config.training.initial_model + if im.file: + params_dict["model_filename"] = im.file + elif im.wandb_alias: + params_dict["wandb_alias"] = im.wandb_alias + params_dict["wandb_project"] = im.wandb_project or ( + config.wandb.project if config.wandb else "deep_quoridor" + ) + elif im.run: + latest_yaml = Path(im.run) / "models" / "latest.yaml" + latest = parse_yaml_file_as(LatestModel, latest_yaml) + params_dict["model_filename"] = latest.filename +``` + +Note the `if/elif/elif` chain (the three sources are mutually exclusive per the Task 1 validator). + +- [ ] **Step 4: Run the test to confirm it passes** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/config_test.py::test_initial_model_run_resolves_to_latest_filename -v +``` + +Expected: PASS. + +Also re-run the full `config_test.py` to confirm no regressions: + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/config_test.py -v +``` + +Expected: all pass. + +- [ ] **Step 5: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/v2/common.py deep_quoridor/test/config_test.py +git commit -m "vibe: resolve initial_model.run via latest.yaml" +``` + +End with the Co-Authored-By trailer. + +--- + +## Task 5: Rename trainer helper bool params; wire from new config + +**Files:** +- Modify: `deep_quoridor/src/v2/trainer.py` (both helpers + the `train()` site that calls them) +- Modify: `deep_quoridor/test/test_trainer_helpers.py` + +- [ ] **Step 1: Update the 5 tests to use the new param names** + +In `deep_quoridor/test/test_trainer_helpers.py`, replace all five test functions. The truth table is unchanged; only the bool parameter names change (`offline_mode` → `selfplay_disabled` for `_should_skip_iteration`, and `offline_mode` → `omit_model_lag` for `_build_game_log`). The file's full contents should be: + +```python +from v2.trainer import _build_game_log, _should_skip_iteration +from v2.yaml_models import GameInfo + + +def test_should_skip_when_not_enough_moves(): + # Below batch_size: must skip regardless of mode. + assert _should_skip_iteration( + total_moves=10, batch_size=64, games_per_training_step=1.0, + training_steps=0, last_game=100, selfplay_disabled=False, + ) is True + assert _should_skip_iteration( + total_moves=10, batch_size=64, games_per_training_step=1.0, + training_steps=0, last_game=100, selfplay_disabled=True, + ) is True + + +def test_selfplay_on_honors_games_per_step_gate(): + # Enough moves, but games_per_training_step * (steps+1) > last_game: skip. + assert _should_skip_iteration( + total_moves=1000, batch_size=64, games_per_training_step=1.0, + training_steps=99, last_game=100, selfplay_disabled=False, + ) is False # 1.0 * 100 == last_game; not greater, so train. + assert _should_skip_iteration( + total_moves=1000, batch_size=64, games_per_training_step=1.0, + training_steps=100, last_game=100, selfplay_disabled=False, + ) is True # 1.0 * 101 > 100; throttle. + + +def test_selfplay_off_skips_games_per_step_gate(): + # Same parameters that would throttle now train. + assert _should_skip_iteration( + total_moves=1000, batch_size=64, games_per_training_step=1.0, + training_steps=100, last_game=100, selfplay_disabled=True, + ) is False + assert _should_skip_iteration( + total_moves=1000, batch_size=64, games_per_training_step=1.0, + training_steps=10_000, last_game=100, selfplay_disabled=True, + ) is False + + +def _gi(model_version: int, game_length: int) -> GameInfo: + return GameInfo(model_version=model_version, game_length=game_length, creator="test") + + +def test_build_game_log_includes_model_lag_by_default(): + log = _build_game_log( + game_info=_gi(model_version=5, game_length=42), + model_version=8, last_game=123, omit_model_lag=False, + ) + assert log == { + "game_length": 42, + "model_lag": 8 - 1 - 5, + "Game num": 123, + "Model version": 8, + } + + +def test_build_game_log_omits_model_lag_when_requested(): + log = _build_game_log( + game_info=_gi(model_version=5, game_length=42), + model_version=8, last_game=123, omit_model_lag=True, + ) + assert log == { + "game_length": 42, + "Game num": 123, + "Model version": 8, + } + assert "model_lag" not in log +``` + +- [ ] **Step 2: Run tests to confirm they fail** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_trainer_helpers.py -v +``` + +Expected: tests fail with `TypeError: _should_skip_iteration() got an unexpected keyword argument 'selfplay_disabled'` (same for `omit_model_lag`). + +- [ ] **Step 3: Rename the helper bool params** + +In `deep_quoridor/src/v2/trainer.py`, find the two helpers (added around lines 56-100 in the recent commits). Replace them with: + +```python +def _should_skip_iteration( + total_moves: int, + batch_size: int, + games_per_training_step: float, + training_steps: int, + last_game: int, + selfplay_disabled: bool, +) -> bool: + """Decide whether to skip this iteration of the trainer's main loop. + + Always skip when the buffer holds fewer moves than one batch. When self-play is + enabled, also skip when the trainer is ahead of self-play (the + `games_per_training_step` gate). When self-play is disabled the buffer is static + and there is no production cadence to wait on, so we train every iteration once + enough moves are available. + """ + if total_moves < batch_size: + return True + if selfplay_disabled: + return False + games_needed_to_train = games_per_training_step * (training_steps + 1) + return games_needed_to_train > last_game + + +def _build_game_log( + game_info, + model_version: int, + last_game: int, + omit_model_lag: bool, +) -> dict: + """Per-game wandb log payload emitted when a game is ingested from ready/. + + `model_lag` is meaningful only when games arrive from live self-play of the + current run, since it compares the trainer's current model version against the + version that *produced* the game. When games come from a preloaded buffer + (different lineage), the subtraction is nonsense, so the caller asks us to + omit the key. + """ + log = { + "game_length": game_info.game_length, + "Game num": last_game, + "Model version": model_version, + } + if not omit_model_lag: + log["model_lag"] = model_version - 1 - game_info.model_version + return log +``` + +- [ ] **Step 4: Wire `train()` to derive the new booleans from config** + +In `deep_quoridor/src/v2/trainer.py`'s `train()` function, find the line `offline_mode = config.training.source_run is not None` (added in commit `583b3d9`, around line 121 after the helpers shifted positions). Replace it with: + +```python + selfplay_disabled = not config.self_play.enabled + omit_model_lag = config.training.initial_replay_buffer is not None +``` + +Then find the `wandb_run.log(_build_game_log(game_info, model_version, last_game, offline_mode))` call (around line 205) and change it to: + +```python + wandb_run.log(_build_game_log(game_info, model_version, last_game, omit_model_lag)) +``` + +Then find the `_should_skip_iteration(...)` call (around line 215-222). The keyword arg `offline_mode=offline_mode` becomes `selfplay_disabled=selfplay_disabled`: + +```python + if _should_skip_iteration( + total_moves=total_moves, + batch_size=batch_size, + games_per_training_step=config.training.games_per_training_step, + training_steps=training_steps, + last_game=last_game, + selfplay_disabled=selfplay_disabled, + ): + time.sleep(1) + continue +``` + +- [ ] **Step 5: Run helper tests to confirm they pass** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_trainer_helpers.py -v +``` + +Expected: all 5 pass. + +- [ ] **Step 6: Sanity-check trainer.py still imports** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/python -c "from v2.trainer import train; print('ok')" +``` + +Expected: prints `ok`. + +- [ ] **Step 7: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/v2/trainer.py deep_quoridor/test/test_trainer_helpers.py +git commit -m "vibe: split trainer offline-mode into two booleans" +``` + +End with the Co-Authored-By trailer. + +--- + +## Task 6: `train_v2.py` refactor — remove `--source-run`, wire from new config + +**Files:** +- Modify: `deep_quoridor/src/train_v2.py` +- Delete: `deep_quoridor/test/test_train_v2_args.py` + +- [ ] **Step 1: Delete the obsolete test file** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git rm deep_quoridor/test/test_train_v2_args.py +``` + +That file only tested `source_run_overrides`, which is being removed. + +- [ ] **Step 2: Refactor `train_v2.py`** + +Replace the entire `deep_quoridor/src/train_v2.py` with the following. The diff from the current file: removes the `source_run_overrides` helper, removes the `--source-run` argparse flag, removes the `extra_overrides` merge block, replaces the `offline_mode = ...` derivation with two booleans, updates the preload call to use `config.training.initial_replay_buffer.run`, and wraps the self-play branch in `if config.self_play.enabled:` instead of `if not offline_mode:`. + +```python +import argparse +import multiprocessing as mp +import os +import subprocess +import time +from pathlib import Path + +from v2 import ( + benchmarks, + check_ai_available, + load_config_and_setup_run, + metrics_dir_for, + preload_symlinks, + run_ai_reporter, + run_selfplay_metrics, + self_play, + train, +) +from v2.common import ShutdownSignal + +# Prevents getting messages in the console every few lines telling you to install weave +os.environ["WANDB_DISABLE_WEAVE"] = "true" + + +def _selfplay_subprocess_env(): + """Environment for the Rust self-play subprocess. + + A selfplay binary built with the ``gpu`` feature loads ONNX Runtime + dynamically, so it needs ``ORT_DYLIB_PATH`` pointing at the onnxruntime-gpu + shared library and the CUDA/cuDNN wheel libs on ``LD_LIBRARY_PATH``. We + discover both from the installed packages so GPU self-play works without + manual shell setup. Returns ``None`` (inherit the current environment) when + onnxruntime isn't installed, in which case a CPU build runs unchanged. + """ + import importlib.util + + spec = importlib.util.find_spec("onnxruntime") + if spec is None or not spec.origin: + return None + pkg_dir = Path(spec.origin).parent + dylibs = sorted(pkg_dir.glob("capi/libonnxruntime.so*")) + if not dylibs: + return None + + site_packages = pkg_dir.parent + nvidia_libs = [str(p) for p in sorted((site_packages / "nvidia").glob("*/lib")) if p.is_dir()] + + env = dict(os.environ) + env["ORT_DYLIB_PATH"] = str(dylibs[-1]) + ld_parts = nvidia_libs + ([env["LD_LIBRARY_PATH"]] if env.get("LD_LIBRARY_PATH") else []) + if ld_parts: + env["LD_LIBRARY_PATH"] = ":".join(ld_parts) + return env + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Train Quoridor agent") + parser.add_argument("config_file", type=str, help="Path to YAML configuration file") + parser.add_argument("-r", "--runs-dir", type=str, default=None, help="Directory for runs") + # TODO: implement this + # parser.add_argument("-c", "--continue", dest="continue_run", action="store_true", help="Continue an existing run") + parser.add_argument( + "-o", + "--overrides", + nargs="*", + help="Configuration overrides (e.g., run_id=my_run self_play.program=rust)", + ) + + args = parser.parse_args() + + runs_dir = args.runs_dir if args.runs_dir is not None else str(Path(__file__).parent.parent) + + config = load_config_and_setup_run(args.config_file, runs_dir, overrides=args.overrides) + + # Validate AI report prerequisites before spawning anything, so a misconfigured + # run aborts early instead of failing silently inside a sibling process. + if config.ai_report is not None: + try: + check_ai_available(config.ai_report.ai) + except Exception as e: + print(f"ERROR: {e}") + exit(1) + + mp.set_start_method("spawn", force=True) + + # Make sure we don't have the shutdown signal from a previous run + ShutdownSignal.clear(config) + + if config.training.initial_replay_buffer is not None: + n_loaded = preload_symlinks( + source_run=Path(config.training.initial_replay_buffer.run), + dest_ready=config.paths.replay_buffers_ready, + buffer_size=config.training.replay_buffer_size, + ) + print(f"Preloaded {n_loaded} games from {config.training.initial_replay_buffer.run}") + + train_process = mp.Process(target=train, args=[config]) + train_process.start() + + benchmark_processes = benchmarks.create_benchmark_processes(config) + [p.start() for p in benchmark_processes] + + ai_report_process = None + if config.ai_report is not None: + ai_report_process = mp.Process(target=run_ai_reporter, args=[config]) + ai_report_process.start() + + self_play_processes = [] + rust_subprocesses = [] + + if config.self_play.enabled: + if config.self_play.program == "rust": + # Spawn Rust self-play processes in continuous mode + selfplay_env = _selfplay_subprocess_env() + if selfplay_env is not None: + print(f"Self-play GPU env: ORT_DYLIB_PATH={selfplay_env['ORT_DYLIB_PATH']}") + metrics_dir = metrics_dir_for(config) + os.makedirs(metrics_dir, exist_ok=True) + config_file_path = str(config.paths.config_file) + for i in range(config.self_play.num_processes): + cmd = [ + config.self_play.rust_selfplay_binary, + "--config", + config_file_path, + "--output-dir", + str(config.paths.replay_buffers_ready), + "--continuous", + "--latest-model-yaml", + str(config.paths.latest_model_yaml), + "--shutdown-file", + str(ShutdownSignal.file_path(config)), + "--metrics-dir", + metrics_dir, + ] + proc = subprocess.Popen(cmd, env=selfplay_env) + rust_subprocesses.append(proc) + print(f"Started Rust self-play process {proc.pid}") + selfplay_metrics_process = mp.Process(target=run_selfplay_metrics, args=[config]) + selfplay_metrics_process.start() + self_play_processes.append(selfplay_metrics_process) + else: + for i in range(config.self_play.num_processes): + p = mp.Process(target=self_play, args=[config]) + p.start() + self_play_processes.append(p) + + train_process.join() + ShutdownSignal.signal(config) + print("Shutting down!") + + b_count_prev, sf_count_prev, ai_count_prev = -1, -1, -1 + while True: + b_count = sum([p.is_alive() for p in benchmark_processes]) + sf_count = sum([p.is_alive() for p in self_play_processes]) + sf_count += sum([p.poll() is None for p in rust_subprocesses]) + ai_count = 1 if ai_report_process is not None and ai_report_process.is_alive() else 0 + if b_count_prev != b_count or sf_count_prev != sf_count or ai_count_prev != ai_count: + print( + f"Waiting for {b_count} benchmark processes, {sf_count} self_play processes" + f" and {ai_count} ai_report processes" + ) + b_count_prev, sf_count_prev, ai_count_prev = b_count, sf_count, ai_count + + if (b_count + sf_count + ai_count) == 0: + break + time.sleep(1) + + ShutdownSignal.clear(config) +``` + +- [ ] **Step 3: Sanity-check the script imports + `--help`** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/python -c "import train_v2; print('ok')" +``` + +Expected: prints `ok`. + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/python deep_quoridor/src/train_v2.py --help 2>&1 | head -20 +``` + +Expected: usage text with `config_file`, `-r/--runs-dir`, `-o/--overrides` — and **no** `--source-run` flag. + +- [ ] **Step 4: Re-run all affected tests** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/config_test.py deep_quoridor/test/test_offline_preload.py deep_quoridor/test/test_trainer_helpers.py deep_quoridor/test/test_selfplay_metrics.py -v +``` + +Expected: all pass. (`test_train_v2_args.py` has been deleted so it's not in the list.) + +- [ ] **Step 5: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/train_v2.py deep_quoridor/test/test_train_v2_args.py +git commit -m "vibe: drop --source-run; wire offline from config" +``` + +End with the Co-Authored-By trailer. + +--- + +## Task 7: End-to-end smoke verification + format pass + +This task verifies the wired-together behavior against a tiny source run. Two smoke configurations: pure offline (yesterday's behavior, reproduced via the new config form) and mixed mode (preload + self-play on). It also runs a formatting pass. + +**Files:** No code changes for the smoke tests. The formatting commit may touch the files modified in Tasks 1-6. + +- [ ] **Step 1: Prepare a fake "source" run dir** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +PYTHONPATH=deep_quoridor/src .venv/bin/python - <<'PY' +from pathlib import Path +import numpy as np +from pydantic_yaml import to_yaml_file +from v2.yaml_models import GameInfo + +source = Path("/tmp/smoke_source_run/replay_buffers") +source.mkdir(parents=True, exist_ok=True) + +for i in range(1, 6): + game_length = 8 + np.savez( + source / f"game_{i:07d}.npz", + input_arrays=np.zeros((game_length, 1), dtype=np.float32), + policies=np.zeros((game_length, 1), dtype=np.float32), + action_masks=np.zeros((game_length, 1), dtype=np.float32), + values=np.zeros(game_length, dtype=np.float32), + players=np.zeros(game_length, dtype=np.int32), + ) + to_yaml_file( + source / f"game_{i:07d}.yaml", + GameInfo(model_version=i, game_length=game_length, creator="smoke"), + ) +print("source_run prepared at /tmp/smoke_source_run") +PY +``` + +- [ ] **Step 2: Create the offline-mode config** + +Save this to `/tmp/smoke_offline.yaml`: + +```yaml +run_id: smoke-offline-$DATETIME +quoridor: + board_size: 5 + max_walls: 3 + max_steps: 50 +alphazero: + network: + type: mlp + mcts_n: 25 + mcts_c_puct: 1.2 +self_play: + enabled: false + num_processes: 1 + games_per_process: 4 + alphazero: + mcts_noise_epsilon: 0.25 +training: + games_per_training_step: 1.0 + learning_rate: 0.001 + batch_size: 32 + weight_decay: 0.0001 + replay_buffer_size: 3 + finish_after: "3 models" + initial_replay_buffer: + run: /tmp/smoke_source_run +``` + +- [ ] **Step 3: Run pure-offline smoke** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src timeout 30 .venv/bin/python deep_quoridor/src/train_v2.py /tmp/smoke_offline.yaml -r /tmp/smoke_runs 2>&1 | tee /tmp/smoke_offline.log +``` + +(Like yesterday: the trainer will eventually crash on a tensor-shape mismatch because the fake arrays don't match the real network's input shape. We're verifying boot + preload + self-play-skip, not training correctness.) + +- [ ] **Step 4: Verify offline-smoke output** + +```bash +grep -E "Preloaded .* games from|Started Rust self-play|self_play process" /tmp/smoke_offline.log +``` + +Expected: +- `Preloaded 3 games from /tmp/smoke_source_run` line present. +- NO `Started Rust self-play process` lines. +- NO `self_play process` spawning messages (from the python branch). + +```bash +ls -la /tmp/smoke_runs/runs/smoke-offline-*/replay_buffers/ 2>/dev/null | head -10 +``` + +Expected: at least 3 symlinks pointing into `/tmp/smoke_source_run/replay_buffers/`. + +- [ ] **Step 5: Create the mixed-mode config and run it** + +Save this to `/tmp/smoke_mixed.yaml`: + +```yaml +run_id: smoke-mixed-$DATETIME +quoridor: + board_size: 5 + max_walls: 3 + max_steps: 50 +alphazero: + network: + type: mlp + mcts_n: 25 + mcts_c_puct: 1.2 +self_play: + enabled: true + program: python + num_processes: 1 + games_per_process: 4 + alphazero: + mcts_noise_epsilon: 0.25 +training: + games_per_training_step: 1.0 + learning_rate: 0.001 + batch_size: 32 + weight_decay: 0.0001 + replay_buffer_size: 3 + finish_after: "3 models" + initial_replay_buffer: + run: /tmp/smoke_source_run +``` + +Run: + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src timeout 15 .venv/bin/python deep_quoridor/src/train_v2.py /tmp/smoke_mixed.yaml -r /tmp/smoke_runs 2>&1 | tee /tmp/smoke_mixed.log || true +``` + +(`|| true` so a non-zero exit from `timeout` doesn't fail the script — we expect the trainer to crash on the same shape mismatch and possibly kill the run before clean shutdown.) + +- [ ] **Step 6: Verify mixed-mode output** + +```bash +grep -E "Preloaded .* games from" /tmp/smoke_mixed.log +``` + +Expected: `Preloaded 3 games from /tmp/smoke_source_run` is present. + +```bash +ps -eo pid,cmd 2>/dev/null | grep -i "self_play\|train_v2" | grep -v grep | head +``` + +May show python self-play processes if they're still alive. Either way, the key signal is in the log — verify that self-play python processes are mentioned by checking the trainer's behavior. (Lower-confidence signal in mixed mode because the trainer crashes early; the `Preloaded` line is the primary check that `initial_replay_buffer` is honored independently of self-play.) + +- [ ] **Step 7: Cleanup** + +```bash +rm -rf /tmp/smoke_source_run /tmp/smoke_runs /tmp/smoke_offline.yaml /tmp/smoke_mixed.yaml /tmp/smoke_offline.log /tmp/smoke_mixed.log +``` + +- [ ] **Step 8: Format pass** + +Run ruff format on the touched files: + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && .venv/bin/ruff format deep_quoridor/src/v2/config.py deep_quoridor/src/v2/common.py deep_quoridor/src/v2/trainer.py deep_quoridor/src/train_v2.py deep_quoridor/test/config_test.py deep_quoridor/test/test_trainer_helpers.py +``` + +If anything was reformatted, commit it separately per AGENTS.md: + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && git status --short +cd /home/jbinney/ws/deep_rabbit_hole && git add -u && git commit -m "vibe: ruff format" +``` + +(End with the Co-Authored-By trailer.) If nothing changed, skip this commit. + +--- + +## Done criteria + +- All tests in `config_test.py`, `test_offline_preload.py`, `test_trainer_helpers.py`, `test_selfplay_metrics.py` pass. +- `test_train_v2_args.py` is gone. +- `train_v2.py --help` does not show `--source-run`. +- Offline smoke run prints `Preloaded N games from ` and spawns NO self-play processes. +- Mixed-mode smoke run prints `Preloaded N games from ` and DOES proceed to spawn self-play. +- Touched files clean under `ruff format`. diff --git a/docs/superpowers/plans/2026-06-13-run-benchmarks-v2.md b/docs/superpowers/plans/2026-06-13-run-benchmarks-v2.md new file mode 100644 index 00000000..95e96896 --- /dev/null +++ b/docs/superpowers/plans/2026-06-13-run-benchmarks-v2.md @@ -0,0 +1,602 @@ +# `run_benchmarks_v2.py` Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +I'm using AGENTS.md + +**Goal:** A new script `deep_quoridor/src/run_benchmarks_v2.py` that takes an existing run directory and spawns just the benchmark processes described in that run's saved `config.yaml`, looping like `train_v2.py` until Ctrl-C. + +**Architecture:** Reuse `benchmarks.create_benchmark_processes(config)` unchanged. The script loads the saved `config.yaml` via the existing `load_user_config(...)`, builds a `Config` with `Config.from_user(..., create_dirs=False)` so existing dirs are untouched, runs three startup checks (run_dir, config.yaml, models/latest.yaml all exist), then spawns the benchmark processes and waits with the same shutdown-loop pattern train_v2 uses. Ctrl-C signals `ShutdownSignal` and joins. + +**Tech Stack:** Python 3.12, pytest, pydantic, multiprocessing. + +**Spec:** `docs/superpowers/specs/2026-06-11-run-benchmarks-v2-design.md` + +--- + +## File structure + +- `deep_quoridor/src/run_benchmarks_v2.py` — new script. (create) +- `deep_quoridor/test/test_run_benchmarks_v2.py` — unit tests for path-derivation, config-loading, and startup checks. (create) + +**Run Python tests with:** +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_run_benchmarks_v2.py -v +``` + +**Commit style (AGENTS.md):** `vibe: ` imperative subject ≤ 50 chars. End commit messages with: +``` +Co-Authored-By: Claude Opus 4.7 (1M context) +``` + +**Branch state:** This continues on `jdb/train-on-existing-selfplay-games`. The spec is committed at `557f22e`. + +--- + +## Task 1: Path-derivation + config-load helpers + +**Files:** +- Create: `deep_quoridor/src/run_benchmarks_v2.py` +- Create: `deep_quoridor/test/test_run_benchmarks_v2.py` + +- [ ] **Step 1: Write the failing tests** + +Create `deep_quoridor/test/test_run_benchmarks_v2.py` with these tests: + +```python +from pathlib import Path + +import pytest +import yaml + +from run_benchmarks_v2 import _derive_base_dir, _load_config + + +EXAMPLE_CONFIG = { + "run_id": "test-run", + "quoridor": {"board_size": 5, "max_walls": 3, "max_steps": 50}, + "alphazero": {"network": {"type": "mlp"}, "mcts_n": 300, "mcts_c_puct": 1.2}, + "self_play": {"num_processes": 2, "games_per_process": 16, "alphazero": {"mcts_noise_epsilon": 0.25}}, + "training": { + "games_per_training_step": 25.0, + "learning_rate": 0.001, + "batch_size": 256, + "weight_decay": 0.0001, + "replay_buffer_size": 1000000, + }, + "benchmarks": [ + { + "every": "10 models", + "jobs": [ + {"type": "tournament", "prefix": "raw", "times": 10, "opponents": ["random", "greedy"]}, + ], + }, + ], +} + + +def _make_run_dir(tmp_path: Path, run_id: str = "test-run") -> Path: + """Create a runs// structure with a valid config.yaml inside.""" + run_dir = tmp_path / "runs" / run_id + run_dir.mkdir(parents=True) + cfg = dict(EXAMPLE_CONFIG) + cfg["run_id"] = run_id + (run_dir / "config.yaml").write_text(yaml.safe_dump(cfg, sort_keys=False)) + return run_dir + + +def test_derive_base_dir_uses_grandparent(tmp_path): + run_dir = tmp_path / "runs" / "my-run" + assert _derive_base_dir(run_dir) == str(tmp_path) + + +def test_load_config_returns_full_config(tmp_path): + run_dir = _make_run_dir(tmp_path, run_id="my-run") + config = _load_config(run_dir, overrides=None) + assert config.run_id == "my-run" + assert config.training.learning_rate == 0.001 + assert len(config.benchmarks) == 1 + # paths derived from the run dir + assert config.paths.run_dir == run_dir + + +def test_load_config_applies_overrides(tmp_path): + run_dir = _make_run_dir(tmp_path) + config = _load_config(run_dir, overrides=["training.learning_rate=0.05"]) + assert config.training.learning_rate == 0.05 + + +def test_load_config_does_not_create_dirs(tmp_path): + """Config.from_user(..., create_dirs=False) — no replay_buffers/, etc. spawned.""" + run_dir = _make_run_dir(tmp_path) + _load_config(run_dir, overrides=None) + assert not (run_dir / "replay_buffers").exists() + assert not (run_dir / "models").exists() + + +def test_load_config_raises_when_config_yaml_missing(tmp_path): + run_dir = tmp_path / "runs" / "my-run" + run_dir.mkdir(parents=True) + with pytest.raises(FileNotFoundError, match="config.yaml"): + _load_config(run_dir, overrides=None) +``` + +- [ ] **Step 2: Run tests to confirm they fail** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_run_benchmarks_v2.py -v +``` + +Expected: `ModuleNotFoundError: No module named 'run_benchmarks_v2'`. + +- [ ] **Step 3: Create the script with the two helpers** + +Create `deep_quoridor/src/run_benchmarks_v2.py`: + +```python +"""Run just the benchmark schedules from an existing run's config.yaml. + +Usage: + python deep_quoridor/src/run_benchmarks_v2.py [-o key=val ...] + +Spawns one process per `config.benchmarks` schedule and waits until Ctrl-C. +Reuses `benchmarks.create_benchmark_processes` from the v2 package; does not +train, run self-play, or generate AI reports. +""" + +from pathlib import Path + +from v2.config import Config, load_user_config + + +def _derive_base_dir(run_dir: Path) -> str: + """Given a run dir laid out as `base_dir/runs//`, return `base_dir`. + + The run-dir convention used by `train_v2.py`'s `load_config_and_setup_run` + places each run under `/runs//`, so the parent of `runs/` + is the base_dir the rest of the v2 machinery expects. + """ + return str(run_dir.parent.parent) + + +def _load_config(run_dir: Path, overrides: list[str] | None) -> Config: + """Load `/config.yaml` and build a Config without touching disk. + + Uses `Config.from_user(..., create_dirs=False)` so the existing run directory + isn't disturbed and no `config.yaml` snapshot is rewritten. Raises + `FileNotFoundError` if the config file is missing. + """ + config_yaml = run_dir / "config.yaml" + if not config_yaml.is_file(): + raise FileNotFoundError(f"No config.yaml in {run_dir}") + user_config = load_user_config(str(config_yaml), overrides=overrides) + return Config.from_user(user_config, _derive_base_dir(run_dir), create_dirs=False) +``` + +- [ ] **Step 4: Run tests to confirm they pass** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_run_benchmarks_v2.py -v +``` + +Expected: all 5 tests pass. + +- [ ] **Step 5: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/run_benchmarks_v2.py deep_quoridor/test/test_run_benchmarks_v2.py +git commit -m "vibe: add run_benchmarks_v2 config-load helpers" +``` + +End with the Co-Authored-By trailer. + +--- + +## Task 2: Startup checks + `main()` with empty-benchmarks early exit + +**Files:** +- Modify: `deep_quoridor/src/run_benchmarks_v2.py` +- Modify: `deep_quoridor/test/test_run_benchmarks_v2.py` + +- [ ] **Step 1: Write the failing tests** + +First, update the imports at the top of `deep_quoridor/test/test_run_benchmarks_v2.py`: + +- Add `from argparse import Namespace` near the other top-level imports. +- Extend the existing `from run_benchmarks_v2 import _derive_base_dir, _load_config` line to: + ```python + from run_benchmarks_v2 import _check_run_dir, _derive_base_dir, _load_config, main + ``` + +Then append these test functions to the end of the file: + +```python +def test_check_run_dir_raises_when_run_dir_missing(tmp_path): + with pytest.raises(FileNotFoundError, match="Run directory not found"): + _check_run_dir(tmp_path / "does-not-exist") + + +def test_check_run_dir_raises_when_config_yaml_missing(tmp_path): + run_dir = tmp_path / "runs" / "my-run" + run_dir.mkdir(parents=True) + with pytest.raises(FileNotFoundError, match="config.yaml"): + _check_run_dir(run_dir) + + +def test_check_run_dir_raises_when_latest_yaml_missing(tmp_path): + run_dir = _make_run_dir(tmp_path) + with pytest.raises(FileNotFoundError, match="models/latest.yaml"): + _check_run_dir(run_dir) + + +def test_check_run_dir_passes_when_all_present(tmp_path): + run_dir = _make_run_dir(tmp_path) + (run_dir / "models").mkdir() + (run_dir / "models" / "latest.yaml").write_text("filename: /tmp/m.pt\nversion: 0\n") + _check_run_dir(run_dir) # no exception + + +def test_main_exits_zero_when_no_benchmarks(tmp_path, capsys): + run_dir = _make_run_dir(tmp_path) + (run_dir / "models").mkdir() + (run_dir / "models" / "latest.yaml").write_text("filename: /tmp/m.pt\nversion: 0\n") + + # Strip the benchmarks section from the config.yaml. + cfg = yaml.safe_load((run_dir / "config.yaml").read_text()) + cfg["benchmarks"] = [] + (run_dir / "config.yaml").write_text(yaml.safe_dump(cfg, sort_keys=False)) + + args = Namespace(run_dir=str(run_dir), overrides=None) + exit_code = main(args) + assert exit_code == 0 + captured = capsys.readouterr() + assert "No benchmarks configured" in captured.out +``` + +- [ ] **Step 2: Run tests to confirm they fail** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_run_benchmarks_v2.py -v +``` + +Expected: 5 new tests fail with `ImportError` on `_check_run_dir` and `main`. + +- [ ] **Step 3: Add the startup-check helper and a minimal `main()`** + +Append to `deep_quoridor/src/run_benchmarks_v2.py`: + +```python +def _check_run_dir(run_dir: Path) -> None: + """Verify the run directory has the layout we need before spawning processes. + + Aborts early on a missing `latest.yaml` so the benchmark processes don't enter + `LatestModel.wait_for_creation`'s blocking wait (no training is producing + models in this script). + """ + if not run_dir.is_dir(): + raise FileNotFoundError(f"Run directory not found: {run_dir}") + if not (run_dir / "config.yaml").is_file(): + raise FileNotFoundError(f"No config.yaml in {run_dir}") + latest_yaml = run_dir / "models" / "latest.yaml" + if not latest_yaml.is_file(): + raise FileNotFoundError( + f"No models/latest.yaml in {run_dir}; the run has no trained model to benchmark." + ) + + +def main(args) -> int: + """Entry point. Returns the exit code.""" + run_dir = Path(args.run_dir).resolve() + _check_run_dir(run_dir) + config = _load_config(run_dir, args.overrides) + + if not config.benchmarks: + print(f"No benchmarks configured in {run_dir}/config.yaml; nothing to run.") + return 0 + + # Spawning is added in Task 3. + return 0 +``` + +- [ ] **Step 4: Run tests to confirm they pass** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_run_benchmarks_v2.py -v +``` + +Expected: all 10 tests pass. + +- [ ] **Step 5: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/run_benchmarks_v2.py deep_quoridor/test/test_run_benchmarks_v2.py +git commit -m "vibe: add run_benchmarks_v2 startup checks + main" +``` + +End with the Co-Authored-By trailer. + +--- + +## Task 3: Spawn benchmark processes + Ctrl-C shutdown + smoke test + +**Files:** +- Modify: `deep_quoridor/src/run_benchmarks_v2.py` + +No unit tests for this task — `mp.Process` spawning is covered end-to-end by the smoke test in Step 5. + +- [ ] **Step 1: Add the spawn-and-wait body to `main()` and the `__main__` block** + +Replace the entire contents of `deep_quoridor/src/run_benchmarks_v2.py` with: + +```python +"""Run just the benchmark schedules from an existing run's config.yaml. + +Usage: + python deep_quoridor/src/run_benchmarks_v2.py [-o key=val ...] + +Spawns one process per `config.benchmarks` schedule and waits until Ctrl-C. +Reuses `benchmarks.create_benchmark_processes` from the v2 package; does not +train, run self-play, or generate AI reports. +""" + +import argparse +import multiprocessing as mp +import os +import time +from pathlib import Path + +from v2 import benchmarks +from v2.common import ShutdownSignal +from v2.config import Config, load_user_config + +# Match train_v2.py: suppress wandb's "install weave" log spam. +os.environ["WANDB_DISABLE_WEAVE"] = "true" + + +def _derive_base_dir(run_dir: Path) -> str: + """Given a run dir laid out as `base_dir/runs//`, return `base_dir`. + + The run-dir convention used by `train_v2.py`'s `load_config_and_setup_run` + places each run under `/runs//`, so the parent of `runs/` + is the base_dir the rest of the v2 machinery expects. + """ + return str(run_dir.parent.parent) + + +def _load_config(run_dir: Path, overrides: list[str] | None) -> Config: + """Load `/config.yaml` and build a Config without touching disk. + + Uses `Config.from_user(..., create_dirs=False)` so the existing run directory + isn't disturbed and no `config.yaml` snapshot is rewritten. Raises + `FileNotFoundError` if the config file is missing. + """ + config_yaml = run_dir / "config.yaml" + if not config_yaml.is_file(): + raise FileNotFoundError(f"No config.yaml in {run_dir}") + user_config = load_user_config(str(config_yaml), overrides=overrides) + return Config.from_user(user_config, _derive_base_dir(run_dir), create_dirs=False) + + +def _check_run_dir(run_dir: Path) -> None: + """Verify the run directory has the layout we need before spawning processes. + + Aborts early on a missing `latest.yaml` so the benchmark processes don't enter + `LatestModel.wait_for_creation`'s blocking wait (no training is producing + models in this script). + """ + if not run_dir.is_dir(): + raise FileNotFoundError(f"Run directory not found: {run_dir}") + if not (run_dir / "config.yaml").is_file(): + raise FileNotFoundError(f"No config.yaml in {run_dir}") + latest_yaml = run_dir / "models" / "latest.yaml" + if not latest_yaml.is_file(): + raise FileNotFoundError( + f"No models/latest.yaml in {run_dir}; the run has no trained model to benchmark." + ) + + +def main(args) -> int: + """Entry point. Returns the exit code.""" + run_dir = Path(args.run_dir).resolve() + _check_run_dir(run_dir) + config = _load_config(run_dir, args.overrides) + + if not config.benchmarks: + print(f"No benchmarks configured in {run_dir}/config.yaml; nothing to run.") + return 0 + + mp.set_start_method("spawn", force=True) + ShutdownSignal.clear(config) + + benchmark_processes = benchmarks.create_benchmark_processes(config) + for p in benchmark_processes: + p.start() + print(f"Started {len(benchmark_processes)} benchmark processes") + + try: + b_count_prev = -1 + while True: + b_count = sum(p.is_alive() for p in benchmark_processes) + if b_count != b_count_prev: + print(f"Waiting for {b_count} benchmark processes") + b_count_prev = b_count + if b_count == 0: + break + time.sleep(1) + except KeyboardInterrupt: + print("\nCaught Ctrl-C; signaling shutdown...") + ShutdownSignal.signal(config) + for p in benchmark_processes: + p.join() + + ShutdownSignal.clear(config) + return 0 + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Run just the benchmark schedules from an existing run's config.yaml.", + ) + parser.add_argument( + "run_dir", + type=str, + help="Path to an existing run directory (e.g. /path/to/runs//).", + ) + parser.add_argument( + "-o", + "--overrides", + nargs="*", + help="Configuration overrides (e.g., benchmarks.0.every=2 minutes).", + ) + args = parser.parse_args() + raise SystemExit(main(args)) +``` + +- [ ] **Step 2: Sanity-check the script imports + `--help`** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/python -c "import run_benchmarks_v2; print('ok')" +``` + +Expected: prints `ok`. + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/python deep_quoridor/src/run_benchmarks_v2.py --help 2>&1 | head -20 +``` + +Expected: usage text showing `run_dir` positional and `-o/--overrides` option. + +- [ ] **Step 3: Re-run the unit tests to confirm they still pass** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/pytest deep_quoridor/test/test_run_benchmarks_v2.py -v +``` + +Expected: all 10 tests pass. + +- [ ] **Step 4: Commit** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +git add deep_quoridor/src/run_benchmarks_v2.py +git commit -m "vibe: spawn benchmark processes in run_benchmarks_v2" +``` + +End with the Co-Authored-By trailer. + +- [ ] **Step 5: Smoke test against a fake run directory** + +This step builds a tiny run dir with a valid `config.yaml` and `models/latest.yaml`, points the script at it, and verifies the script gets through preflight + spawn before being interrupted. The benchmark processes will themselves fail quickly (no real model file to load), which is fine for this smoke check — we're verifying entrypoint behavior, not benchmark execution. + +The script writes to `/tmp/smoke_*` which is outside the sandbox; use `dangerouslyDisableSandbox: true` for these commands. + +Prepare: + +```bash +cd /home/jbinney/ws/deep_rabbit_hole +PYTHONPATH=deep_quoridor/src .venv/bin/python - <<'PY' +from pathlib import Path +import yaml +from pydantic_yaml import to_yaml_file +from v2.yaml_models import LatestModel + +base = Path("/tmp/smoke_bench") +run_dir = base / "runs" / "smoke-bench-test" +(run_dir / "models").mkdir(parents=True, exist_ok=True) + +cfg = { + "run_id": "smoke-bench-test", + "quoridor": {"board_size": 5, "max_walls": 3, "max_steps": 50}, + "alphazero": {"network": {"type": "mlp"}, "mcts_n": 25, "mcts_c_puct": 1.2}, + "self_play": {"num_processes": 1, "games_per_process": 4, "alphazero": {"mcts_noise_epsilon": 0.25}}, + "training": { + "games_per_training_step": 1.0, + "learning_rate": 0.001, + "batch_size": 32, + "weight_decay": 0.0001, + "replay_buffer_size": 30, + }, + "benchmarks": [ + { + "every": "10 models", + "jobs": [ + {"type": "dumb_score", "prefix": "raw"}, + ], + }, + ], +} +(run_dir / "config.yaml").write_text(yaml.safe_dump(cfg, sort_keys=False)) +to_yaml_file( + run_dir / "models" / "latest.yaml", + LatestModel(filename=str(run_dir / "models" / "model_0.pt"), version=0), +) +print(f"prepared {run_dir}") +PY +``` + +Run for ~5 seconds then Ctrl-C (use `timeout 5 ... || true` so the non-zero exit from the timeout doesn't fail the shell): + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src timeout 5 .venv/bin/python deep_quoridor/src/run_benchmarks_v2.py /tmp/smoke_bench/runs/smoke-bench-test 2>&1 | tee /tmp/smoke_bench.log || true +``` + +Verify expected output in the log: + +```bash +grep -E "Started [0-9]+ benchmark processes|Waiting for|No benchmarks" /tmp/smoke_bench.log +``` + +Expected: +- `Started 1 benchmark processes` line present. +- At least one `Waiting for N benchmark processes` transition (the benchmark child process may have already crashed because `model_0.pt` doesn't exist on disk — that's expected; what matters is the script reached the spawn-and-wait stage). + +Try the empty-benchmarks early-exit path against the same run: + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/python deep_quoridor/src/run_benchmarks_v2.py /tmp/smoke_bench/runs/smoke-bench-test -o benchmarks=[] 2>&1 +``` + +Expected: prints `No benchmarks configured in /tmp/smoke_bench/runs/smoke-bench-test/config.yaml; nothing to run.` and exits 0. + +Try the missing-latest.yaml abort path: + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && rm /tmp/smoke_bench/runs/smoke-bench-test/models/latest.yaml +cd /home/jbinney/ws/deep_rabbit_hole && PYTHONPATH=deep_quoridor/src .venv/bin/python deep_quoridor/src/run_benchmarks_v2.py /tmp/smoke_bench/runs/smoke-bench-test 2>&1; echo "exit=$?" +``` + +Expected: non-zero exit with `FileNotFoundError: No models/latest.yaml in ...`. + +- [ ] **Step 6: Cleanup** + +```bash +rm -rf /tmp/smoke_bench /tmp/smoke_bench.log +``` + +- [ ] **Step 7: Format pass** + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && .venv/bin/ruff format deep_quoridor/src/run_benchmarks_v2.py deep_quoridor/test/test_run_benchmarks_v2.py +cd /home/jbinney/ws/deep_rabbit_hole && git status --short +``` + +If anything changed, commit separately per AGENTS.md: + +```bash +cd /home/jbinney/ws/deep_rabbit_hole && git add -u && git commit -m "vibe: ruff format" +``` + +End with the Co-Authored-By trailer. If nothing changed, skip. + +--- + +## Done criteria + +- All 10 unit tests in `test_run_benchmarks_v2.py` pass. +- `python run_benchmarks_v2.py --help` shows `run_dir` positional and `-o/--overrides` option. +- Smoke run prints `Started N benchmark processes` against a fake run dir; Ctrl-C / timeout doesn't crash the script before the spawn-and-wait loop is reached. +- Missing-latest.yaml path aborts with a clear error. +- Empty-benchmarks path prints the expected line and exits 0. +- Touched files clean under `ruff format`. diff --git a/docs/superpowers/specs/2026-06-04-train-on-existing-selfplay-design.md b/docs/superpowers/specs/2026-06-04-train-on-existing-selfplay-design.md new file mode 100644 index 00000000..8f2770a7 --- /dev/null +++ b/docs/superpowers/specs/2026-06-04-train-on-existing-selfplay-design.md @@ -0,0 +1,235 @@ +# Train on existing self-play games — design + +**Date:** 2026-06-04 +**Branch:** `jdb/train-on-existing-selfplay-games` +**Status:** approved (design), pending implementation plan + +## Problem + +When the AlphaZero neural network architecture changes (different number of channels, depth, +or even network type), the existing model checkpoints can't be reused — but the self-play +games generated by the previous model are still valuable training data. Today the only way +to start a run with a new architecture is to throw away that history and regenerate games +from scratch, which is expensive. + +We want a mode where `train_v2.py` consumes a previous run's stored games as a fixed replay +buffer, trains a new (fresh-weights) model on it indefinitely, and runs benchmarks normally — +no self-play subprocess, no new games arriving. + +## Approach + +Activated by a new CLI flag: `python train_v2.py CONFIG --source-run `. + +When set, `train_v2.py`: +1. Skips spawning all self-play workers and the `selfplay_metrics_process`. +2. Symlinks the newest games from `/replay_buffers/` into the new run's + `replay_buffers/ready/`. Selects the newest `config.training.replay_buffer_size` + games by filename (the trainer's existing trim limit is `len(moves_per_game) > + replay_buffer_size` — a count of games, not moves). +3. Starts `train`, benchmark processes, and `ai_report` exactly as today. + +The trainer's existing ready/-pickup loop ingests the symlinks indistinguishably from fresh +self-play games — `Sampler` resolves symlinks transparently via `np.load`. Two small trainer +diffs handle the differences a static buffer requires (see below). + +Termination: whatever `config.training.finish_after` is — same as a normal run. If unset, the +run continues until shutdown signal (Ctrl-C / shutdown file), identical to today's behavior. + +## Architecture & data flow + +``` +/ / +└── replay_buffers/ └── replay_buffers/ + ├── game_0000001.{npz,yaml} ├── ready/ + ├── ... │ ├── game_0123450.{npz,yaml} (symlinks) + ├── game_0123450.{npz,yaml} ◄─── newest │ ├── game_0123451.{npz,yaml} (symlinks) + ├── game_0123451.{npz,yaml} games │ └── ... ← train_v2 creates these + └── ... selected │ before spawning train() + │ + ▼ + trainer's ready/-pickup loop: + sort ascending → move → rename game_NNNNNNN.npz + → end up in /replay_buffers/ as symlinks + +/models/ +├── latest.yaml +└── checkpoints/ + ├── model_0.pt ← fresh weights, new architecture + ├── model_1.pt + └── ... + +/config.yaml ← records training.source_run for reproducibility +``` + +The trainer's main loop is unchanged in shape: + +``` +loop: + ingest *.npz from ready/ → replay_buffers/ (one-shot, at startup, in offline mode) + trim moves_per_game to <= replay_buffer_size + if total_moves < batch_size: sleep + else: sample batch, train one step, save model_{N}.pt, log to wandb +``` + +The only loop change in offline mode is dropping the `games_per_training_step` gate so +training continues against the static buffer. + +## Config + +Add one field to `TrainingConfig`: + +```python +source_run: Optional[str] = None +``` + +"Offline mode" is defined as `config.training.source_run is not None`. Single source of truth +— every branch in `train_v2.py` and `trainer.py` checks this field. + +`--source-run ` is sugar that, in `train_v2.py`, appends two overrides before calling +`load_config_and_setup_run`: + +- `training.source_run=` +- `self_play.program=python` + +The second override prevents `load_config_and_setup_run` from rejecting the run when the +source's old config has `self_play.program=rust` but no rust binary exists locally. The +self-play config fields are unused in offline mode either way; train_v2 logs the override on +startup so the behavior is visible. + +## `train_v2.py` orchestration + +- Add `--source-run` to argparse. +- If set, append the two overrides described above. +- After `load_config_and_setup_run`, if `config.training.source_run is not None`: + - Run the preload procedure (next section). Abort with a clear error on any failure. + - **Skip** the entire `if config.self_play.program == "rust": ... else: ...` block. Don't + spawn the `selfplay_metrics_process`. +- Spawn `train_process`, benchmark processes, and (optionally) `ai_report_process` exactly as + today. +- The shutdown wait loop works unchanged with empty `self_play_processes` / + `rust_subprocesses` lists. + +## Preload procedure + +Lives in a new module `v2/offline_preload.py`, exposing two functions: + +```python +def select_games(filenames: list[str], buffer_size: int) -> list[str]: + """Return the newest `buffer_size` filenames sorted ascending (chronological). + If fewer are available, return them all.""" + +def preload_symlinks(source_run: Path, dest_ready: Path, buffer_size: int) -> int: + """List /replay_buffers/*.npz, pick the newest `buffer_size` games by + filename, symlink both `.npz` and `.yaml` of each chosen game into `dest_ready`. + Return the count of games linked.""" +``` + +`buffer_size` is in **games** — matches the trainer's existing trim semantics +(`len(moves_per_game) > config.training.replay_buffer_size` at `trainer.py:208`). + +Procedure inside `preload_symlinks`: + +1. Validate `/replay_buffers/` exists and contains at least one `.npz`. +2. List `*.npz` filenames sorted ascending (source numbers monotonically, so this is + chronological). +3. Take the last `buffer_size` filenames (or all of them, if fewer exist). +4. For each selected game, verify its `.yaml` sidecar exists (abort if not), then create + symlinks for both `.npz` and `.yaml` in `dest_ready`, keeping the source basenames. + +The trainer's existing ready/-pickup loop sorts ascending, renames sequentially to +`game_NNNNNNN.npz`, moves into `replay_buffers/`, parses yaml, updates internal lists. Order +is preserved; the Sampler resolves symlinks on `np.load`. + +## Trainer changes (`trainer.py`) + +Two small diffs inside `train()`, both gated on `config.training.source_run is not None`: + +1. **Skip the games_per_training_step gate.** Currently: + + ```python + games_needed_to_train = config.training.games_per_training_step * (training_steps + 1) + if total_moves < batch_size or games_needed_to_train > last_game: + time.sleep(1) + continue + ``` + + In offline mode, only `total_moves < batch_size` applies. The first iteration still waits + for ingestion to finish; after that, every iteration trains. + +2. **Suppress `model_lag`.** The per-game wandb log currently writes + `model_lag = model_version - 1 - game_info.model_version`. In offline mode + `game_info.model_version` came from a different training run; the subtraction is + meaningless. Omit just that key. Everything else (`game_length`, `Game num`, + `Model version`, and the per-step block below) is unchanged. + +## Initial model + +The whole point of this mode is to change network architecture, so the source's `.pt`/`.onnx` +checkpoints are not loaded. The existing `config.training.initial_model` field still works +for callers that want to bootstrap from a matching-architecture checkpoint, but it is the +caller's responsibility to ensure the architecture matches. Pointing `initial_model` at the +source run's last checkpoint will fail at load time if the network shape differs — that's +acceptable; we don't add extra validation. + +## Edge cases & failure modes + +- Source dir missing, has no `replay_buffers/`, or contains zero `.npz` files → abort with + the source path in the error. +- Any selected `.npz` lacks its sibling `.yaml` → abort, listing the offending file. (Only + selected games' yamls are checked; non-selected games' missing yamls are ignored.) +- Symlink creation fails (permissions, filesystem) → abort, propagating the OS error with the + path. +- Source has fewer than `replay_buffer_size` games → ingest everything; log how many games + were loaded vs requested. +- User sets `training.source_run` in yaml directly (no CLI flag) → works identically; the CLI + flag is just sugar. The rust-binary-validation skip *only* fires via the CLI flag, so a + yaml-only user must also set `self_play.program=python` themselves. Acceptable — the CLI is + the documented entry point. +- Source config had different `board_size` / `max_walls` than new config → the `.npz` arrays + will mismatch the new network's input shape and training will fail at the first batch. + **Out of scope for v1** — document the requirement that quoridor params must match. A + startup check is a reasonable future addition. +- Source run still being written to by another process → out of scope; doc says "point at a + dormant source run." +- `--source-run` plus a yaml `initial_model` from the same source → loads only if the + architecture matches; otherwise fails at model-load time. No special handling. + +## Testing + +**Unit tests:** + +- `select_games(filenames, buffer_size)`: + - source > buffer: returns the newest `buffer_size` filenames in ascending (chronological) + order. + - source < buffer: returns all filenames. + - empty source: returns empty list. + - input-order independence: result is the same regardless of input list order. +- `preload_symlinks(source, dest_ready, buffer_size)`: + - creates both `.npz` and `.yaml` symlinks per game. + - returns the count of games linked. + - symlinks resolve via `np.load`. +- `train_v2.py` argparse: `--source-run X` injects `training.source_run=X` and + `self_play.program=python` into the overrides list before + `load_config_and_setup_run`. + +**Integration test:** + +- Fixture: a small "source" run dir with ~5 synthetic `.npz`+`.yaml` pairs (real numpy arrays + small enough to be cheap, real `GameInfo` yamls). +- Run `train_v2.py --source-run ` with a config setting `finish_after: "2 models"` + and `replay_buffer_size` smaller than the fixture's game count. +- Assert: at least `model_1.pt` and `model_2.pt` are written to the new run's + `checkpoints/`; no rust subprocess is invoked; symlinks exist in the new run's + `replay_buffers/` after ingestion; the per-game ingestion wandb log call for a + symlinked source game contains `game_length` but not `model_lag`. + +## Out of scope (YAGNI) + +- Cross-architecture checkpoint reuse (e.g., partial weight transfer). +- Mixing games from multiple source runs. +- Startup validation that `board_size` / `max_walls` match between source and new configs + (deferred — failure mode is loud at first batch). +- Continuing/resuming an offline run from a partial state. A re-launch with `--source-run` + starts fresh. +- Sampling games other than "newest first" (e.g., uniform across all source games, + model-version-stratified). Future option if needed. diff --git a/docs/superpowers/specs/2026-06-05-train-from-previous-run-design.md b/docs/superpowers/specs/2026-06-05-train-from-previous-run-design.md new file mode 100644 index 00000000..4ad08a2c --- /dev/null +++ b/docs/superpowers/specs/2026-06-05-train-from-previous-run-design.md @@ -0,0 +1,254 @@ +# Train from a previous run — design + +**Date:** 2026-06-05 +**Branch:** `jdb/train-on-existing-selfplay-games` +**Status:** approved (design), pending implementation plan + +**Supersedes:** `docs/superpowers/specs/2026-06-04-train-on-existing-selfplay-design.md` +(the offline-mode feature shipped yesterday; this refactor generalizes it). + +## Problem + +Yesterday we shipped `--source-run`, a single CLI flag that did three things at once: pull +the previous run's replay buffer, infer "skip self-play", and force `self_play.program=python` +to dodge the rust-binary check. The three concerns are actually independent: + +1. **Inherit weights** from a previous run's latest checkpoint (continue training, possibly + with the same architecture). +2. **Inherit replay buffer** from a previous run's stored games (warm-start the buffer). +3. **Disable self-play** entirely (consume a static buffer; the "old offline mode"). + +Wanting any single one without the others is a real use case. Wanting all three is the +yesterday-shipped offline mode. The current schema can't express the partial cases cleanly, +and the CLI flag bypasses config — leaking the mode into the call site rather than the run's +recorded config. + +This design replaces the single `--source-run` mechanism with three independent config knobs, +removes the CLI flag, and keeps the existing `preload_symlinks` machinery unchanged. + +## Approach + +Three independent additions to the existing config, all opt-in, all default to today's +backward-compatible behavior: + +```yaml +training: + initial_model: + run: /path/to/old/run # NEW — joins file: and wandb_alias: as a third source + initial_replay_buffer: + run: /path/to/old/run # NEW — replaces the just-shipped training.source_run + +self_play: + enabled: false # NEW — defaults true; explicit on/off +``` + +`--source-run` and the `source_run_overrides` helper are removed. The user expresses each +concern explicitly, either in the yaml or via the existing `-o key=val` override mechanism. + +The three concerns are honored independently inside `train_v2.py` and `trainer.py` — see +Section "Trainer wiring" below. In particular, mixed mode (preload + self-play on) is now a +legitimate combination: preloaded games seed the buffer, new self-play games stream in on +top. + +## Config schema + +```python +class InitialModel(StrictBaseModel): + file: Optional[str] = None + wandb_project: Optional[str] = None + wandb_alias: Optional[str] = None + run: Optional[str] = None # NEW + + # Validator: at most one of {file, wandb_alias, run} may be set. + +class InitialReplayBuffer(StrictBaseModel): + run: str # required when initial_replay_buffer is set. + # Points at a run dir (parent of replay_buffers/), mirroring + # initial_model.run. The preloader reads /replay_buffers/. + +class TrainingConfig(StrictBaseModel): + # ... existing fields ... + initial_replay_buffer: Optional[InitialReplayBuffer] = None # replaces source_run + # ... existing fields ... + +class SelfPlayConfig(StrictBaseModel): + enabled: bool = True # NEW + num_processes: int + # ... other existing fields unchanged; all ignored when enabled=False +``` + +### Cross-field validators + +- `InitialModel.run` joins the existing mutual-exclusion validator. At most one of + `{file, wandb_alias, run}` may be set. +- `Config`-level (new): refuse to load when `not self_play.enabled and + training.initial_replay_buffer is None` — the trainer would hang forever waiting for + games that never arrive. + +### Loader changes + +- `load_config_and_setup_run`'s rust-binary existence check is now gated on + `config.self_play.enabled and config.self_play.program == "rust"`. Today's check fires + whenever `program=="rust"`; the `enabled=False` shortcut means a user can disable + self-play without needing a rust binary present. + +### Removed + +- `training.source_run` (only landed yesterday; nothing depends on it). +- CLI flag `--source-run`. +- Helper `source_run_overrides` in `train_v2.py`. +- Test file `deep_quoridor/test/test_train_v2_args.py` (only existed to cover the removed + helper). + +## Resolving `initial_model.run` → weights file + +`v2/common.py`'s `create_alphazero` already builds `params_dict["model_filename"]` from +`initial_model.file`. The `run` branch reuses that hand-off: + +```python +if im.file: + params_dict["model_filename"] = im.file +elif im.wandb_alias: + params_dict["wandb_alias"] = im.wandb_alias + params_dict["wandb_project"] = im.wandb_project or (...) +elif im.run: + latest = parse_yaml_file_as(LatestModel, Path(im.run) / "models" / "latest.yaml") + params_dict["model_filename"] = latest.filename +``` + +The existing `LatestModel` schema (`v2/yaml_models.py`) already exposes `filename` as an +absolute path to a `.pt` file — exactly what `model_filename` expects. No new on-disk format +or path-resolution rules. + +Architecture-mismatch failure mode: if the prior `.pt` doesn't match the new network shape, +loading fails at `create_alphazero` time with the existing torch error. We don't try to +guard this — same behavior as setting `initial_model.file` to an incompatible checkpoint +today. + +## Trainer wiring + +Today's trainer has a single `offline_mode = config.training.source_run is not None` that +both (a) skips the `games_per_training_step` gate and (b) suppresses the `model_lag` wandb +metric. The two concerns split cleanly with the new config: + +- `selfplay_disabled = not config.self_play.enabled` — the static-buffer property. No + production cadence to wait on; the gate skip in `_should_skip_iteration` is tied to this. +- `omit_model_lag = config.training.initial_replay_buffer is not None` — the lineage + property. Preloaded games' `game_info.model_version` came from a different training run, + making the `model_version - 1 - game_info.model_version` subtraction meaningless. The + `_build_game_log` suppression is tied to this. In mixed mode (preload + self-play on), + later self-play games also lose `model_lag` for the rest of the run; acceptable v1 + simplification — the metric is most useful for diagnosing a self-play-only setup. + +Both helpers (`_should_skip_iteration`, `_build_game_log`) keep their shapes; only the bool +parameter is renamed (`offline_mode` → `selfplay_disabled` / `omit_model_lag`). + +The split matters: with preload-only mode (mixed self-play on), the gate must still be +honored — new games arrive at the normal cadence, and skipping the gate would let the +trainer race ahead of self-play. Tying the gate skip to `not self_play.enabled` gets that +right. + +## `train_v2.py` wiring + +Inside `if __name__ == "__main__":`: + +```python +config = load_config_and_setup_run(args.config_file, runs_dir, overrides=args.overrides) + +# ...existing AI-report check, mp.set_start_method, ShutdownSignal.clear... + +if config.training.initial_replay_buffer is not None: + n_loaded = preload_symlinks( + source_run=Path(config.training.initial_replay_buffer.run), + dest_ready=config.paths.replay_buffers_ready, + buffer_size=config.training.replay_buffer_size, + ) + print(f"Preloaded {n_loaded} games from {config.training.initial_replay_buffer.run}") + +# ... train_process, benchmark_processes, ai_report_process as today ... + +self_play_processes = [] +rust_subprocesses = [] + +if config.self_play.enabled: + if config.self_play.program == "rust": + # ... existing rust branch ... + else: + # ... existing python branch ... +``` + +The shutdown wait loop at the bottom of `train_v2.py` works unchanged — `self_play_processes` +and `rust_subprocesses` are always initialized. + +## Migration + +`source_run` has been on the branch since commit `31ff54e` (2026-06-04) and was never +released or used in any committed yaml config. We delete it outright — no alias, no +deprecation period. + +A user with an existing yaml that sets `training.source_run` will get a pydantic +`extra_forbidden` error on load, with the field name in the error message; they update to +the new schema. + +## Edge cases & failure modes + +- `initial_model.run` set, but `/models/latest.yaml` is missing → fails at + `create_alphazero` time with a clear `FileNotFoundError` (matches today's behavior for + `initial_model.file` pointing at a missing file). +- `initial_model.run` set with two of `{file, wandb_alias, run}` → caught by the pydantic + validator at load time. +- `initial_replay_buffer.run` missing dir or no `.npz` files → fails at preload time via + the existing `preload_symlinks` errors. +- `self_play.enabled=False` AND `initial_replay_buffer is None` → caught by the new + Config-level validator at load time. Clear message: no source of games. +- `self_play.enabled=True` AND `initial_replay_buffer` set → mixed mode. Trainer's existing + ready/-pickup loop handles both naturally. The `omit_model_lag` flag is honored for the + whole run. +- `self_play.enabled=False` AND `program=rust` in yaml → rust binary check skipped; the + program field is unused. No need to override it manually. +- Source run still being written to by another process → out of scope (same as + yesterday's spec); document "point at a dormant source run." + +## Testing + +**Unit tests:** + +- `config_test.py` additions: + - `initial_model.run` accepted, default None. + - `initial_model` rejects setting two or more of `{file, wandb_alias, run}`. + - `self_play.enabled` accepted, defaults True. + - `initial_replay_buffer.run` parses correctly. + - Config-level validator rejects `enabled=False` with no `initial_replay_buffer`. +- `test_trainer_helpers.py` updates: + - Rename bool params; verify the gate-skip is tied to `selfplay_disabled` and `model_lag` + suppression is tied to `omit_model_lag`. The existing 5 tests already cover the truth + table — just rename the params. + +**Deleted:** + +- `test_train_v2_args.py` (the only thing it tested no longer exists). + +**No changes:** + +- `test_offline_preload.py` — preload functions are unchanged. + +**Manual verification:** + +- Re-run a variant of yesterday's smoke test with the new config: build a tiny source run; + run train_v2 with `initial_replay_buffer.run=` and `self_play.enabled=false` in the + yaml; assert the same observable behavior (preload print, symlinks present, no rust + spawn). +- Spot-check mixed mode: same yaml but with `self_play.enabled=true` — boot should preload + AND spawn self-play. + +## Out of scope (YAGNI) + +- Loading replay buffer from anywhere other than a previous run dir (e.g. arbitrary file + list). The sub-config shape leaves room for it (`initial_replay_buffer.from_files: [...]`) + but we don't add it now. +- Loading model from a specific older checkpoint of a run (i.e. `model_5.pt` rather than + latest). Users wanting this still use `initial_model.file: /models/checkpoints/model_5.pt`. +- Cross-run partial weight transfer (architecture mismatch tolerated). Same out-of-scope + note as yesterday's spec. +- Startup validation that prior-run quoridor params match the new config. Same deferred + note as yesterday's spec — failure mode is loud at first batch. diff --git a/docs/superpowers/specs/2026-06-11-run-benchmarks-v2-design.md b/docs/superpowers/specs/2026-06-11-run-benchmarks-v2-design.md new file mode 100644 index 00000000..92878f5b --- /dev/null +++ b/docs/superpowers/specs/2026-06-11-run-benchmarks-v2-design.md @@ -0,0 +1,179 @@ +# `run_benchmarks_v2.py` — design + +**Date:** 2026-06-11 +**Branch:** `jdb/train-on-existing-selfplay-games` +**Status:** approved (design), pending implementation plan + +## Problem + +`train_v2.py` orchestrates four kinds of work — training, self-play, benchmarks, AI report — +and there is no way to run only the benchmarks against an existing run. When training is done +or when iterating on benchmark configuration, the user wants to point at a run dir and start +just the benchmark schedules described in that run's `config.yaml`. + +## Approach + +A new script `deep_quoridor/src/run_benchmarks_v2.py`, parallel to `train_v2.py`. It takes a +run directory as its positional argument, loads the saved `config.yaml`, and spawns the same +`mp.Process` set that `train_v2.py` would spawn for benchmarks — using the existing +`benchmarks.create_benchmark_processes(config)` function unchanged. No changes to +`benchmarks.py`, the config schema, or any other module. + +The script loops like `train_v2.py` does (the user picked "loop like train_v2" during +brainstorming): benchmark processes keep running on their `every:` triggers until +`ShutdownSignal` is set. Termination is via Ctrl-C — the script catches `KeyboardInterrupt`, +signals shutdown, and joins. + +## CLI + +```bash +python deep_quoridor/src/run_benchmarks_v2.py [-o key=val ...] +``` + +Positional argument: +- `` — path to an existing run directory, e.g. `/path/to/runs//`. + +Options: +- `-o, --overrides key=val [key=val ...]` — same shape as `train_v2.py --overrides`. Applied + to the loaded `UserConfig` before `Config.from_user`. Useful for adjusting benchmark + frequency (`benchmarks.0.every=2 minutes`) or trying a different opponent list + (`benchmarks.0.jobs.0.opponents=[random,greedy]`) without editing the saved yaml. + +## Path & config loading + +The standard run-dir layout is `base_dir/runs//`. Given `` the script +derives: + +```python +run_dir = Path(args.run_dir).resolve() +base_dir = str(run_dir.parent.parent) # the parent of "runs/" +# run_id comes from the saved config.yaml, not the directory name, so $DATETIME +# substitution doesn't cause a mismatch. +``` + +Loading uses the existing public `load_user_config(file, overrides=None)` which loads the +yaml and applies overrides in one call, then `Config.from_user`: + +```python +config_yaml = run_dir / "config.yaml" +if not config_yaml.is_file(): + raise FileNotFoundError(f"No config.yaml in {run_dir}") + +user_config = load_user_config(str(config_yaml), overrides=args.overrides) +config = Config.from_user(user_config, base_dir, create_dirs=False) +``` + +`create_dirs=False` so existing directories aren't disturbed and no `config.yaml` snapshot is +rewritten. `load_user_config` is already exported from `v2/config.py`; no schema or helper +visibility changes are needed. + +We pull `run_id` from the loaded `UserConfig`, not from `run_dir.name`. `UserConfig`'s +`replace_datetime_placeholder` validator interpolates `$DATETIME` at load time, so the +in-memory `run_id` matches what was written to disk in the original run — which is also what +the run_dir is named. + +## Startup checks + +Before spawning any process: + +1. `` must exist and be a directory. +2. `/config.yaml` must exist. +3. `/models/latest.yaml` must exist. Without it, `run_benchmark`'s + `LatestModel.wait_for_creation` would block forever (no training is producing models in + this script). Abort with a clear message naming the missing file. +4. `config.benchmarks` must be non-empty. If empty, log + `"No benchmarks configured in /config.yaml; nothing to run."` and exit 0. +5. `ShutdownSignal.clear(config)` — a previous run may have left the signal set, which would + make `run_benchmark`'s loop exit on its first iteration. Clear it before spawning. + +## Orchestration + +```python +mp.set_start_method("spawn", force=True) +ShutdownSignal.clear(config) + +benchmark_processes = benchmarks.create_benchmark_processes(config) +[p.start() for p in benchmark_processes] +print(f"Started {len(benchmark_processes)} benchmark processes") + +try: + # Same shutdown-wait pattern train_v2.py uses, simplified to benchmarks only. + b_count_prev = -1 + while True: + b_count = sum(p.is_alive() for p in benchmark_processes) + if b_count != b_count_prev: + print(f"Waiting for {b_count} benchmark processes") + b_count_prev = b_count + if b_count == 0: + break + time.sleep(1) +except KeyboardInterrupt: + print("\nCaught Ctrl-C; signaling shutdown...") + ShutdownSignal.signal(config) + for p in benchmark_processes: + p.join() + +ShutdownSignal.clear(config) +``` + +The benchmark loop in `run_benchmark` exits when `ShutdownSignal.is_set(config)` returns +True (inside `freq.wait(...)`). Ctrl-C triggers the signal; processes finish their current +iteration and exit cleanly. + +## wandb behavior + +Each benchmark process calls `wandb.init(id=f"{config.run_id}-benchmark-{idx}", +resume="allow", ...)`. Re-running this script appends to the existing benchmark wandb runs — +the right behavior for "I ran some benchmarks during training, now I'm running more after +training finished." + +No new wandb config or behavior. Inherits whatever the saved `config.yaml` says. + +## Failure modes + +- `` not a directory → `FileNotFoundError` or our explicit check, abort. +- `config.yaml` missing → explicit check at startup. +- `latest.yaml` missing → explicit check at startup (avoids the blocking + `wait_for_creation`). +- pydantic validation fails (e.g. a `-o` override is invalid) → propagate the validation + error. +- `config.benchmarks` empty → log and exit 0. +- `` doesn't match the `base_dir/runs//` shape → the derived `base_dir` is + still a valid path; `Config.from_user` succeeds because `create_dirs=False`. The benchmark + processes still find `latest.yaml` via the absolute `paths.latest_model_yaml`. No special + guard needed — non-standard layouts just work as long as `/models/latest.yaml` + resolves. + +## Testing + +**Unit test (small):** + +- A script-level entrypoint test that calls a refactored `main(args)` function with a + mocked `mp.Process` (or by stubbing `benchmarks.create_benchmark_processes` to return + `[]`), confirming that when `config.benchmarks` is empty the script exits 0 with the + expected "nothing to run" log line. + +**Manual smoke verification:** + +- Take an existing run dir (from yesterday's smoke tests or any past training run that + produced a `latest.yaml`). Invoke `run_benchmarks_v2.py `. Confirm: + - Process starts without error. + - It prints `"Started N benchmark processes"`. + - At least one benchmark process logs its first iteration ("Running TournamentBenchmarkJob + …" etc.). + - Ctrl-C cleanly shuts everything down. + +The script orchestration itself is otherwise covered by the existing `benchmarks.py` tests +(if any) and by train_v2's existing flow — we're not changing benchmark execution behavior, +only the entrypoint that triggers it. + +## Out of scope (YAGNI) + +- A `--once` flag that runs each benchmark schedule exactly once and exits. Skipped per the + brainstorming decision. +- Running a subset of benchmarks (e.g. `--benchmark-index 1`). All configured benchmarks + run. +- Changing the `wandb` group/run-id naming for benchmark-only invocations. We deliberately + reuse the same ids so the rerun appends to the original benchmark run. +- A general-purpose "run any subset of train_v2 stages" mode. This script is + benchmarks-only. diff --git a/experiments/2026_05_23_jon_b9w10_performance/config.yaml b/experiments/2026_05_23_jon_b9w10_performance/config.yaml index b0cd8d87..c97710eb 100644 --- a/experiments/2026_05_23_jon_b9w10_performance/config.yaml +++ b/experiments/2026_05_23_jon_b9w10_performance/config.yaml @@ -2,13 +2,13 @@ run_id: jdb-b9w10-baseconfig quoridor: board_size: 9 max_walls: 10 - max_steps: 100 + max_steps: 120 alphazero: network: type: resnet - num_blocks: 6 + num_blocks: 12 num_channels: 32 - mcts_n: 1000 + mcts_n: 8000 mcts_c_puct: 1.2 wandb: project: B9W10 @@ -23,16 +23,17 @@ self_play: program: rust num_processes: 1 mcts_worker_threads: 18 - games_per_process: 128 + games_per_process: 32 eval_batch_size: 1024 - leaf_parallelism: 16 + leaf_parallelism: 128 virtual_loss: 3 enable_tree_reuse: true eval_cache_max_size: 1000000 eval_max_wait_ms: 1 alphazero: mcts_noise_epsilon: 0.25 - temperature: 3.0 + temperature: 0.5 + drop_t_on_step: 16 training: games_per_training_step: 100 learning_rate: 0.001 @@ -50,15 +51,15 @@ benchmarks: prefix: raw times: 10 opponents: - - random - greedy:p_random=0.3,nick=greedy-03 - greedy + - simple:branching_factor=16,nick=simple-bf16 - every: 10 models jobs: - type: tournament prefix: "" times: 10 opponents: - - random - greedy:p_random=0.3,nick=greedy-03 - greedy + - simple:branching_factor=16,nick=simple-bf16