diff --git a/README.md b/README.md index 049d8f3..25274a1 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,39 @@ impact-gate score --warn-at 50000 --block-at 200000 --enforcement block Exit codes. `0` means ok or warn (the change is allowed). `2` means blocked (impact too high under `--enforcement block`). `1` means a usage or environment error. +Every report also lists the **files to consider for refactoring**, ranked by their share +of the impact. The change-level number gates; the per-file ranking points at where the +decay is concentrating, so a file quietly growing into a god-class surfaces as a +candidate before it blocks anything. + +## Grade against a distribution (the curve) + +A raw threshold is hard to set: a typical change's impact varies by orders of magnitude +across languages and projects. Instead of guessing a number, grade a change by its +**percentile** against a distribution, and gate on the percentile. + +```bash +# Build (or refresh) the project's own impact distribution from the merged history. +# Writes .impact-gate-baseline.json. Re-run it as the branch moves. +impact-gate baseline --base-ref main + +# Gate on the grade instead of an absolute number. +impact-gate score --curve --warn-percentile 90 --block-percentile 98 +``` + +The grade blends two distributions: + +- a **seed prior** shipped with the tool — per-language percentile tables built from a + 20-repo open-source corpus, with a pooled fallback for languages not in the table; +- the **project baseline** — the repo's own per-change distribution, walked from the + merged mainline (only landed work; in-flight branches are never reached). + +The blend weights the project by `w = n / (n + K)`, where `n` is the number of landed +changes behind the baseline and `K` (`curve_prior_weight`, default 200) is how much +history it takes to trust the project over the seed. A fresh repo with no baseline file +grades on the seed alone; a deep history leans on itself. The grade shows in every +format next to the raw number. + ## Configure with `.impact-gate.yml` (repo root) ```yaml @@ -59,10 +92,19 @@ block_at: 200000 # impact above which to block enforcement: warn # off, warn, or block. Start on warn. Flip to block when ready. tolerance: 1.0 # CI-adjustable multiplier on both thresholds. Above 1 is more lenient. # measure_config: .impact-measure.yml # optional: ignore globs and language overrides + +# Grading curve (percentile gate). When enabled, warn_at/block_at are ignored and the +# gate uses the percentiles below instead. +curve_enabled: false # gate on the percentile grade instead of absolute numbers +warn_percentile: 90 # grade at or above this warns +block_percentile: 98 # grade at or above this blocks +curve_prior_weight: 200 # K in w = n/(n+K): history needed to trust the project over the seed +baseline_file: .impact-gate-baseline.json # where `impact-gate baseline` caches the distribution ``` CLI flags override the file. A CI job can pass `--tolerance` or `--warn-at`. So a team -can dial tolerance without editing the repo. +can dial tolerance without editing the repo. The curve knobs have flags too: `--curve`, +`--warn-percentile`, `--block-percentile`, `--baseline-file`. ## Use in GitHub Actions @@ -99,9 +141,9 @@ Make the check required in branch protection to gate merges. The comment needs - Core CLI. Score staged, worktree, or range. Warn or block. Text, JSON, markdown. Done. - GitHub Action. Composite action, job-summary report, and a sticky PR comment. Done. -- Baseline and grading curve. Profile the project history to set thresholds - automatically. Blend a seed-corpus prior with the project's own impact distribution. - Grade a change by its percentile. This is next. +- Baseline and grading curve. `impact-gate baseline` profiles the project history; the + gate blends a seed-corpus prior with the project's own distribution and grades a change + by its percentile (`score --curve`). Done. - Distribution. A Dockerfile so it runs on any CI with Docker. A `pip` package. - More CI plugins. A GitLab CI template and a Jenkins shared library. - Hooks and IDE. An `impact-gate install-hook` for pre-commit. Editor integration over LSP. diff --git a/impact_gate/cli.py b/impact_gate/cli.py index ca5332c..38bf8f8 100644 --- a/impact_gate/cli.py +++ b/impact_gate/cli.py @@ -10,11 +10,12 @@ import argparse import os +import subprocess import sys from .core.config import MeasureConfig -from . import __version__, gitio, report +from . import __version__, baseline, gitio, report from .config import ENFORCEMENTS, GateConfig from .engine import score_change @@ -36,12 +37,25 @@ def _add_score_args(p: argparse.ArgumentParser) -> None: p.add_argument("--enforcement", choices=ENFORCEMENTS) p.add_argument("--tolerance", type=float) p.add_argument("--measure-config", help="Surveyor-style YAML for ignore globs etc.") + # grading curve (percentile gate) overrides + p.add_argument("--curve", dest="curve_enabled", action="store_const", const=True, + default=None, help="gate on the change's percentile grade against the " + "baseline distribution instead of absolute thresholds") + p.add_argument("--baseline-file", dest="baseline_file", + help="project baseline cache to grade against (default: " + ".impact-gate-baseline.json)") + p.add_argument("--warn-percentile", dest="warn_percentile", type=float) + p.add_argument("--block-percentile", dest="block_percentile", type=float) + + +# Gate knobs an argparse flag may override on top of the config file, when given. +_OVERRIDE_ATTRS = ("warn_at", "block_at", "enforcement", "tolerance", "measure_config", + "curve_enabled", "baseline_file", "warn_percentile", "block_percentile") def _resolve_config(args) -> GateConfig: cfg = GateConfig.load(args.config, args.repo) - for attr in ("warn_at", "block_at", "enforcement", "tolerance", - "measure_config"): + for attr in _OVERRIDE_ATTRS: val = getattr(args, attr, None) if val is not None: setattr(cfg, attr, val) @@ -59,15 +73,24 @@ def _cmd_score(args) -> int: return 1 score = score_change(changed, mcfg) - level = cfg.level(score.impact) - blocked = cfg.blocks(score.impact) + + grade = None + if cfg.curve_enabled: + bl = baseline.load_baseline(os.path.join(args.repo, cfg.baseline_file)) + grade = baseline.grade_change(score, baseline=bl, + prior_weight_K=cfg.curve_prior_weight) + level = cfg.level_for_grade(grade.percentile) + blocked = cfg.blocks_grade(grade.percentile) + else: + level = cfg.level(score.impact) + blocked = cfg.blocks(score.impact) if args.format == "json": - print(report.render_json(score, cfg, level, args.mode, args.base, blocked)) + print(report.render_json(score, cfg, level, args.mode, args.base, blocked, grade)) elif args.format == "markdown": - print(report.render_markdown(score, cfg, level, args.mode, args.base, blocked)) + print(report.render_markdown(score, cfg, level, args.mode, args.base, blocked, grade)) else: - print(report.render_text(score, cfg, level, args.mode, args.base, blocked)) + print(report.render_text(score, cfg, level, args.mode, args.base, blocked, grade)) if blocked: print("\nimpact-gate: change BLOCKED. Impact exceeds the block threshold. " "Simplify the change or refactor the code it touches, then retry.", @@ -75,6 +98,34 @@ def _cmd_score(args) -> int: return 2 if blocked else 0 +def _cmd_baseline(args) -> int: + """Walk the merged mainline into the project distribution and cache it to disk.""" + cfg = _resolve_config(args) + mcfg = MeasureConfig.load(cfg.measure_config) + out_path = os.path.join(args.repo, cfg.baseline_file) + try: + bl = baseline.build_baseline( + args.repo, mcfg, + base_ref=args.base_ref, + max_commits=args.max_commits, + exclude_subject_pattern=args.exclude_subject_pattern, + ) + except (gitio.DiffError, OSError, ValueError, + subprocess.CalledProcessError) as e: + print(f"impact-gate: could not build baseline (is '{args.base_ref or 'main'}' " + f"a branch with history?): {e}", file=sys.stderr) + return 1 + if bl.n == 0: + print(f"impact-gate: no landed changes found on '{bl.base_ref}'; nothing to " + "baseline. Check --base-ref points at a branch with history.", + file=sys.stderr) + return 1 + baseline.save_baseline(bl, out_path) + print(f"impact-gate: baseline written to {out_path} " + f"({bl.n} observations from '{bl.base_ref}').") + return 0 + + def _cmd_comment(args) -> int: from . import ghapi token = args.token or os.environ.get("GITHUB_TOKEN") @@ -105,6 +156,21 @@ def main(argv: list[str] | None = None) -> int: _add_score_args(s) s.set_defaults(func=_cmd_score) + b = sub.add_parser("baseline", + help="build and cache the project baseline distribution") + b.add_argument("--repo", default=".", help="path to the git repo (default: .)") + b.add_argument("--config", help="path to an .impact-gate.yml (else auto-discovered)") + b.add_argument("--base-ref", dest="base_ref", + help="mainline branch to walk (default: main, or config's base_ref)") + b.add_argument("--max-commits", dest="max_commits", type=int, + help="cap how many recent mainline commits are walked") + b.add_argument("--exclude-subject-pattern", dest="exclude_subject_pattern", + help="regex on a merge subject to skip that MR entirely") + b.add_argument("--baseline-file", dest="baseline_file", + help="where to write the cache (default: .impact-gate-baseline.json)") + b.add_argument("--measure-config", help="Surveyor-style YAML for ignore globs etc.") + b.set_defaults(func=_cmd_baseline) + c = sub.add_parser("comment", help="upsert a sticky PR comment with a report (CI)") c.add_argument("--body-file", help="markdown file to post (default: read stdin)") c.add_argument("--repo-slug", help="owner/name (default: $GITHUB_REPOSITORY)") diff --git a/impact_gate/config.py b/impact_gate/config.py index 1252fea..4e1fc16 100644 --- a/impact_gate/config.py +++ b/impact_gate/config.py @@ -69,6 +69,20 @@ def blocks(self, impact: float) -> bool: one case that fails the gate. In 'warn' mode a too-high change still passes.""" return self.enforcement == "block" and self.level(impact) == "block" + def level_for_grade(self, percentile: float) -> str: + """Curve mode: 'block' / 'warn' / 'ok' from a change's grade percentile. A change + at or above `block_percentile` blocks; at or above `warn_percentile` warns.""" + if percentile >= self.block_percentile: + return "block" + if percentile >= self.warn_percentile: + return "warn" + return "ok" + + def blocks_grade(self, percentile: float) -> bool: + """Curve analogue of `blocks`: fails the gate only under 'block' enforcement.""" + return (self.enforcement == "block" + and self.level_for_grade(percentile) == "block") + @classmethod def load(cls, path: str | None = None, repo_path: str = ".") -> "GateConfig": """Load from an explicit path, else the first `.impact-gate.y*ml` in repo_path.""" diff --git a/impact_gate/report.py b/impact_gate/report.py index 1374a17..f5f5774 100644 --- a/impact_gate/report.py +++ b/impact_gate/report.py @@ -28,8 +28,23 @@ def _thresholds_note(cfg: GateConfig) -> str: return " (" + " · ".join(parts) + ")" if parts else "" +def _grade_source(grade) -> str: + """How the grade was arrived at: pure seed at cold start, else the blend weight.""" + if grade.project_percentile is None: + return "seed only" + return f"blended n={grade.n}, w={grade.weight:.2f}" + + +def _grade_line(grade, cfg: GateConfig) -> str: + """The percentile-grade line for text output, carrying the curve thresholds.""" + lang = grade.language or "pooled" + return (f" grade: {grade.percentile:g}th percentile " + f"(warn ≥ p{cfg.warn_percentile:g} · block ≥ p{cfg.block_percentile:g}; " + f"{lang}, {_grade_source(grade)})") + + def render_text(score: ChangeScore, cfg: GateConfig, level: str, - mode: str, base: str, blocked: bool) -> str: + mode: str, base: str, blocked: bool, grade=None) -> str: if score.empty: return "impact-gate: no source changes to score." # Tag reflects the OUTCOME under the current enforcement, not just the severity. @@ -37,10 +52,13 @@ def render_text(score: ChangeScore, cfg: GateConfig, level: str, # nudge that it will fail once enforcement is 'block'. This is the warn->block on-ramp. tag = "BLOCK" if blocked else ("OK" if level == "ok" else "WARN") desc = _MODE_DESC[mode].format(base=base) - lines = [ - f"Change impact: {score.impact:,} [{tag}]{_thresholds_note(cfg)}", - f" files changed: {score.files_changed} ({desc})", - ] + # In curve mode the grade line carries the (percentile) thresholds; in absolute mode + # they hang off the headline instead. + note = "" if grade is not None else _thresholds_note(cfg) + lines = [f"Change impact: {score.impact:,} [{tag}]{note}"] + if grade is not None: + lines.append(_grade_line(grade, cfg)) + lines.append(f" files changed: {score.files_changed} ({desc})") if level == "block" and not blocked: lines.append(" note: over the block threshold. This will fail once " "enforcement is set to 'block'.") @@ -65,31 +83,47 @@ def render_text(score: ChangeScore, cfg: GateConfig, level: str, def render_json(score: ChangeScore, cfg: GateConfig, level: str, - mode: str, base: str, blocked: bool) -> str: - return json.dumps({ + mode: str, base: str, blocked: bool, grade=None) -> str: + thresholds = { + "warn": cfg.effective_warn(), + "block": cfg.effective_block(), + "enforcement": cfg.enforcement, + "tolerance": cfg.tolerance, + } + if grade is not None: + thresholds["warn_percentile"] = cfg.warn_percentile + thresholds["block_percentile"] = cfg.block_percentile + out = { "impact": score.impact, "level": level, "blocked": blocked, + "curve": cfg.curve_enabled, "files_changed": score.files_changed, "mode": mode, "base": base, - "thresholds": { - "warn": cfg.effective_warn(), - "block": cfg.effective_block(), - "enforcement": cfg.enforcement, - "tolerance": cfg.tolerance, - }, + "thresholds": thresholds, "files": [{"path": f.path, "lang": f.lang, "cost": f.cost, "mutation": f.mutation, "godclass": f.godclass, "mut_fns": f.mut_fns, "new_fns": f.new_fns} for f in score.files], "top_units": [{"path": u.path, "name": u.name, "container": u.container, "cc": u.cc, "wmc_other": u.wmc_other, "cost": u.cost, "kind": u.kind} for u in score.units[:10]], - }, indent=2) + } + if grade is not None: + out["grade"] = { + "percentile": grade.percentile, + "value": grade.value, + "seed_percentile": grade.seed_percentile, + "project_percentile": grade.project_percentile, + "weight": grade.weight, + "n": grade.n, + "language": grade.language, + } + return json.dumps(out, indent=2) def render_markdown(score: ChangeScore, cfg: GateConfig, level: str, - mode: str, base: str, blocked: bool) -> str: + mode: str, base: str, blocked: bool, grade=None) -> str: """GitHub/GitLab-friendly summary. Written to the CI job summary.""" if score.empty: return "**impact-gate:** no source changes to score." @@ -103,11 +137,17 @@ def render_markdown(score: ChangeScore, cfg: GateConfig, level: str, "|---|---|", f"| files changed | {score.files_changed} |", ] - w, b = cfg.effective_warn(), cfg.effective_block() - if w is not None: - lines.append(f"| warn threshold | {int(w):,} |") - if b is not None: - lines.append(f"| block threshold | {int(b):,} |") + if grade is not None: + lines.append(f"| grade | {grade.percentile:g}th percentile " + f"({grade.language or 'pooled'}, {_grade_source(grade)}) |") + lines.append(f"| warn / block percentile | p{cfg.warn_percentile:g} / " + f"p{cfg.block_percentile:g} |") + else: + w, b = cfg.effective_warn(), cfg.effective_block() + if w is not None: + lines.append(f"| warn threshold | {int(w):,} |") + if b is not None: + lines.append(f"| block threshold | {int(b):,} |") lines.append(f"| scope | {desc} |") if level == "block" and not blocked: lines += ["", "> Over the block threshold. This will fail once enforcement " diff --git a/tests/test_cli.py b/tests/test_cli.py index 4d53dea..2426d37 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,8 +1,9 @@ """Layer 3: CLI and gate behavior. Calls cli.main and checks exit code + output.""" import json +import os from impact_gate.cli import main -from gitutil import commit, write +from gitutil import cc_func, commit, write BASE = "def f():\n return 1\n" CHANGED = "def f():\n return 1\n\ndef g():\n return 2\n" @@ -14,6 +15,13 @@ def _prepare(repo): write(repo, "m.py", CHANGED) # worktree change, impact == 2 +def _history(repo): + """Three landed commits on main -> three baseline observations.""" + write(repo, "a.py", cc_func("a", 3)); commit(repo, "m1") + write(repo, "a.py", cc_func("a", 3) + cc_func("b", 4)); commit(repo, "m2") + write(repo, "b.py", cc_func("c", 5)); commit(repo, "m3") + + def test_block_mode_returns_exit_2(repo, capsys): _prepare(repo) code = main(["score", "--repo", str(repo), "--mode", "worktree", @@ -73,3 +81,74 @@ def test_bad_base_returns_exit_1(repo, capsys): code = main(["score", "--repo", str(repo), "--mode", "range", "--base", "nope"]) assert code == 1 assert "merge-base" in capsys.readouterr().err + + +# --------------------------------------------------------------- baseline command + +def test_baseline_command_writes_cache(repo, capsys): + _history(repo) + code = main(["baseline", "--repo", str(repo)]) + assert code == 0 + assert "baseline written" in capsys.readouterr().out + data = json.load(open(os.path.join(str(repo), ".impact-gate-baseline.json"))) + assert data["_meta"]["n"] == 3 # three landed commits + assert len(data["distribution"]) == 3 + assert data["distribution"] == sorted(data["distribution"]) # ascending on disk + + +def test_baseline_command_honours_baseline_file_flag(repo, capsys): + _history(repo) + code = main(["baseline", "--repo", str(repo), "--baseline-file", "custom.json"]) + assert code == 0 + assert os.path.exists(os.path.join(str(repo), "custom.json")) + + +def test_baseline_command_on_empty_history_errs_cleanly(repo, capsys): + # Fresh repo, no commits on main: a friendly error and exit 1, never a traceback. + code = main(["baseline", "--repo", str(repo)]) + assert code == 1 + assert "impact-gate:" in capsys.readouterr().err + + +# ------------------------------------------------------------------ percentile gate + +def test_curve_gate_blocks_a_high_grade_change(repo, capsys): + _history(repo) + main(["baseline", "--repo", str(repo)]) + capsys.readouterr() + # A large change grades high on the curve; block enforcement fails it. + write(repo, "big.py", "".join(cc_func(n, 30) for n in "abcdefgh")) + code = main(["score", "--repo", str(repo), "--mode", "worktree", "--curve", + "--enforcement", "block", + "--warn-percentile", "40", "--block-percentile", "60"]) + out = capsys.readouterr().out + assert code == 2 + assert "[BLOCK]" in out + assert "grade:" in out + + +def test_curve_low_change_passes_and_reports_the_grade(repo, capsys): + _history(repo) + main(["baseline", "--repo", str(repo)]) + capsys.readouterr() + write(repo, "b.py", cc_func("c", 5) + cc_func("z", 2)) # small addition + code = main(["score", "--repo", str(repo), "--mode", "worktree", "--curve", + "--format", "json"]) + data = json.loads(capsys.readouterr().out) + assert code == 0 + assert data["curve"] is True + assert data["blocked"] is False + assert data["grade"]["n"] == 3 # blended against the baseline + assert data["grade"]["language"] == "python" + assert 0 <= data["grade"]["percentile"] <= 100 + + +def test_curve_without_baseline_grades_on_seed_only(repo, capsys): + _prepare(repo) # no baseline file built + code = main(["score", "--repo", str(repo), "--mode", "worktree", "--curve", + "--format", "json"]) + data = json.loads(capsys.readouterr().out) + assert code == 0 + assert data["grade"]["n"] == 0 + assert data["grade"]["project_percentile"] is None # pure seed at cold start + assert data["grade"]["percentile"] == data["grade"]["seed_percentile"] diff --git a/tests/test_data.py b/tests/test_data.py index 3732365..078eee6 100644 --- a/tests/test_data.py +++ b/tests/test_data.py @@ -57,6 +57,22 @@ def test_validate_rejects_bad_curve_knobs(): GateConfig(curve_prior_weight=-1).validate() +def test_level_for_grade_maps_percentile_to_verdict(): + cfg = GateConfig(warn_percentile=90, block_percentile=98) + assert cfg.level_for_grade(50) == "ok" + assert cfg.level_for_grade(90) == "warn" # at the warn line + assert cfg.level_for_grade(95) == "warn" + assert cfg.level_for_grade(98) == "block" # at the block line + assert cfg.level_for_grade(99.9) == "block" + + +def test_blocks_grade_only_under_block_enforcement(): + high = 99.0 + assert GateConfig(enforcement="block", block_percentile=98).blocks_grade(high) is True + # A high grade under warn enforcement is allowed (the warn->block on-ramp). + assert GateConfig(enforcement="warn", block_percentile=98).blocks_grade(high) is False + + def test_config_file_overrides_curve_knobs(tmp_path): (tmp_path / ".impact-gate.yml").write_text( "curve_enabled: true\nwarn_percentile: 80\nblock_percentile: 95\n"