diff --git a/tests/test_benchmark_agents.py b/tests/test_benchmark_agents.py index 49a25972..12bf63fb 100644 --- a/tests/test_benchmark_agents.py +++ b/tests/test_benchmark_agents.py @@ -507,3 +507,25 @@ def fake_run_agent(self, repo_path, prompt, model, base_url, api_key, provider, ) assert any(event.type == 'tool_started' for event in result.events) + + +def test_villani_run_agent_discovers_nested_telemetry_files(monkeypatch, tmp_path: Path) -> None: + from villani_code.benchmark.adapters.base import AdapterRunResult + from villani_code.benchmark.models import FieldQuality, TelemetryQuality + + def fake_run_agent(self, repo_path, prompt, model, base_url, api_key, provider, timeout, benchmark_config_json=None, debug_dir=None): + return AdapterRunResult(stdout='ok', stderr='', exit_code=0, timeout=False, runtime_seconds=0.1, telemetry_quality=TelemetryQuality.INFERRED, telemetry_field_quality_map={'num_shell_commands': FieldQuality.INFERRED}, events=[]) + + monkeypatch.setattr('villani_code.benchmark.agents.base.AgentRunner.run_agent', fake_run_agent) + mdir = tmp_path / '.villani_code' / 'missions' / 'm1' + mdir.mkdir(parents=True) + (mdir / 'runtime_events.jsonl').write_text('{\"ts\": 11.0, \"type\": \"tool_started\", \"name\": \"Read\"}\n', encoding='utf-8') + ddir = tmp_path / 'villani_debug' / 'run1' + ddir.mkdir(parents=True) + (ddir / 'events.jsonl').write_text('{\"timestamp\": 12.0, \"event\": \"tool_finished\", \"name\": \"Read\"}\n', encoding='utf-8') + + runner = VillaniAgentRunner() + result = runner.run_agent(repo_path=tmp_path, prompt='fix', model='m', base_url=None, api_key=None, provider='anthropic', timeout=10) + types = {e.type for e in result.events} + assert 'tool_started' in types + assert 'tool_finished' in types diff --git a/tests/test_state_runtime.py b/tests/test_state_runtime.py index 16e604a7..3c2fcaa8 100644 --- a/tests/test_state_runtime.py +++ b/tests/test_state_runtime.py @@ -471,35 +471,47 @@ def save_inventory(self, _inventory): def test_fail_first_localization_runs_without_strong_signal(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: _seed_repo(tmp_path) + calls: dict[str, object] = {} events: list[dict] = [] runner = SimpleNamespace( repo=tmp_path, benchmark_config=SimpleNamespace(visible_verification=["pytest -q"], expected_files=[]), + _task_execution_contract=state_runtime.TaskExecutionContract(["pytest -q"], [], [], [], True, False), _execution_plan=SimpleNamespace(relevant_files=[]), _pending_verification="", event_callback=events.append, ) - def fake_run(cmd, **_kwargs): - assert cmd == ["bash", "-lc", "pytest -q"] - - class P: - returncode = 1 - stdout = "FAILED tests/test_api.py::test_runtime - AssertionError: boom" - stderr = "" - - return P() + def fake_verify(_runner, command, timeout_seconds=120, cwd=None): + calls["command"] = command + calls["timeout_seconds"] = timeout_seconds + calls["cwd"] = cwd + return { + "command": command, + "exit_code": 1, + "timed_out": False, + "stdout_excerpt": "FAILED tests/test_api.py::test_runtime - AssertionError: boom", + "stderr_excerpt": "", + "first_failing_test": "tests/test_api.py::test_runtime", + "error_summary": "AssertionError: boom", + "raw_failure_excerpt": "FAILED tests/test_api.py::test_runtime - AssertionError: boom", + "passed": False, + } - monkeypatch.setattr(state_runtime.subprocess, "run", fake_run) + monkeypatch.setattr(state_runtime, "run_task_verification_command", fake_verify) evidence = state_runtime.run_pre_edit_failure_localization(runner) assert evidence is not None assert evidence["first_failing_test"] == "tests/test_api.py::test_runtime" + assert calls["command"] == "pytest -q" + assert calls["timeout_seconds"] == 120 + assert calls["cwd"] is not None + assert Path(str(calls["cwd"])) != tmp_path assert any(e.get("type") == "pre_edit_failure_signal_attempted" for e in events) assert any(e.get("type") == "pre_edit_failure_signal_captured" for e in events) -def test_fail_first_localization_skips_with_clear_expected_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: +def test_fail_first_localization_runs_with_clear_expected_file(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: _seed_repo(tmp_path) events: list[dict] = [] runner = SimpleNamespace( @@ -514,13 +526,17 @@ def test_fail_first_localization_skips_with_clear_expected_file(tmp_path: Path, ) def fake_run(_cmd, **_kwargs): - raise AssertionError("verification should be skipped when expected file is clear") + class P: + returncode = 1 + stdout = "FAILED tests/test_api.py::test_runtime - AssertionError: boom" + stderr = "" + return P() monkeypatch.setattr(state_runtime.subprocess, "run", fake_run) evidence = state_runtime.run_pre_edit_failure_localization(runner) - assert evidence is None - assert any(e.get("type") == "pre_edit_failure_signal_skipped" for e in events) + assert evidence is not None + assert any(e.get("type") == "pre_edit_failure_signal_captured" for e in events) def test_parse_failure_signal_extracts_test_and_traceback() -> None: @@ -664,3 +680,135 @@ def test_diagnosis_confidence_weak_without_file_evidence(tmp_path: Path) -> None } confidence = state_runtime.classify_diagnosis_target_confidence(runner, diagnosis, failure_evidence=None) assert confidence == "weak" + + +def test_task_evidence_packet_contains_excerpt_and_scope() -> None: + runner = SimpleNamespace( + benchmark_config=SimpleNamespace(enabled=True, visible_verification=["pytest -q"], allowlist_paths=["src/app"], expected_files=["src/app/core.py"]), + _execution_plan=SimpleNamespace(relevant_files=["tests/test_core.py"]), + ) + packet = state_runtime.build_task_evidence_packet(runner, {"exit_code":1, "timed_out":False, "error_summary":"AssertionError: x"}) + assert "Failure excerpt:" in packet + assert "Allowed edit paths: src/app" in packet + + +def test_build_task_execution_contract_from_benchmark_metadata() -> None: + runner = SimpleNamespace( + benchmark_config=SimpleNamespace(visible_verification=["pytest -q"], allowlist_paths=["src"], expected_files=["src/app.py"]), + _execution_plan=SimpleNamespace(relevant_files=["tests/test_app.py"]), + ) + contract = state_runtime.build_task_execution_contract(runner, "fix bug") + assert contract.verification_commands == ["pytest -q"] + assert "src" in contract.allowed_edit_paths + assert contract.requires_patch is True + +def test_run_task_verification_command_uses_portable_shell(monkeypatch, tmp_path: Path) -> None: + calls = {} + runner = SimpleNamespace(repo=tmp_path) + def fake_run(cmd, **kwargs): + calls['shell'] = kwargs.get('shell') + class P: returncode=0; stdout='ok'; stderr='' + return P() + monkeypatch.setattr(state_runtime.subprocess, 'run', fake_run) + out = state_runtime.run_task_verification_command(runner, 'pytest -q') + assert calls['shell'] is True + assert out['passed'] is True + + +def test_pre_edit_localization_uses_shared_verification_helper(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + _seed_repo(tmp_path) + calls = {} + events: list[dict] = [] + runner = SimpleNamespace( + repo=tmp_path, + benchmark_config=SimpleNamespace(visible_verification=["pytest -q"], expected_files=[]), + _execution_plan=SimpleNamespace(relevant_files=[]), + _task_execution_contract=state_runtime.TaskExecutionContract(["pytest -q"], [], [], [], True, False), + _pending_verification="", + event_callback=events.append, + ) + + def fake_verify(_runner, command, timeout_seconds=120, cwd=None): + calls['command'] = command + calls['cwd'] = cwd + calls['timeout'] = timeout_seconds + return {"command": command, "exit_code": 1, "timed_out": False, "first_failing_test": "t::x", "error_summary": "AssertionError", "raw_failure_excerpt": "FAILED t::x"} + + monkeypatch.setattr(state_runtime, 'run_task_verification_command', fake_verify) + evidence = state_runtime.run_pre_edit_failure_localization(runner) + assert evidence is not None + assert calls['command'] == 'pytest -q' + assert calls['cwd'] is not None + + +def test_inject_task_evidence_message_uses_safe_helper(monkeypatch: pytest.MonkeyPatch) -> None: + events = [] + runner = SimpleNamespace(event_callback=events.append) + calls = {} + def fake_prepend(messages, text): + calls['text'] = text + return False + monkeypatch.setattr(state_runtime, 'prepend_text_to_latest_safe_user_message', fake_prepend) + ok = state_runtime.inject_task_evidence_message(runner, [{"role":"user","content":"x"}], "pkt") + assert ok is False + assert any(e.get('type') == 'task_evidence_injection_failed' for e in events) + + +def test_runner_preserves_task_execution_contract_during_run(tmp_path: Path) -> None: + class Client: + def create_message(self, _payload, stream): + assert stream is False + return {"role": "assistant", "content": []} + + runner = Runner(client=Client(), repo=tmp_path, model="m", stream=False, benchmark_config=SimpleNamespace(enabled=False, visible_verification=["pytest -q"], expected_files=["src/app.py"], allowlist_paths=["src"], task_id="t", allowed_support_files=[])) + out = runner.run("fix bug") + assert out + assert runner._task_execution_contract is not None + assert runner._task_execution_contract.verification_commands == ["pytest -q"] + + +def test_post_edit_verification_uses_contract_command(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + calls = {} + runner = SimpleNamespace( + repo=tmp_path, + _task_execution_contract=state_runtime.TaskExecutionContract(["pytest -q"], [], [], [], True, False), + _patch_sanity_retry_pending=False, + _first_attempt_write_lock_active=True, + _task_verification_repair_attempts=1, + event_callback=lambda _e: None, + ) + + monkeypatch.setattr(state_runtime, '_run_patch_sanity_check', lambda _r: {"ran": False, "passed": True, "checked_files": []}) + monkeypatch.setattr(state_runtime, 'run_verification', lambda _r, _t: 'verification-ran') + + def fake_task_verify(_runner, command, timeout_seconds=120, cwd=None): + calls['command'] = command + return {"command": command, "passed": True, "timed_out": False, "exit_code": 0} + + monkeypatch.setattr(state_runtime, 'run_task_verification_command', fake_task_verify) + out = state_runtime.run_post_edit_verification(runner, 'Patch execution') + assert out == 'verification-ran' + assert calls['command'] == 'pytest -q' + assert runner._task_verification_repair_attempts == 0 + + +def test_first_model_request_contains_task_evidence_packet(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + captured = {} + class StopAfterFirstModelRequest(Exception): + pass + class Client: + def create_message(self, payload, stream): + captured['payload'] = payload + raise StopAfterFirstModelRequest + + runner = Runner(client=Client(), repo=tmp_path, model='m', stream=False, benchmark_config=SimpleNamespace(enabled=False, visible_verification=['pytest -q'], expected_files=['src/app.py'], allowlist_paths=['src'], task_id='t', allowed_support_files=[])) + monkeypatch.setattr(state_runtime, 'run_pre_edit_failure_localization', lambda _r: {"command":"pytest -q", "exit_code":1, "timed_out":False, "error_summary":"AssertionError: boom", "raw_failure_excerpt":"FAILED tests/test_api.py::test_runtime - AssertionError: boom"}) + monkeypatch.setattr(state_runtime, 'run_pre_edit_diagnosis', lambda *_a, **_k: None) + with pytest.raises(StopAfterFirstModelRequest): + runner.run('fix bug') + text = str(captured['payload']['messages'][0]['content']) + assert 'Task evidence:' in text + assert 'Verification command:' in text + assert 'Failure excerpt:' in text + assert 'Allowed edit paths:' in text + assert 'AssertionError: boom' in text diff --git a/villani_code/benchmark/agents/villani.py b/villani_code/benchmark/agents/villani.py index 89a1936a..a43c6609 100644 --- a/villani_code/benchmark/agents/villani.py +++ b/villani_code/benchmark/agents/villani.py @@ -4,6 +4,7 @@ import sys import time from pathlib import Path +from typing import Iterable from villani_code.benchmark.adapters.base import AdapterEvent, AdapterRunResult from villani_code.benchmark.agents.base import AgentRunner @@ -77,19 +78,31 @@ def run_agent( benchmark_config_json=benchmark_config_json, debug_dir=debug_dir, ) - events_file = repo_path / ".villani_code" / "runtime_events.jsonl" events: list[AdapterEvent] = [] - if events_file.exists(): - for raw in events_file.read_text(encoding="utf-8").splitlines(): - if not raw.strip(): - continue - payload = json.loads(raw) - runtime_type = str(payload.get("type") or "").strip() - if not runtime_type: - runtime_type = str(payload.get("event") or "").strip() - if not runtime_type: - runtime_type = "runtime_event" - events.append(AdapterEvent(type=runtime_type, timestamp=float(payload.get("ts", time.time())), payload=payload)) + candidates: list[Path] = [] + candidates.extend(sorted((repo_path / ".villani_code" / "missions").glob("*/runtime_events.jsonl"))) + candidates.extend(sorted((repo_path / ".villani_code" / "missions").glob("*/tool_calls.jsonl"))) + candidates.extend(sorted((repo_path / "villani_debug").glob("*/events.jsonl"))) + candidates.extend(sorted((repo_path / "villani_debug").glob("*/tool_calls.jsonl"))) + legacy = repo_path / ".villani_code" / "runtime_events.jsonl" + if legacy.exists(): + candidates.append(legacy) + if candidates: + groups: dict[Path, list[Path]] = {} + for cp in candidates: + groups.setdefault(cp.parent, []).append(cp) + best_parent = max(groups.keys(), key=lambda pp: max(x.stat().st_mtime for x in groups[pp] if x.exists())) + for path in sorted(groups[best_parent], key=lambda x: x.name): + for raw in path.read_text(encoding="utf-8", errors="replace").splitlines(): + if not raw.strip(): + continue + try: + payload = json.loads(raw) + except Exception: + continue + runtime_type = str(payload.get("type") or payload.get("event") or "runtime_event").strip() or "runtime_event" + ts = float(payload.get("ts") or payload.get("timestamp") or time.time()) + events.append(AdapterEvent(type=runtime_type, timestamp=ts, payload=payload)) return AdapterRunResult( **base.model_dump(exclude={"events", "telemetry_quality", "telemetry_field_quality_map"}), events=base.events + events, diff --git a/villani_code/state.py b/villani_code/state.py index 1a2c80d4..04c3b5b7 100644 --- a/villani_code/state.py +++ b/villani_code/state.py @@ -513,6 +513,12 @@ def __init__( self._current_verification_before_contents: dict[str, str] = {} self._verification_baseline_changed: set[str] = set() self._scope_expansion_used = False + self._last_task_verification_result = None + self._last_task_verification_passed = False + self._last_task_verification_failed = False + self._task_verification_repair_attempts = 0 + self._out_of_scope_mutation_paths: set[str] = set() + self._out_of_scope_mutation_detected = False self._task_mode: TaskMode = TaskMode.GENERAL self._task_contract: dict[str, Any] = {} self._last_verification_fingerprint = "" @@ -736,16 +742,44 @@ def run( self._ensure_project_memory_and_plan(instruction) self._task_mode = classify_task_mode(instruction) diagnosis = None + from villani_code import state_runtime + self._task_execution_contract = state_runtime.build_task_execution_contract(self, instruction) diagnosed_target_file = "" required_initial_read = "" initial_read_enforced = False pre_edit_failure_evidence = None diagnosis_confidence = "weak" - if self.small_model or self.villani_mode or self.benchmark_config.enabled: + if self.small_model or self.villani_mode or bool(self._task_execution_contract.verification_commands): try: from villani_code import state_runtime pre_edit_failure_evidence = state_runtime.run_pre_edit_failure_localization(self) + evidence_packet = state_runtime.build_task_evidence_packet(self, pre_edit_failure_evidence) + if evidence_packet: + self.event_callback({ + "type": "task_evidence_packet_built", + "byte_length": len(evidence_packet.encode("utf-8", errors="replace")), + "has_verification_command": "Verification command:" in evidence_packet, + "has_failure_excerpt": "Failure excerpt:" in evidence_packet, + "has_allowed_edit_paths": "Allowed edit paths:" in evidence_packet, + "has_expected_files": "Candidate files:" in evidence_packet, + }) + injected = state_runtime.inject_task_evidence_message(self, messages, evidence_packet) + self.event_callback({ + "type": "task_evidence_packet_injected" if injected else "task_evidence_packet_injection_failed", + "byte_length": len(evidence_packet.encode("utf-8", errors="replace")), + "has_verification_command": "Verification command:" in evidence_packet, + "has_failure_excerpt": "Failure excerpt:" in evidence_packet, + "has_allowed_edit_paths": "Allowed edit paths:" in evidence_packet, + "has_expected_files": "Candidate files:" in evidence_packet, + }) + if not injected: + messages.append({"role": "user", "content": [{"type": "text", "text": evidence_packet}]}) + fallback_ok = state_runtime.prepend_text_to_latest_safe_user_message(messages, "Task evidence fallback attached.") + if fallback_ok: + self.event_callback({"type": "task_evidence_packet_fallback_injected"}) + else: + raise RuntimeError("task evidence injection failed before first model call") diagnosis = state_runtime.run_pre_edit_diagnosis( self, instruction, @@ -899,6 +933,38 @@ def run( self._last_validation_artifact_signature = "" self._last_emitted_validation_fingerprint = "" self._scope_expansion_used = False + self._last_task_verification_result = None + self._last_task_verification_passed = False + self._last_task_verification_failed = False + self._task_verification_repair_attempts = 0 + def _task_requires_patch() -> bool: + text = instruction.lower() + asks_fix = any(k in text for k in ["fix", "implement", "update", "patch"]) + contract = getattr(self, "_task_execution_contract", None) + if contract is None: + return asks_fix + return bool(contract.requires_patch or asks_fix) + + def _has_in_scope_patch() -> bool: + changed = set(self._git_changed_files()) - set(baseline_changed) + if not changed: + return False + contract = getattr(self, "_task_execution_contract", None) + if contract is None: + return True + scoped = { + str(p).replace("\\", "/").lstrip("./") + for p in (list(contract.allowed_edit_paths) + list(contract.expected_files)) + if str(p).strip() + } + if not scoped: + return True + normalized_changed = {str(c).replace("\\", "/").lstrip("./") for c in changed} + return any( + (c in scoped) or any(c.startswith(prefix.rstrip("/") + "/") for prefix in scoped) + for c in normalized_changed + ) + self._first_attempt_write_lock_active = bool(required_initial_read) self._first_attempt_locked_target = required_initial_read if self._first_attempt_write_lock_active: @@ -1329,6 +1395,46 @@ def _budget_reason( self.event_callback({"type": "benchmark_scope_reminder_injected", "task_id": self.benchmark_config.task_id, "reason": "no_meaningful_edit"}) messages.append({"role": "user", "content": [{"type": "text", "text": reminder}]}) continue + if _task_requires_patch() and not _has_in_scope_patch(): + self._benchmark_noop_completion_attempts += 1 + if self._benchmark_noop_completion_attempts > 2: + return _finish_bounded(response, "incomplete_no_patch", False) + messages.append({"role": "user", "content": [{"type": "text", "text": "You described or implied a fix, but no in-scope repository file was modified. Make exactly one targeted Patch or Write tool call now. Do not summarise."}]}) + continue + if self._out_of_scope_mutation_detected: + current_changed = {str(p).replace("\\", "/").lstrip("./") for p in self._git_changed_files()} + still_offending = sorted(current_changed.intersection(self._out_of_scope_mutation_paths)) + if still_offending: + messages.append({"role": "user", "content": [{"type": "text", "text": "Out-of-scope file changes are still present. Revert those files or remove the invalid changes before completing."}]}) + continue + self._out_of_scope_mutation_detected = False + self._out_of_scope_mutation_paths = set() + contract = getattr(self, "_task_execution_contract", None) + if contract and contract.verification_commands and _has_in_scope_patch() and self._last_task_verification_failed: + self._task_verification_repair_attempts += 1 + if self._task_verification_repair_attempts > 2: + return _finish_bounded(response, "verification_failed_after_patch", False) + vr = self._last_task_verification_result or {} + changed_files = sorted(set(self._git_changed_files()) - set(baseline_changed)) + repair = [ + "Verification failed after your patch.", + "", + f"Command:\n{vr.get('command','')}", + "", + f"Exit code:\n{vr.get('exit_code')}", + "", + f"Timed out:\n{bool(vr.get('timed_out'))}", + "", + f"Failure excerpt:\n{vr.get('error_summary') or vr.get('stderr_excerpt') or vr.get('stdout_excerpt') or 'n/a'}", + "", + f"Files changed:\n{', '.join(changed_files) or 'none'}", + "", + "Do not claim completion. Make one targeted repair patch, or revert the bad patch if the failure shows the change is wrong.", + ] + if vr.get("timed_out"): + repair.append("The verification command timed out. Treat this as a failure signal. Inspect loops, pagination, waits, subprocesses, or unbounded retries related to your change.") + messages.append({"role":"user","content":[{"type":"text","text":"\n".join(repair)}]}) + continue reason = _budget_reason(completed=True) if reason: return _finish_bounded(response, reason, reason == "completed") diff --git a/villani_code/state_runtime.py b/villani_code/state_runtime.py index 9e03824d..b6a816df 100644 --- a/villani_code/state_runtime.py +++ b/villani_code/state_runtime.py @@ -10,6 +10,7 @@ import sys import tempfile from typing import Any +from dataclasses import dataclass from villani_code.autonomy import VerificationStatus from villani_code.indexing import DEFAULT_IGNORE, RepoIndex @@ -147,6 +148,56 @@ def parse_failure_signal(stdout: str, stderr: str) -> dict[str, Any]: return evidence +@dataclass +class TaskExecutionContract: + verification_commands: list[str] + allowed_edit_paths: list[str] + expected_files: list[str] + relevant_spec_or_test_files: list[str] + requires_patch: bool + allows_new_files: bool = False + +def build_task_execution_contract(runner: Any, instruction: str = "") -> TaskExecutionContract: + cfg = getattr(runner, "benchmark_config", None) + plan = getattr(runner, "_execution_plan", None) + verification_commands = list(getattr(cfg, "visible_verification", []) if cfg else []) + allowed = list(getattr(cfg, "allowlist_paths", []) if cfg else []) + expected = list(getattr(cfg, "expected_files", []) if cfg else []) + relevant = list(getattr(plan, "relevant_files", []) if plan else []) + txt = instruction.lower() + asks_fix = any(k in txt for k in ["fix", "implement", "update", "patch"]) + requires_patch = bool(asks_fix or verification_commands or allowed or expected) + return TaskExecutionContract(verification_commands, allowed, expected, relevant, requires_patch, False) + +def run_task_verification_command( + runner: Any, command: str, timeout_seconds: int = 120, cwd: Path | None = None +) -> dict[str, Any]: + result = {"command": command, "exit_code": None, "timed_out": False, "stdout_excerpt": "", "stderr_excerpt": "", "first_failing_test": "", "error_summary": "", "raw_failure_excerpt": "", "passed": False} + try: + proc = subprocess.run( + command, + cwd=cwd or runner.repo, + shell=True, + capture_output=True, + text=True, + timeout=timeout_seconds, + ) + result["exit_code"] = int(proc.returncode) + result["stdout_excerpt"] = "\n".join(str(proc.stdout or "").splitlines()[:20]) + result["stderr_excerpt"] = "\n".join(str(proc.stderr or "").splitlines()[:20]) + if proc.returncode != 0: + result.update(parse_failure_signal(proc.stdout, proc.stderr)) + result["passed"] = proc.returncode == 0 + except subprocess.TimeoutExpired as exc: + result["timed_out"] = True + result["exit_code"] = 124 + result["stdout_excerpt"] = "\n".join(str(exc.stdout or "").splitlines()[:20]) + result["stderr_excerpt"] = "\n".join(str(exc.stderr or "").splitlines()[:20]) + except Exception as exc: + result["exit_code"] = -1 + result["error_summary"] = f"command_error:{exc.__class__.__name__}" + return result + def _has_useful_failure_signal(evidence: dict[str, Any] | None) -> bool: if not evidence: return False @@ -157,21 +208,16 @@ def _has_useful_failure_signal(evidence: dict[str, Any] | None) -> bool: def run_pre_edit_failure_localization(runner: Any) -> dict[str, Any] | None: - cfg = getattr(runner, "benchmark_config", None) - visible_commands = list(getattr(cfg, "visible_verification", []) if cfg else []) + contract = getattr(runner, "_task_execution_contract", None) + if contract is None: + contract = build_task_execution_contract(runner) + visible_commands = list(getattr(contract, "verification_commands", []) or []) visible_command = str(visible_commands[0]).strip() if visible_commands else "" - expected_file = _single_clear_file(list(getattr(cfg, "expected_files", []) if cfg else [])) - plan = getattr(runner, "_execution_plan", None) - relevant_file = _single_clear_file(list(getattr(plan, "relevant_files", []) if plan else [])) - has_traceback = bool(getattr(runner, "_pending_verification", "").strip()) - broad_visible = _is_broad_visible_verification(visible_command) - strong_signal = bool(expected_file or relevant_file or has_traceback or not broad_visible) - - if strong_signal or not visible_command: + if not visible_command: runner.event_callback( { "type": "pre_edit_failure_signal_skipped", - "reason": "strong_signal" if strong_signal else "missing_visible_verification", + "reason": "missing_visible_verification", "visible_verification_command": visible_command, } ) @@ -186,13 +232,10 @@ def run_pre_edit_failure_localization(runner: Any) -> dict[str, Any] | None: try: with tempfile.TemporaryDirectory(prefix="villani-pre-edit-") as temp_root: isolated_repo = Path(temp_root) / "repo" - shutil.copytree(runner.repo, isolated_repo) + shutil.copytree(runner.repo, isolated_repo, ignore=shutil.ignore_patterns(".git",".villani_code","villani_debug","__pycache__",".pytest_cache",".mypy_cache",".ruff_cache","node_modules","dist","build")) runner.event_callback({"type": "pre_edit_failure_signal_isolated", "isolated": True}) - proc = subprocess.run( - ["bash", "-lc", visible_command], - cwd=isolated_repo, - capture_output=True, - text=True, + evidence = run_task_verification_command( + runner, visible_command, timeout_seconds=120, cwd=isolated_repo ) except Exception as exc: # pragma: no cover - defensive path runner.event_callback( @@ -204,23 +247,13 @@ def run_pre_edit_failure_localization(runner: Any) -> dict[str, Any] | None: ) return None - evidence: dict[str, Any] = { - "first_failing_test": "", - "traceback_file": "", - "traceback_line": None, - "error_summary": "", - "raw_failure_excerpt": "", - "command": visible_command, - "exit_code": int(proc.returncode), - } - if proc.returncode != 0: - evidence.update(parse_failure_signal(proc.stdout, proc.stderr)) + evidence.setdefault("command", visible_command) runner.event_callback( { "type": "pre_edit_failure_signal_captured", "visible_verification_command": visible_command, - "exit_code": int(proc.returncode), + "exit_code": int(evidence.get("exit_code", -1) or -1), "failure_evidence_extracted": _has_useful_failure_signal(evidence), "first_failing_test": evidence.get("first_failing_test", ""), "traceback_file": evidence.get("traceback_file", ""), @@ -230,6 +263,33 @@ def run_pre_edit_failure_localization(runner: Any) -> dict[str, Any] | None: return evidence +def inject_task_evidence_message(runner: Any, messages: list[dict[str, Any]], packet: str) -> bool: + ok = prepend_text_to_latest_safe_user_message(messages, packet) + if not ok: + runner.event_callback({"type": "task_evidence_injection_failed"}) + return ok + +def build_task_evidence_packet(runner: Any, failure_evidence: dict[str, Any] | None = None) -> str: + contract = getattr(runner, "_task_execution_contract", None) + if contract is None: + contract = build_task_execution_contract(runner) + if not (contract.verification_commands or contract.allowed_edit_paths or contract.expected_files): + return "" + lines = ["Task evidence:"] + cmd = ", ".join(contract.verification_commands[:1]) + lines.append(f"- Verification command: {cmd or 'n/a'}") + if failure_evidence is not None: + lines.append(f"- Exit code: {failure_evidence.get('exit_code')}") + lines.append(f"- Timed out: {bool(failure_evidence.get('timed_out', False))}") + lines.append(f"- First failing test: {failure_evidence.get('first_failing_test') or 'n/a'}") + lines.append(f"- Failure excerpt: {failure_evidence.get('error_summary') or failure_evidence.get('raw_failure_excerpt', '')[:220] or 'n/a'}") + lines.append(f"- Allowed edit paths: {', '.join(contract.allowed_edit_paths[:6]) or 'n/a'}") + lines.append(f"- Candidate files: {', '.join(contract.expected_files[:6]) or 'n/a'}") + lines.append(f"- Relevant test/spec files if known: {', '.join(contract.relevant_spec_or_test_files[:6]) or 'n/a'}") + lines.append("- Use verification and source evidence before patching. Apply an actual in-scope patch before summarising.") + lines.append("- Do not create scratch/helper files unless explicitly requested or in scope.") + return "\n".join(lines) + def classify_diagnosis_target_confidence( runner: Any, diagnosis: dict[str, str], @@ -856,6 +916,15 @@ def run_post_edit_verification(runner: Any, trigger: str = "edit") -> str: ) runner._patch_sanity_retry_pending = False verification = run_verification(runner, trigger) + contract = getattr(runner, "_task_execution_contract", None) + if contract and contract.verification_commands: + task_result = run_task_verification_command(runner, contract.verification_commands[0]) + runner._last_task_verification_result = task_result + runner._last_task_verification_passed = bool(task_result.get("passed") and not task_result.get("timed_out")) + runner._last_task_verification_failed = not runner._last_task_verification_passed + if runner._last_task_verification_passed: + runner._task_verification_repair_attempts = 0 + runner.event_callback({"type":"task_verification_result","result":task_result,"passed":runner._last_task_verification_passed}) runner._first_attempt_write_lock_active = False return verification @@ -904,6 +973,15 @@ def run_post_edit_verification(runner: Any, trigger: str = "edit") -> str: } ) verification = run_verification(runner, f"{trigger} (after_sanity_retry_failed)") + contract = getattr(runner, "_task_execution_contract", None) + if contract and contract.verification_commands: + task_result = run_task_verification_command(runner, contract.verification_commands[0]) + runner._last_task_verification_result = task_result + runner._last_task_verification_passed = bool(task_result.get("passed") and not task_result.get("timed_out")) + runner._last_task_verification_failed = not runner._last_task_verification_passed + if runner._last_task_verification_passed: + runner._task_verification_repair_attempts = 0 + runner.event_callback({"type":"task_verification_result","result":task_result,"passed":runner._last_task_verification_passed}) runner._first_attempt_write_lock_active = False return verification diff --git a/villani_code/state_tooling.py b/villani_code/state_tooling.py index c84c64ca..288df230 100644 --- a/villani_code/state_tooling.py +++ b/villani_code/state_tooling.py @@ -622,6 +622,45 @@ def execute_tool_with_policy( forced=False, turn_index=runner._current_turn_index if isinstance(getattr(runner, "_current_turn_index", None), int) else 0, ) + if tool_name == "Bash" and not result.get("is_error"): + contract = getattr(runner, "_task_execution_contract", None) + if contract and (contract.allowed_edit_paths or contract.expected_files): + baseline = set(getattr(runner, "_verification_baseline_changed", set())) + changed = { + str(p).replace("\\", "/").lstrip("./") + for p in runner._git_changed_files() + if str(p).strip() + } - { + str(p).replace("\\", "/").lstrip("./") + for p in baseline + } + scoped = { + str(p).replace("\\", "/").lstrip("./") + for p in (list(contract.allowed_edit_paths) + list(contract.expected_files)) + if str(p).strip() + } + offending = sorted( + path for path in changed + if not ((path in scoped) or any(path.startswith(prefix.rstrip("/") + "/") for prefix in scoped)) + ) + if offending: + runner._out_of_scope_mutation_detected = True + runner._out_of_scope_mutation_paths.update(offending) + runner.event_callback( + { + "type": "out_of_scope_shell_mutation_detected", + "offending_paths": offending[:12], + } + ) + return { + "is_error": True, + "content": ( + "Your shell command modified files outside the task's allowed edit scope. " + "Those changes are not valid progress. Modify only the expected/allowed files " + "unless the task explicitly asks for new files. " + f"Offending paths: {', '.join(offending[:12])}" + ), + } return _benchmark_post_write_python_validation(runner, tool_name, tool_input, result)