Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions src/skillspector/nodes/report.py
Original file line number Diff line number Diff line change
Expand Up @@ -924,7 +924,16 @@ def report(state: SkillspectorState) -> dict[str, object]:
inference_usage = state.get("inference_usage") or []

_attempted, _succeeded, degraded = _llm_runtime_status(use_llm, llm_call_log)
provider_available, provider_error = is_llm_available()
has_recorded_failure = any(not r.get("ok") for r in llm_call_log)
provider_unavailable = bool(use_llm and not provider_available and has_recorded_failure)
degraded = degraded or provider_unavailable
degraded_notice = _llm_degradation_notice(use_llm, llm_call_log)
if provider_unavailable and degraded_notice is None:
degraded_notice = (
"LLM analysis was requested but the configured provider was unavailable"
f" ({provider_error or 'unknown reason'}); results may reflect static analysis only."
)
if degraded:
logger.warning(
"LLM stage degraded: %d/%d LLM call(s) failed; report reflects static analysis only",
Expand Down Expand Up @@ -957,6 +966,10 @@ def report(state: SkillspectorState) -> dict[str, object]:
entirely_uninspected = (
entirely_uninspected_value if isinstance(entirely_uninspected_value, int) else 0
)

# Fail closed when a deep scan is degraded, including an unavailable
# provider with mixed call telemetry, or when inspection completeness says
# some content was not inspected. Leave the score and severity untouched.
if (degraded or fatal_exception or entirely_uninspected > 0) and risk_recommendation == "SAFE":
risk_recommendation = "CAUTION"

Expand Down
29 changes: 29 additions & 0 deletions tests/nodes/test_report.py
Original file line number Diff line number Diff line change
Expand Up @@ -987,6 +987,35 @@ def test_degraded_scan_floors_recommendation_at_caution() -> None:
assert result["risk_recommendation"] == "CAUTION" # but never SAFE when degraded


def test_unavailable_provider_floors_recommendation_even_with_success_records(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Provider truth wins when swallowed batch failures produced false success records."""
monkeypatch.setattr(
"skillspector.nodes.report.is_llm_available",
lambda: (False, "codex binary not found"),
)
state: SkillspectorState = {
"filtered_findings": [],
"component_metadata": [],
"has_executable_scripts": False,
"manifest": {},
"output_format": "json",
"use_llm": True,
"llm_call_log": [
llm_call_record("semantic_developer_intent", ok=True),
llm_call_record("semantic_quality_policy", ok=True),
llm_call_record("semantic_security_discovery", ok=False, error="binary missing"),
],
}

result = report(state)
assert result["risk_recommendation"] == "CAUTION"
payload = json.loads(result["report_body"])
assert payload["risk_assessment"]["recommendation"] == "CAUTION"
assert payload["metadata"]["llm_available"] is False


def test_non_degraded_clean_scan_stays_safe() -> None:
"""Without degradation, a clean scan still reports SAFE (no over-flooring)."""
state: SkillspectorState = {
Expand Down
Loading