Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
71 changes: 64 additions & 7 deletions scripts/cloud_run_runtime_guard.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ def _load_services() -> list[str]:
for target in targets:
if not isinstance(target, dict):
continue
if not _target_enabled(target):
continue
runtime_target = target.get("runtime_target") or target.get(
"runtime_target_json"
)
Expand Down Expand Up @@ -85,19 +87,43 @@ def _load_services() -> list[str]:
return unique


def _service_job_aliases(service: str) -> list[str]:
service_name = str(service or "").strip()
if not service_name:
return []
aliases = [service_name]
if service_name.endswith("-service"):
aliases.append(service_name.removesuffix("-service"))
return list(dict.fromkeys(aliases))


def _scheduler_job_pattern_for_services(services: list[str]) -> str:
candidates: list[str] = []
for service in services:
service_name = str(service or "").strip()
if not service_name:
continue
candidates.append(service_name)
if service_name.endswith("-service"):
candidates.append(service_name.removesuffix("-service"))
candidates.extend(_service_job_aliases(service))
unique = list(dict.fromkeys(candidates))
return "|".join(re.escape(candidate) for candidate in unique)


def _entry_job_name(entry: dict[str, Any]) -> str:
labels = _labels(entry)
return str(labels.get("job_id") or labels.get("job_name") or "")


def _scheduler_entry_since(
entry: dict[str, Any],
service_since_by_name: dict[str, dt.datetime],
fallback: dt.datetime,
) -> dt.datetime:
job_name = _entry_job_name(entry)
matches = [
service_since
for service, service_since in service_since_by_name.items()
if any(alias and alias in job_name for alias in _service_job_aliases(service))

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Match scheduler jobs to only their owning service

In multi-service runtime guards where one service alias is a substring of another service's scheduler job, this substring match can associate a job with multiple services and then max(matches) uses the newest unrelated revision window. For example, charles-schwab-live-u7654-mega-service-scheduler also matches the alias charles-schwab derived from charles-schwab-service; if that shorter service was deployed later, legitimate scheduler failures for the live service before that timestamp are skipped as stale. The scheduler entry should resolve to the best/exact owning service rather than every substring alias.

Useful? React with 👍 / 👎.

]
return max(matches) if matches else fallback


def _run_gcloud(args: list[str]) -> subprocess.CompletedProcess[str]:
return subprocess.run(args, text=True, capture_output=True, check=False)

Expand Down Expand Up @@ -184,6 +210,27 @@ def _runtime_target(target: dict[str, Any]) -> dict[str, Any]:
return runtime_target if isinstance(runtime_target, dict) else {}


def _coerce_bool(value: Any, default: bool) -> bool:
if value is None:
return default
if isinstance(value, bool):
return value
text = str(value).strip().lower()
if not text:
return default
return text in {"1", "true", "yes", "y", "on"}


def _target_enabled(target: dict[str, Any]) -> bool:
runtime_target = _runtime_target(target)
for key in ("runtime_target_enabled", "RUNTIME_TARGET_ENABLED"):
if key in target:
return _coerce_bool(target.get(key), True)
if key in runtime_target:
return _coerce_bool(runtime_target.get(key), True)
Comment on lines +226 to +230

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Honor disabled flags under target env

When CLOUD_RUN_SERVICE_TARGETS_JSON uses the same per-target shape accepted by scripts/build_cloud_run_env_sync_plan.py ({"env": {"RUNTIME_TARGET_ENABLED": "false"}}), _target_enabled() still returns true because it checks only the top-level target and runtime_target object. _load_services() then includes that disabled service via the target's service_name, so the guard continues querying and alerting on services the sync plan marks disabled; include the target env mapping when evaluating the flag.

Useful? React with 👍 / 👎.

return True


def _target_service_names(target: dict[str, Any]) -> list[str]:
runtime_target = _runtime_target(target)
for key in ("service", "service_name", "cloud_run_service"):
Expand Down Expand Up @@ -427,6 +474,7 @@ def main() -> int:
issues: list[str] = []
details: list[str] = []
success_count = 0
service_since_by_name: dict[str, dt.datetime] = {}

try:
services = _load_services()
Expand All @@ -440,6 +488,7 @@ def main() -> int:

for service in services:
service_since = _cloud_run_log_since(project, service, since) if ignore_pre_ready_logs else since
service_since_by_name[service] = service_since
service_since_text = _format_timestamp(service_since)
log_filter = (
'resource.type="cloud_run_revision" '
Expand Down Expand Up @@ -473,7 +522,15 @@ def main() -> int:
for entry in entries
if regex.search(str(_labels(entry).get("job_id") or _labels(entry).get("job_name") or ""))
]
failures = [entry for entry in entries if _is_failure(entry)]
failures = []
for entry in entries:
if not _is_failure(entry):
continue
entry_timestamp = _parse_timestamp(entry.get("timestamp"))
entry_since = _scheduler_entry_since(entry, service_since_by_name, since)
if entry_timestamp and entry_timestamp < entry_since:
continue
failures.append(entry)
if failures:
issues.append(f"{len(failures)} Cloud Scheduler failure log(s)")
details.extend(_summarize(entry) for entry in failures[:5])
Expand Down
35 changes: 35 additions & 0 deletions tests/test_cloud_run_runtime_guard.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,3 +90,38 @@ def test_region_for_service_prefers_target_region(monkeypatch):
)

assert guard._region_for_service("charles-schwab-service") == "asia-east1"


def test_load_services_ignores_disabled_runtime_targets(monkeypatch):
monkeypatch.delenv("RUNTIME_GUARD_CLOUD_RUN_SERVICES", raising=False)
monkeypatch.delenv("CLOUD_RUN_SERVICES", raising=False)
monkeypatch.delenv("CLOUD_RUN_SERVICE", raising=False)
monkeypatch.setenv(
"CLOUD_RUN_SERVICE_TARGETS_JSON",
json.dumps(
{
"targets": [
{"service": "enabled-service", "RUNTIME_TARGET_ENABLED": "true"},
{"service": "disabled-service", "RUNTIME_TARGET_ENABLED": "false"},
{"service": "disabled-lower-service", "runtime_target_enabled": "false"},
]
}
),
)

assert guard._load_services() == ["enabled-service"]


def test_scheduler_entry_since_uses_matching_service_revision_window():
fallback = dt.datetime(2026, 7, 1, 1, 0, tzinfo=dt.timezone.utc)
service_since = dt.datetime(2026, 7, 1, 2, 0, tzinfo=dt.timezone.utc)
entry = {"resource": {"labels": {"job_id": "enabled-service-scheduler"}}}

assert (
guard._scheduler_entry_since(entry, {"enabled-service": service_since}, fallback)
== service_since
)
assert (
guard._scheduler_entry_since(entry, {"other-service": service_since}, fallback)
== fallback
)
Loading