Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .claude-plugin/marketplace.json
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@
"name": "microshift-release",
"source": "./plugins/microshift-release",
"description": "Set of tools to perform MicroShift Release Testing activities",
"version": "2.0.0"
"version": "2.0.1"
},
{
"name": "next",
Expand Down
2 changes: 1 addition & 1 deletion plugins/microshift-release/.claude-plugin/plugin.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"name": "microshift-release",
"description": "Set of tools to perform MicroShift Release Testing activities",
"version": "2.0.0",
"version": "2.0.1",
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
"author": {
"name": "agullon"
},
Expand Down
4 changes: 4 additions & 0 deletions plugins/microshift-release/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -225,6 +225,10 @@ bash unit_tests/test_request_release-to-art.sh
## Requirements

- VPN connection (for Brew RPM checks, advisory reports)
- Red Hat CA in the system trust store, for the GitLab checks (shipment MR,
advisory YAML). Red Hat-managed machines have it; a container or CI image
must add it. GitLab requests carry `GITLAB_API_TOKEN`, so TLS is verified
and there is no `verify=False` fallback.
- Kerberos login: `kinit`
- GitLab API token: `GITLAB_API_TOKEN`
- Jira API token: `JIRA_API_TOKEN` and `JIRA_USERNAME`
Expand Down
113 changes: 95 additions & 18 deletions plugins/microshift-release/scripts/lib/artifacts.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,20 @@
import os
import re
import subprocess
from urllib.parse import quote, unquote, urlparse

import requests
import truststore
import urllib3

logger = logging.getLogger(__name__)

# gitlab.cee.redhat.com chains to a Red Hat internal root that certifi (the
# requests default bundle) doesn't ship. Use the OS trust store instead, where
# RH-managed machines already have it, so token-bearing requests keep TLS
# verification on without per-user REQUESTS_CA_BUNDLE setup.
truststore.inject_into_ssl()
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a process-global change: every TLS connection the process makes after importing lib.artifacts switches from certifi to the OS trust store. On RH-managed machines that's fine (and it's a genuine improvement over verify=False), but on machines whose OS store lacks the RH internal root — containers, non-RH hosts — every GitLab request would now fail verification where verify=False previously worked.

I verified this in a clean container: with truststore.inject_into_ssl(), requests to gitlab.cee.redhat.com fail with SSLCertVerificationError, while the old verify=False path returns 401 (i.e. reachable). On the callers' side this surfaces through the "No access … check GITLAB_API_TOKEN permissions" message, which would be misleading for a TLS issue.

Would a session-scoped adapter (or per-request ssl_context) scoped to gitlab.cee.redhat.com get the same benefit without the global side effect?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for testing it in a container. I'd keep the global injection: these scripts are meant to run only on Red Hat-managed machines, or on a CI image with the Red Hat CA in the system trust store. There, the OS store is a superset of certifi, so the other requests (Pyxis, lifecycle, Jira) are unaffected. The point of the change is to stop sending GITLAB_API_TOKEN over unverified TLS, and I'd rather fail than fall back to verify=False.

You're right that a missing CA looked like a token problem, though. Fixed in eda5987: TLS failures are now logged as errors and reported as "TLS verification failed for gitlab.cee.redhat.com — the Red Hat CA must be in the system trust store" instead of the token-permissions hint, and the requirement is documented in the README.


urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

# NVR format regex patterns per release type
Expand Down Expand Up @@ -52,12 +60,30 @@
# GitLab internal instance for ocp-shipment-data
_GITLAB_BASE = "https://gitlab.cee.redhat.com"
_SHIPMENT_PROJECT_PATH = "hybrid-platforms/art/ocp-shipment-data"
_GITLAB_TLS_HINT = ("TLS verification failed for gitlab.cee.redhat.com — the "
"Red Hat CA must be in the system trust store "
"(Red Hat-managed machine or CI image with the CA)")
# Set when a GitLab request fails TLS verification, so callers that only see
# a None response can report the TLS cause instead of a token problem.
_gitlab_tls_failed = False


def _http_get(url, timeout=15, verify=True, headers=None, proxies=None):
def _http_get(url, timeout=15, verify=True, headers=None, proxies=None,
allow_redirects=True):
"""Perform an HTTP GET, returning the response or raising RequestException."""
return requests.get(url, timeout=timeout, verify=verify, headers=headers,
allow_redirects=True, proxies=proxies)
allow_redirects=allow_redirects, proxies=proxies)


def _redact_url(url):
"""Return only a URL's path, dropping host, credentials, and query so
internal hostnames never reach the logs."""
if not url:
return url
try:
return urlparse(url).path or "<redacted-url>"
except ValueError:
return "<invalid-url>"


def validate_nvr_format(nvr, release_type):
Expand Down Expand Up @@ -379,15 +405,60 @@ def _gitlab_get(path):
Returns:
requests.Response or None on failure.
"""
global _gitlab_tls_failed # noqa: PLW0603
headers = _gitlab_headers() or {}
url = f"{_GITLAB_BASE}/api/v4/{path.lstrip('/')}"
try:
return _http_get(url, headers=headers, verify=False, timeout=15)
# Trusted origin (_GITLAB_BASE); verify TLS and never follow a redirect
# that could forward PRIVATE-TOKEN to another host.
return _http_get(url, headers=headers, timeout=15,
allow_redirects=False)
except requests.exceptions.SSLError as exc:
_gitlab_tls_failed = True
logger.error("%s: %s", _GITLAB_TLS_HINT, exc)
return None
except requests.RequestException as exc:
logger.debug("GitLab API request failed for %s: %s", path, exc)
# Warn, not debug: callers report a None response as "no access",
# which hides TLS/network failures behind a misleading token hint.
logger.warning("GitLab API request failed for %s: %s", path, exc)
Comment thread
agullon marked this conversation as resolved.
Resolved
return None


def _gitlab_raw_to_api_path(web_url):
"""Convert a GitLab web raw URL to its REST API raw-file path.

GitLab's web ``/-/raw/<ref>/<file>`` route authenticates via the browser
session cookie and ignores the PRIVATE-TOKEN header, so a PAT-based fetch
must go through the API ``projects/:id/repository/files/:path/raw?ref=``
route instead (the same route _gitlab_get uses successfully elsewhere).

Returns the API path for _gitlab_get, or None if web_url is not a
recognizable GitLab raw URL.
"""
if not web_url:
return None
try:
parsed = urlparse(web_url)
except ValueError:
return None
base = urlparse(_GITLAB_BASE)
if (parsed.scheme, parsed.netloc) != (base.scheme, base.netloc):
return None
marker = "/-/raw/"
if marker not in parsed.path:
return None
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
project_path, remainder = parsed.path.split(marker, 1)
ref, _, file_path = remainder.partition("/")
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
project_path = unquote(project_path.strip("/"))
ref, file_path = unquote(ref), unquote(file_path)
if not (project_path and ref and file_path):
return None
enc_project = quote(project_path, safe="")
enc_file = quote(file_path, safe="")
return (f"projects/{enc_project}/repository/files/{enc_file}"
f"/raw?ref={quote(ref, safe='')}")


def _get_gitlab_project_id():
"""Look up the numeric ID for the ocp-shipment-data GitLab project.

Expand Down Expand Up @@ -420,11 +491,9 @@ def fetch_shipment_mr(version):
"""
project_id = _get_gitlab_project_id()
if project_id is None:
return {
"found": False,
"reason": f"No access to {_SHIPMENT_PROJECT_PATH} — check GITLAB_API_TOKEN permissions",
"skipped": True,
}
reason = (_GITLAB_TLS_HINT if _gitlab_tls_failed else
f"No access to {_SHIPMENT_PROJECT_PATH} — check GITLAB_API_TOKEN permissions")
return {"found": False, "reason": reason, "skipped": True}

# Search all MRs (open + merged) filtering by version in title
search_version = version.replace("-", " ")
Expand Down Expand Up @@ -695,20 +764,28 @@ def fetch_advisory_details(advisory_url):
"containerImage", "sha", "repo"}]} or None on failure.
"""
import yaml as _yaml # noqa: PLC0415
shown_url = _redact_url(advisory_url)
# The web /-/raw/ route ignores PRIVATE-TOKEN and bounces private projects
# to the HTML sign-in page, so always fetch via the authenticated API route.
api_path = _gitlab_raw_to_api_path(advisory_url)
if not api_path:
logger.warning("Advisory URL is not a GitLab raw URL: %s", shown_url)
return None
resp = _gitlab_get(api_path)
if resp is None:
return None # _gitlab_get already logged the request failure
if resp.status_code != 200:
logger.warning("Advisory YAML fetch returned HTTP %d: %s",
resp.status_code, shown_url)
return None
try:
resp = _http_get(advisory_url, verify=False, timeout=15)
if resp.status_code != 200:
logger.warning("Advisory YAML fetch returned HTTP %d: %s",
resp.status_code, advisory_url)
return None
content = _yaml.safe_load(resp.text)
except (requests.RequestException, _yaml.YAMLError) as exc:
logger.warning("Advisory YAML fetch/parse failed for %s: %s",
advisory_url, exc)
except _yaml.YAMLError as exc:
logger.warning("Advisory YAML parse failed for %s: %s", shown_url, exc)
return None

if not isinstance(content, dict):
logger.warning("Advisory YAML is not a mapping: %s", advisory_url)
logger.warning("Advisory YAML is not a mapping: %s", shown_url)
return None
spec = content.get("spec", {})
raw_images = spec.get("content", {}).get("images", [])
Expand Down
70 changes: 52 additions & 18 deletions plugins/microshift-release/scripts/post_release.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import json
import logging
import re
import subprocess
import sys
from collections import Counter
from concurrent.futures import ThreadPoolExecutor, as_completed
Expand Down Expand Up @@ -93,11 +94,21 @@

_DOCS_RELEASE_NOTES_URL = (
"https://docs.redhat.com/en/documentation/"
"red_hat_build_of_microshift/{minor}/"
"red_hat_build_of_microshift/{doc_version}/"
"html-single/red_hat_build_of_microshift_release_notes/"
)


def _docs_version(minor):
"""Return the docs.redhat.com version segment for an X.Y stream.

The first minor of a major is published under the bare major ("5",
not "5.0", which returns 404/403); later minors use X.Y ("5.1").
"""
major, _, y = minor.partition(".")
return major if y == "0" else minor


def _rhel_versions(version_info):
"""Return RHEL versions to check based on the MicroShift version."""
minor = _minor_tuple(version_info["minor"])
Expand Down Expand Up @@ -458,17 +469,20 @@ def check_rpms_cdn(advisory, vpn_ok):
return _fail(check_id, "No CDN push found",
[f"Advisory: {advisory_name}"])

all_complete = all(
(p.get("status") or "").upper() == "COMPLETE" for p in cdn_pushes
)
if all_complete:
return _pass(check_id,
f"CDN push complete ({len(cdn_pushes)} job(s))",
[f"Advisory: {advisory_name}"])
statuses = [p.get("status", "?") for p in cdn_pushes]
return _fail(check_id,
f"CDN push not complete: {', '.join(statuses)}",
[f"Advisory: {advisory_name}"])
# Failed pushes are retried as new jobs, so only the most recent job
# (highest ID) reflects the current CDN state.
latest = max(cdn_pushes, key=lambda p: p.get("id") or 0)
status = (latest.get("status") or "?").upper()
if status == "COMPLETE":
details = [f"Advisory: {advisory_name}"]
retries = len(cdn_pushes) - 1
if retries:
details.append(f"Latest of {len(cdn_pushes)} push jobs "
f"({retries} earlier attempt(s))")
return _pass(check_id, "CDN push complete", details)
return _fail(check_id, f"Latest CDN push not complete: {status}",
[f"Advisory: {advisory_name}",
f"Push job: {latest.get('id', '?')}"])


_ERRATA_RPM_RE = re.compile(
Expand Down Expand Up @@ -523,27 +537,47 @@ def check_rpms_downloads(errata_info, version_info):
[f"URL: {portal_url}"])


def _curl_get(url, timeout=15):
Comment thread
agullon marked this conversation as resolved.
Resolved
"""GET url with the curl CLI and return (http_status, body).

docs.redhat.com's bot filter answers 403 to python-requests (and any
other non-curl User-Agent) but serves curl, so fetch it with real curl.
Raises OSError if curl is missing, RuntimeError if curl itself fails.
"""
marker = "\n__HTTP_STATUS__:"
proc = subprocess.run(
["curl", "-sSL", "--max-time", str(timeout),
"-w", marker + "%{http_code}", url],
capture_output=True, text=True, timeout=timeout + 5, check=False,
)
if proc.returncode != 0:
raise RuntimeError(proc.stderr.strip() or f"curl exit {proc.returncode}")
body, _, status = proc.stdout.rpartition(marker)
return int(status), body


def check_docs_published(version_info):
"""Release notes published on docs.redhat.com with version mentioned."""
check_id = "pr_docs_published"
minor = version_info["minor"]
version = version_info["version"]
url = _DOCS_RELEASE_NOTES_URL.format(minor=minor)
url = _DOCS_RELEASE_NOTES_URL.format(doc_version=_docs_version(minor))

try:
resp = requests.get(url, timeout=15, allow_redirects=True)
if resp.status_code != 200:
status, body = _curl_get(url)
if status != 200:
return _fail(check_id,
f"Release notes page returned HTTP {resp.status_code}",
f"Release notes page returned HTTP {status}",
[f"URL: {url}"])
if version in resp.text:
if version in body:
return _pass(check_id, f"Version {version} found in release notes",
[f"URL: {url}"])
return _fail(check_id,
f"Release notes page exists but {version} not mentioned",
[f"URL: {url}",
"The z-stream release notes may not be published yet"])
except requests.RequestException as e:
except (OSError, RuntimeError, ValueError,
subprocess.TimeoutExpired) as e:
return _warn(check_id, f"Could not reach docs.redhat.com: {e}",
[f"URL: {url}"])

Expand Down
1 change: 1 addition & 0 deletions plugins/microshift-release/scripts/requirements.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
requests>=2.32.2,<3
requests-gssapi>=1.3.0,<2
pyyaml>=6.0,<7
truststore>=0.10.1,<1
Loading
Loading