CI Channel Deploy

Signed-off-by: Mitch Gaffigan <mitch.gaffigan@comcast.net>

Author: mgaffigan

ci/runner/Dockerfile

@@ -1,9 +1,8 @@
 FROM docker:27-cli
 
-RUN apk add --no-cache bash curl python3
-RUN ln -sf /usr/bin/python3 /usr/bin/python
+RUN apk add --no-cache bash curl python3 py3-lxml
 
 WORKDIR /app
-COPY run.py /app/run.py
+COPY *.py /app/
 
 ENTRYPOINT ["python3", "/app/run.py"]

ci/runner/api.py

@@ -0,0 +1,89 @@
+import ssl
+import urllib.error
+import urllib.parse
+import urllib.request
+from http.cookiejar import CookieJar
+
+REQUESTED_WITH_HEADER = "OpenIntegrationEngine-CI"
+
+
+class ApiClient:
+    def __init__(self, base_url: str):
+        self.base_url = base_url.rstrip("/")
+        self.opener = build_opener()
+
+    def request(
+        self,
+        path: str,
+        method: str = "GET",
+        data: bytes | None = None,
+        content_type: str | None = None,
+        accept: str = "application/json",
+        timeout: int = 15,
+    ) -> tuple[int, str]:
+        headers = {
+            "Accept": accept,
+            "X-Requested-With": REQUESTED_WITH_HEADER,
+        }
+        if content_type is not None:
+            headers["Content-Type"] = content_type
+
+        request = urllib.request.Request(
+            f"{self.base_url}{path}",
+            data=data,
+            method=method,
+            headers=headers,
+        )
+
+        try:
+            with self.opener.open(request, timeout=timeout) as response:
+                body = response.read().decode("utf-8", errors="replace")
+                return response.status, body
+        except urllib.error.HTTPError as error:
+            body = error.read().decode("utf-8", errors="replace")
+            raise RuntimeError(f"HTTP {error.code} for {method} {path}: {body}") from error
+        except urllib.error.URLError as error:
+            raise RuntimeError(f"Request failed for {method} {path}: {error}") from error
+
+    def create_channel(self, channel_xml: bytes) -> None:
+        self.request(
+            "/api/channels/",
+            method="POST",
+            data=channel_xml,
+            content_type="application/xml",
+            accept="application/json",
+        )
+
+    def deploy_channel(self, channel_id: str) -> None:
+        self.request(f"/api/channels/{channel_id}/_deploy", method="POST")
+
+    def undeploy_channel(self, channel_id: str) -> None:
+        self.request(f"/api/channels/{channel_id}/_undeploy", method="POST")
+
+    def remove_channel(self, channel_id: str) -> None:
+        self.request(f"/api/channels/{channel_id}", method="DELETE")
+
+
+def build_opener() -> urllib.request.OpenerDirector:
+    cookie_jar = CookieJar()
+    ssl_context = ssl.create_default_context()
+    ssl_context.check_hostname = False
+    ssl_context.verify_mode = ssl.CERT_NONE
+    https_handler = urllib.request.HTTPSHandler(context=ssl_context)
+    return urllib.request.build_opener(https_handler, urllib.request.HTTPCookieProcessor(cookie_jar))
+
+
+def login_or_fail(base_url: str, username: str, password: str, timeout: int) -> ApiClient:
+    client = ApiClient(base_url)
+    payload = urllib.parse.urlencode({"username": username, "password": password}).encode("utf-8")
+    status, body = client.request(
+        "/api/users/_login",
+        method="POST",
+        data=payload,
+        content_type="application/x-www-form-urlencoded",
+        timeout=min(timeout, 15),
+    )
+    if status == 200 and "SUCCESS" in body:
+        print("Authenticated successfully.", flush=True)
+        return client
+    raise RuntimeError(f"Unexpected login response: HTTP {status} body={body}")

ci/runner/channeltests.py

@@ -0,0 +1,107 @@
+from dataclasses import dataclass
+import importlib
+from pathlib import Path
+import sys
+
+from api import ApiClient
+
+
+@dataclass(frozen=True)
+class ChannelFixture:
+    test_dir: Path
+    channel_dir: Path
+    channel_file: Path
+    channel_id: str
+    channel_name: str
+
+
+@dataclass(frozen=True)
+class ProvisionedChannel:
+    fixture: ChannelFixture
+    channel_id: str
+
+
+def resolve_tests_root(workspace: str, tests_root: str) -> Path:
+    candidate = Path(tests_root)
+    if candidate.is_absolute():
+        return candidate
+    return Path(workspace) / candidate
+
+
+def parse_channel_fixture(channel_file: Path) -> ChannelFixture:
+    channel_id = parse_channel_id(channel_file)
+    return ChannelFixture(
+        test_dir=channel_file.parents[2],
+        channel_dir=channel_file.parent,
+        channel_file=channel_file,
+        channel_id=channel_id,
+        channel_name=channel_file.parent.name,
+    )
+
+
+def parse_channel_id(channel_file: Path) -> str:
+    etree = importlib.import_module("lxml.etree")
+    document = etree.parse(str(channel_file))
+    channel_id = document.getroot().findtext("id")
+    if channel_id is None or not channel_id.strip():
+        raise RuntimeError(f"Channel fixture is missing id: {channel_file}")
+    return channel_id.strip()
+
+
+def test_runs_for_configuration(test_dir: Path, configuration: str) -> bool:
+    configurations_file = test_dir / "configurations"
+    if not configurations_file.exists():
+        return True
+
+    configurations = [line.strip() for line in configurations_file.read_text(encoding="utf-8").splitlines() if line.strip()]
+    return configuration in configurations
+
+
+def discover_channels(tests_root: Path, configuration: str) -> list[ChannelFixture]:
+    if not tests_root.exists():
+        return []
+
+    fixtures: list[ChannelFixture] = []
+    for channel_file in sorted(tests_root.glob("**/channels/*/channel.xml")):
+        fixture = parse_channel_fixture(channel_file)
+        if test_runs_for_configuration(fixture.test_dir, configuration):
+            fixtures.append(fixture)
+    return fixtures
+
+
+def deploy_channel_tests(client: ApiClient, channel_fixtures: list[ChannelFixture]) -> list[ProvisionedChannel]:
+    provisioned_channels: list[ProvisionedChannel] = []
+
+    for fixture in channel_fixtures:
+        print(f"Creating channel {fixture.channel_name} from {fixture.channel_file}", flush=True)
+        client.create_channel(fixture.channel_file.read_bytes())
+        print(f"Deploying channel {fixture.channel_name} ({fixture.channel_id})", flush=True)
+        client.deploy_channel(fixture.channel_id)
+        provisioned_channels.append(ProvisionedChannel(fixture=fixture, channel_id=fixture.channel_id))
+
+    return provisioned_channels
+
+
+def cleanup_channel_tests(client: ApiClient, provisioned_channels: list[ProvisionedChannel]) -> None:
+    for provisioned in reversed(provisioned_channels):
+        safe_cleanup_channel("undeploy", undeploy_channel, client, provisioned)
+
+    for provisioned in reversed(provisioned_channels):
+        safe_cleanup_channel("remove", remove_channel, client, provisioned)
+
+
+def undeploy_channel(client: ApiClient, provisioned: ProvisionedChannel) -> None:
+    print(f"Undeploying channel {provisioned.fixture.channel_name} ({provisioned.channel_id})", flush=True)
+    client.undeploy_channel(provisioned.channel_id)
+
+
+def remove_channel(client: ApiClient, provisioned: ProvisionedChannel) -> None:
+    print(f"Removing channel {provisioned.fixture.channel_name} ({provisioned.channel_id})", flush=True)
+    client.remove_channel(provisioned.channel_id)
+
+
+def safe_cleanup_channel(action: str, cleanup, client: ApiClient, provisioned: ProvisionedChannel) -> None:
+    try:
+        cleanup(client, provisioned)
+    except Exception as error:
+        print(f"Ignoring {action} failure for {provisioned.fixture.channel_name}: {error}", file=sys.stderr, flush=True)

ci/runner/compose.py

@@ -0,0 +1,50 @@
+import subprocess
+import uuid
+from pathlib import Path
+
+
+def sanitize_project_name(name: str) -> str:
+    filtered = "".join(character if character.isalnum() else "-" for character in name.lower())
+    filtered = filtered.strip("-") or "oie-ci"
+    return f"oie-ci-{filtered}-{uuid.uuid4().hex[:8]}"
+
+
+def run_command(command: list[str], env: dict[str, str]) -> subprocess.CompletedProcess:
+    print(f"+ {' '.join(command)}", flush=True)
+    return subprocess.run(command, env=env, check=False)
+
+
+def compose_up(compose_file: Path, project_name: str, env: dict[str, str], timeout: int) -> None:
+    command = [
+        "docker",
+        "compose",
+        "-f",
+        str(compose_file),
+        "-p",
+        project_name,
+        "up",
+        "-d",
+        "--wait",
+        "--wait-timeout",
+        str(timeout),
+    ]
+    result = run_command(command, env)
+    if result.returncode != 0:
+        raise RuntimeError("docker compose up failed")
+
+
+def compose_down(compose_file: Path, project_name: str, env: dict[str, str]) -> None:
+    command = [
+        "docker",
+        "compose",
+        "-f",
+        str(compose_file),
+        "-p",
+        project_name,
+        "down",
+        "-v",
+        "--remove-orphans",
+    ]
+    result = run_command(command, env)
+    if result.returncode != 0:
+        raise RuntimeError("docker compose down failed")

ci/runner/main.py

@@ -0,0 +1,128 @@
+import argparse
+import json
+import os
+import sys
+from pathlib import Path
+
+from api import login_or_fail
+from channeltests import (
+    ChannelFixture,
+    cleanup_channel_tests,
+    deploy_channel_tests,
+    discover_channels,
+    resolve_tests_root,
+)
+from compose import compose_down, compose_up, sanitize_project_name
+
+DEFAULT_TIMEOUT_SECONDS = 600
+DEFAULT_BASE_URL = "https://host.docker.internal:8443"
+DEFAULT_USERNAME = "admin"
+DEFAULT_PASSWORD = "admin"
+DEFAULT_TESTS_ROOT = "ci/tests"
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Boot and tear down an OIE docker-compose test configuration.")
+    parser.add_argument("--workspace", default="/workspace", help="Workspace root containing ci/configurations.")
+    parser.add_argument("--configuration", help="Configuration name mapped to ci/configurations/<name>.compose.yml.")
+    parser.add_argument("--compose-file", help="Explicit compose file path. Overrides --configuration.")
+    parser.add_argument("--server-image", required=True, help="Server image tag to inject into the compose environment.")
+    parser.add_argument("--base-url", default=DEFAULT_BASE_URL, help="Base URL used for readiness and login checks.")
+    parser.add_argument("--username", default=DEFAULT_USERNAME, help="REST username.")
+    parser.add_argument("--password", default=DEFAULT_PASSWORD, help="REST password.")
+    parser.add_argument("--tests-root", default=DEFAULT_TESTS_ROOT, help="Root directory containing test fixtures.")
+    parser.add_argument("--timeout", type=int, default=DEFAULT_TIMEOUT_SECONDS, help="Timeout in seconds for compose wait and login readiness.")
+    parser.add_argument("--keep-alive", action="store_true", help="Leave the compose stack running after a successful check.")
+    return parser.parse_args()
+
+
+def resolve_compose_file(args: argparse.Namespace) -> Path:
+    if args.compose_file:
+        compose_file = Path(args.compose_file)
+        if not compose_file.is_absolute():
+            compose_file = Path(args.workspace) / compose_file
+        return compose_file
+
+    if not args.configuration:
+        raise ValueError("Either --configuration or --compose-file must be provided.")
+
+    return Path(args.workspace) / "ci" / "configurations" / f"{args.configuration}.compose.yml"
+
+
+def build_run_summary(
+    compose_file: Path,
+    config_name: str,
+    project_name: str,
+    server_image: str,
+    tests_root: Path,
+    channel_fixtures: list[ChannelFixture],
+    base_url: str,
+    keep_alive: bool,
+) -> dict[str, object]:
+    return {
+        "compose_file": str(compose_file),
+        "configuration": config_name,
+        "project_name": project_name,
+        "server_image": server_image,
+        "tests_root": str(tests_root),
+        "channels": [str(fixture.channel_file.relative_to(tests_root)) for fixture in channel_fixtures],
+        "base_url": base_url,
+        "keep_alive": keep_alive,
+    }
+
+
+def main() -> int:
+    args = parse_args()
+    compose_file = resolve_compose_file(args)
+    tests_root = resolve_tests_root(args.workspace, args.tests_root)
+    if not compose_file.exists():
+        raise FileNotFoundError(f"Compose file not found: {compose_file}")
+
+    config_name = args.configuration or compose_file.stem.replace(".compose", "")
+    channel_fixtures = discover_channels(tests_root, config_name)
+    project_name = sanitize_project_name(config_name)
+    env = os.environ.copy()
+    env["OIE_IMAGE"] = args.server_image
+
+    print(
+        json.dumps(
+            build_run_summary(
+                compose_file,
+                config_name,
+                project_name,
+                args.server_image,
+                tests_root,
+                channel_fixtures,
+                args.base_url,
+                args.keep_alive,
+            ),
+            indent=2,
+        ),
+        flush=True,
+    )
+
+    compose_attempted = False
+    client = None
+    provisioned_channels = []
+    teardown_error = None
+    try:
+        compose_attempted = True
+        compose_up(compose_file, project_name, env, args.timeout)
+        client = login_or_fail(args.base_url, args.username, args.password, args.timeout)
+        provisioned_channels = deploy_channel_tests(client, channel_fixtures)
+
+        print(f"Configuration boot completed. Deployed {len(provisioned_channels)} channel(s).", flush=True)
+        return 0
+    finally:
+        if client is not None:
+            cleanup_channel_tests(client, provisioned_channels)
+
+        if compose_attempted and not args.keep_alive:
+            try:
+                compose_down(compose_file, project_name, env)
+            except Exception as error:
+                print(f"Teardown failed: {error}", file=sys.stderr, flush=True)
+                teardown_error = error
+
+        if teardown_error is not None:
+            raise RuntimeError("Compose teardown failed") from teardown_error