diff --git a/.github/morph/opengauss-template.yaml b/.github/morph/opengauss-template.yaml new file mode 100644 index 0000000..d2d08c1 --- /dev/null +++ b/.github/morph/opengauss-template.yaml @@ -0,0 +1,92 @@ +name: Open Gauss Batteries Included Devbox +description: Installer-driven Open Gauss template that uses the current checkout when run locally and otherwise clones math-inc/OpenGauss, runs the internal installer, exposes the local guide, and opens a ready Gauss session. +steps: + - id: bootstrap-installer-runtime + title: Prepare Open Gauss Installer Runtime + type: command + run: | + set -euo pipefail + if command -v apt-get >/dev/null 2>&1 && [ "$(id -u)" = "0" ]; then + export DEBIAN_FRONTEND=noninteractive + apt-get update -y + apt-get install -y --no-install-recommends git curl ca-certificates tmux + else + command -v git >/dev/null 2>&1 || { printf '%s\n' 'git is required.' >&2; exit 1; } + command -v curl >/dev/null 2>&1 || { printf '%s\n' 'curl is required.' >&2; exit 1; } + fi + mkdir -p "$HOME/.opengauss-template" + - id: resolve-open-gauss-repository + title: Prepare Open Gauss Repository + type: command + run: | + set -euo pipefail + state_dir="$HOME/.opengauss-template" + state_env="$state_dir/runtime.env" + mkdir -p "$state_dir" + + repo_root="" + if [ -x "./scripts/install-internal.sh" ] && [ -f "./pyproject.toml" ] && [ -f "./README.md" ]; then + repo_root="$(pwd)" + else + repo_root="$HOME/OpenGauss" + if [ -d "$repo_root/.git" ]; then + git -C "$repo_root" fetch origin --prune + else + rm -rf "$repo_root" + git clone https://github.com/math-inc/OpenGauss.git "$repo_root" + fi + git -C "$repo_root" fetch origin --prune + git -C "$repo_root" reset --hard origin/main + fi + + gauss_home="${GAUSS_HOME:-$HOME/.gauss}" + workspace_dir="${GAUSS_WORKSPACE_DIR:-$HOME/GaussWorkspace}" + + cat > "$state_env" </tmp/opengauss-guide.log 2>&1 & + sleep 1 + - id: expose-guide + title: Expose Guide Iframe + type: exposeHttpService + name: guide + port: 4310 + autoOpenIframe: true + - id: open-gauss-session + title: Open Gauss tmux Session + type: tmuxSession + name: gauss + command: | + set -euo pipefail + . "$HOME/.opengauss-template/runtime.env" + if [ -f "$GAUSS_HOME/.env" ]; then + set -a + . "$GAUSS_HOME/.env" + set +a + fi + exec "$HOME/.local/bin/gauss-launch-session" diff --git a/.github/workflows/refresh-shared-template.yml b/.github/workflows/refresh-shared-template.yml index 286757e..869759d 100644 --- a/.github/workflows/refresh-shared-template.yml +++ b/.github/workflows/refresh-shared-template.yml @@ -1,4 +1,4 @@ -name: Refresh Shared Template +name: Publish Shared Template on: push: @@ -10,88 +10,29 @@ concurrency: cancel-in-progress: false jobs: - refresh-opengauss-template: + publish-opengauss-template: runs-on: ubuntu-latest permissions: contents: read env: DEVBOX_TEMPLATE_BASE_URL: https://devbox.svc.cloud.morph.so TEMPLATE_ALIAS: opengauss + TEMPLATE_FILE: .github/morph/opengauss-template.yaml + PYTHONUNBUFFERED: "1" MORPH_API_KEY: ${{ secrets.MORPH_API_KEY }} steps: + - uses: actions/checkout@v4 + - name: Validate refresh credentials run: | if [ -z "${MORPH_API_KEY}" ]; then - echo "GitHub secret MORPH_API_KEY is required to refresh the shared template alias." >&2 + echo "GitHub secret MORPH_API_KEY is required to publish the shared template alias." >&2 + exit 1 + fi + if [ ! -f "${TEMPLATE_FILE}" ]; then + echo "Tracked Morph template file is missing: ${TEMPLATE_FILE}" >&2 exit 1 fi - - name: Refresh shared template alias - run: | - python3 - <<'PY' - import json - import os - import sys - import urllib.request - - base = os.environ["DEVBOX_TEMPLATE_BASE_URL"].rstrip("/") - key = os.environ["MORPH_API_KEY"] - alias = os.environ["TEMPLATE_ALIAS"] - - body = { - "description": f"Automatic refresh from {os.environ.get('GITHUB_REPOSITORY', 'repo')}@{os.environ.get('GITHUB_SHA', '')[:12]}" - } - - req = urllib.request.Request( - f"{base}/api/templates/aliases/{alias}/refresh", - data=json.dumps(body).encode("utf-8"), - headers={ - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - }, - method="POST", - ) - - with urllib.request.urlopen(req, timeout=120) as resp: - started = json.load(resp) - - print("refresh_started", json.dumps(started)) - - events_path = started.get("events_path") - if not events_path: - sys.exit("refresh response did not include events_path") - - events_req = urllib.request.Request( - f"{base}{events_path}", - headers={"Authorization": f"Bearer {key}"}, - method="GET", - ) - - completed = None - saw_force_rebuild = False - with urllib.request.urlopen(events_req, timeout=3600) as resp: - for raw in resp: - line = raw.decode("utf-8", errors="replace").strip() - if not line.startswith("data: "): - continue - event = json.loads(line[6:]) - print(json.dumps(event)) - if event.get("type") == "force_rebuild": - saw_force_rebuild = True - if event.get("type") in {"completed", "error", "cancelled"}: - completed = event - break - - if not completed: - sys.exit("refresh events stream ended without a terminal event") - if completed.get("type") != "completed": - sys.exit(f"refresh failed: {completed}") - if not saw_force_rebuild: - sys.exit("refresh completed without force_rebuild event") - - alias_publish = completed.get("aliasPublish") or {} - if not alias_publish.get("published"): - sys.exit(f"alias was not promoted: {alias_publish}") - - print("alias_promoted_to", alias_publish.get("templateId")) - PY + - name: Publish shared template alias + run: python3 scripts/publish_shared_template.py diff --git a/scripts/publish_shared_template.py b/scripts/publish_shared_template.py new file mode 100644 index 0000000..04435eb --- /dev/null +++ b/scripts/publish_shared_template.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import json +import os +import sys +import time +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path + +DEFAULT_TAGS = ["template", "gauss"] + + +class PublishError(RuntimeError): + """Raised when shared-template publishing fails.""" + + +class NotFoundError(PublishError): + """Raised when a Morph resource does not exist.""" + + +@dataclass(frozen=True) +class TemplateMetadata: + name: str + description: str + yaml_text: str + + +def strip_quotes(value: str) -> str: + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + return value[1:-1] + return value + + +def log(message: str) -> None: + print(message, flush=True) + + +def read_template_metadata(path: Path) -> TemplateMetadata: + yaml_text = path.read_text(encoding="utf-8") + name = None + description = None + for line in yaml_text.splitlines(): + if not line or line.startswith("#") or line.startswith(" "): + continue + if line.startswith("name:"): + name = strip_quotes(line.split(":", 1)[1]) + elif line.startswith("description:"): + description = strip_quotes(line.split(":", 1)[1]) + if name and description: + break + if not name or not description: + raise PublishError(f"Template file {path} must define top-level name and description fields.") + return TemplateMetadata(name=name, description=description, yaml_text=yaml_text) + + +class MorphClient: + def __init__(self, base_url: str, api_key: str, opener=urllib.request.urlopen): + self.base_url = base_url.rstrip("/") + self.api_key = api_key + self._opener = opener + + def _request(self, method: str, path: str, payload: dict | None = None, timeout: int = 120) -> dict: + url = path if path.startswith("http://") or path.startswith("https://") else f"{self.base_url}{path}" + data = None if payload is None else json.dumps(payload).encode("utf-8") + request = urllib.request.Request( + url, + data=data, + headers={ + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + }, + method=method, + ) + try: + with self._opener(request, timeout=timeout) as response: + body = response.read() + except urllib.error.HTTPError as exc: + raw = exc.read().decode("utf-8", errors="replace") if hasattr(exc, "read") else "" + if exc.code == 404: + raise NotFoundError(f"{method} {url} returned 404: {raw}") from exc + raise PublishError(f"{method} {url} failed with HTTP {exc.code}: {raw}") from exc + except urllib.error.URLError as exc: + raise PublishError(f"{method} {url} failed: {exc}") from exc + if not body: + return {} + try: + return json.loads(body) + except json.JSONDecodeError as exc: + raise PublishError(f"{method} {url} returned invalid JSON: {body!r}") from exc + + def fetch_alias(self, alias: str) -> dict | None: + try: + return self._request("GET", f"/api/aliases/{alias}") + except NotFoundError: + return None + + def create_template(self, *, metadata: TemplateMetadata, base_snapshot_id: str) -> dict: + return self._request( + "POST", + "/api/templates", + { + "name": metadata.name, + "description": metadata.description, + "yaml": metadata.yaml_text, + "baseSnapshotId": base_snapshot_id, + }, + ) + + def cache_template(self, template_id: str) -> dict: + return self._request("POST", f"/api/templates/{template_id}/cache", {}) + + def fetch_template(self, template_id: str) -> dict: + return self._request("GET", f"/api/templates/{template_id}") + + def share_template(self, *, template_id: str, alias: str, description: str, tags: list[str]) -> dict: + return self._request( + "POST", + f"/api/templates/{template_id}/share", + {"alias": alias, "description": description, "tags": tags}, + ) + + def delete_template(self, template_id: str) -> dict: + return self._request("DELETE", f"/api/templates/{template_id}") + + +def parse_tags(raw_tags: str | None) -> list[str]: + if not raw_tags: + return [] + return [tag.strip() for tag in raw_tags.split(",") if tag.strip()] + + +def wait_for_ready(client: MorphClient, template_id: str, *, timeout_seconds: int, poll_seconds: float) -> dict: + deadline = time.monotonic() + timeout_seconds + last_status = None + while True: + template = client.fetch_template(template_id) + current_status = template.get("status") + if current_status != last_status: + log(f"template_status {template_id} {current_status}") + last_status = current_status + if current_status == "ready": + return template + if current_status in {"failed", "cancelled", "error"}: + raise PublishError(f"Template {template_id} entered terminal status {current_status}: {json.dumps(template)}") + if time.monotonic() >= deadline: + raise PublishError(f"Timed out waiting for template {template_id} to become ready; last status was {current_status!r}.") + time.sleep(poll_seconds) + + +def wait_for_alias_target( + client: MorphClient, + alias: str, + template_id: str, + *, + timeout_seconds: int, + poll_seconds: float, +) -> dict: + deadline = time.monotonic() + timeout_seconds + while True: + alias_state = client.fetch_alias(alias) + if alias_state and alias_state.get("template_id") == template_id: + return alias_state + if time.monotonic() >= deadline: + raise PublishError(f"Alias {alias!r} did not update to template {template_id!r} before timeout.") + time.sleep(poll_seconds) + + +def wait_for_alias_missing( + client: MorphClient, + alias: str, + *, + timeout_seconds: int, + poll_seconds: float, +) -> None: + deadline = time.monotonic() + timeout_seconds + while True: + alias_state = client.fetch_alias(alias) + if alias_state is None: + return + if time.monotonic() >= deadline: + raise PublishError(f"Alias {alias!r} still existed after deleting its shared template.") + time.sleep(poll_seconds) + + +def publish_template( + client: MorphClient, + *, + alias: str, + template_path: Path, + base_snapshot_id: str | None = None, + tags: list[str] | None = None, + timeout_seconds: int = 1800, + poll_seconds: float = 5.0, +) -> dict: + metadata = read_template_metadata(template_path) + alias_state = client.fetch_alias(alias) + resolved_base_snapshot = base_snapshot_id or (alias_state or {}).get("base_snapshot_id") + if not resolved_base_snapshot: + raise PublishError( + f"Template alias {alias!r} does not exist and TEMPLATE_BASE_SNAPSHOT_ID was not provided." + ) + resolved_tags = tags or (alias_state or {}).get("tags") or DEFAULT_TAGS + log(f"publishing_alias {alias} base_snapshot={resolved_base_snapshot} tags={','.join(resolved_tags)}") + + created = client.create_template(metadata=metadata, base_snapshot_id=resolved_base_snapshot) + template_id = created.get("id") + if not template_id: + raise PublishError(f"Create template response did not include an id: {json.dumps(created)}") + log(f"created_template {template_id}") + + cache_result = client.cache_template(template_id) + cache_run_id = cache_result.get("run_id") or cache_result.get("runId") + if cache_run_id: + log(f"cache_started {cache_run_id}") + + ready = wait_for_ready(client, template_id, timeout_seconds=timeout_seconds, poll_seconds=poll_seconds) + log(f"template_ready {template_id} status={ready.get('status')} final_snapshot={ready.get('final_snapshot_id')}") + + if alias_state: + current_template_id = alias_state.get("template_id") + if not current_template_id: + raise PublishError(f"Alias {alias!r} did not include a template_id: {json.dumps(alias_state)}") + client.delete_template(current_template_id) + log(f"deleted_template {current_template_id}") + wait_for_alias_missing(client, alias, timeout_seconds=timeout_seconds, poll_seconds=poll_seconds) + log(f"alias_freed {alias}") + + share_result = client.share_template( + template_id=template_id, + alias=alias, + description=metadata.description, + tags=resolved_tags, + ) + log(f"alias_shared {alias}") + + alias_after = wait_for_alias_target( + client, + alias, + template_id, + timeout_seconds=timeout_seconds, + poll_seconds=poll_seconds, + ) + + return { + "template_id": template_id, + "alias": alias, + "description": metadata.description, + "tags": resolved_tags, + "cache_run_id": cache_run_id, + "share_result": share_result, + } + + +def env(name: str) -> str: + value = os.environ.get(name, "").strip() + if not value: + raise PublishError(f"Environment variable {name} is required.") + return value + + +def main() -> int: + try: + client = MorphClient( + base_url=env("DEVBOX_TEMPLATE_BASE_URL"), + api_key=env("MORPH_API_KEY"), + ) + result = publish_template( + client, + alias=env("TEMPLATE_ALIAS"), + template_path=Path(env("TEMPLATE_FILE")), + base_snapshot_id=os.environ.get("TEMPLATE_BASE_SNAPSHOT_ID", "").strip() or None, + tags=parse_tags(os.environ.get("TEMPLATE_TAGS")), + timeout_seconds=int(os.environ.get("TEMPLATE_TIMEOUT_SECONDS", "1800")), + poll_seconds=float(os.environ.get("TEMPLATE_POLL_SECONDS", "5")), + ) + except PublishError as exc: + print(str(exc), file=sys.stderr) + return 1 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_publish_shared_template.py b/tests/test_publish_shared_template.py new file mode 100644 index 0000000..8f23024 --- /dev/null +++ b/tests/test_publish_shared_template.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +import importlib.util +import json +import sys +from pathlib import Path +from urllib.error import HTTPError + +import pytest + + +SCRIPT_PATH = Path(__file__).resolve().parents[1] / "scripts" / "publish_shared_template.py" + + +def load_module(): + spec = importlib.util.spec_from_file_location("publish_shared_template", SCRIPT_PATH) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class FakeResponse: + def __init__(self, payload: dict): + self.payload = json.dumps(payload).encode("utf-8") + + def read(self) -> bytes: + return self.payload + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + +class FakeOpener: + def __init__(self, handlers): + self.handlers = handlers + self.requests = [] + + def __call__(self, request, timeout=120): + self.requests.append((request.get_method(), request.full_url, request.data, timeout)) + if not self.handlers: + raise AssertionError(f"Unexpected request {request.get_method()} {request.full_url}") + return self.handlers.pop(0)(request) + + +def test_read_template_metadata_requires_top_level_name_and_description(tmp_path): + module = load_module() + template = tmp_path / "template.yaml" + template.write_text( + "name: Test Template\n" + "description: Ready session without extra setup questions.\n" + "steps:\n" + " - id: one\n" + " type: command\n" + " run: echo hi\n", + encoding="utf-8", + ) + metadata = module.read_template_metadata(template) + assert metadata.name == "Test Template" + assert metadata.description == "Ready session without extra setup questions." + + +def test_publish_template_reuses_alias_snapshot_and_tags(tmp_path, monkeypatch): + module = load_module() + monkeypatch.setattr(module.time, "sleep", lambda _: None) + + template = tmp_path / "template.yaml" + template.write_text( + "name: Test Template\n" + "description: Ready session without extra setup questions.\n" + "steps:\n" + " - id: one\n" + " type: command\n" + " run: echo hi\n", + encoding="utf-8", + ) + + def alias_before(_request): + return FakeResponse( + { + "alias": "demo", + "base_snapshot_id": "snap_base", + "tags": ["template", "gauss"], + "template_id": "tpl_current", + } + ) + + def create_template(request): + body = json.loads(request.data.decode("utf-8")) + assert body["baseSnapshotId"] == "snap_base" + assert body["name"] == "Test Template" + assert "extra setup questions" in body["description"] + assert "steps:" in body["yaml"] + return FakeResponse({"id": "tpl_new", "status": "draft"}) + + def cache_template(_request): + return FakeResponse({"template_id": "tpl_new", "run_id": "run_1"}) + + template_polls = iter( + [ + {"id": "tpl_new", "status": "building"}, + {"id": "tpl_new", "status": "ready", "final_snapshot_id": "snap_new"}, + ] + ) + + def fetch_template(_request): + return FakeResponse(next(template_polls)) + + def delete_template(_request): + return FakeResponse({}) + + def alias_missing(_request): + raise HTTPError("https://devbox.example.test/api/aliases/demo", 404, "not found", hdrs=None, fp=None) + + def share_template(request): + body = json.loads(request.data.decode("utf-8")) + assert body["alias"] == "demo" + assert body["tags"] == ["template", "gauss"] + return FakeResponse({"published": True}) + + def alias_after(_request): + return FakeResponse({"alias": "demo", "template_id": "tpl_new", "tags": ["template", "gauss"]}) + + opener = FakeOpener( + [alias_before, create_template, cache_template, fetch_template, fetch_template, delete_template, alias_missing, share_template, alias_after] + ) + client = module.MorphClient("https://devbox.example.test", "token", opener=opener) + + result = module.publish_template( + client, + alias="demo", + template_path=template, + timeout_seconds=1, + poll_seconds=0, + ) + + assert result["template_id"] == "tpl_new" + assert result["alias"] == "demo" + assert result["tags"] == ["template", "gauss"] + assert [method for method, *_ in opener.requests] == ["GET", "POST", "POST", "GET", "GET", "DELETE", "GET", "POST", "GET"] + + +def test_publish_template_requires_base_snapshot_for_new_alias(tmp_path): + module = load_module() + + template = tmp_path / "template.yaml" + template.write_text( + "name: Test Template\n" + "description: Ready session without extra setup questions.\n", + encoding="utf-8", + ) + + def alias_missing(request): + raise HTTPError(request.full_url, 404, "not found", hdrs=None, fp=None) + + client = module.MorphClient("https://devbox.example.test", "token", opener=FakeOpener([alias_missing])) + + with pytest.raises(module.PublishError, match="TEMPLATE_BASE_SNAPSHOT_ID"): + module.publish_template(client, alias="missing", template_path=template, timeout_seconds=1, poll_seconds=0) + + +def test_publish_template_creates_and_shares_when_alias_is_missing(tmp_path, monkeypatch): + module = load_module() + monkeypatch.setattr(module.time, "sleep", lambda _: None) + + template = tmp_path / "template.yaml" + template.write_text( + "name: Test Template\n" + "description: Ready session without extra setup questions.\n" + "steps:\n" + " - id: one\n" + " type: command\n" + " run: echo hi\n", + encoding="utf-8", + ) + + def alias_missing(request): + raise HTTPError(request.full_url, 404, "not found", hdrs=None, fp=None) + + def create_template(request): + body = json.loads(request.data.decode("utf-8")) + assert body["baseSnapshotId"] == "snap_base" + return FakeResponse({"id": "tpl_new", "status": "draft"}) + + def cache_template(_request): + return FakeResponse({"run_id": "run_1"}) + + template_polls = iter( + [ + {"id": "tpl_new", "status": "building"}, + {"id": "tpl_new", "status": "ready", "final_snapshot_id": "snap_new"}, + ] + ) + + def fetch_template(_request): + return FakeResponse(next(template_polls)) + + def share_template(request): + body = json.loads(request.data.decode("utf-8")) + assert body["alias"] == "demo" + assert body["tags"] == ["template", "gauss"] + return FakeResponse({"published": True}) + + def alias_after(_request): + return FakeResponse({"alias": "demo", "template_id": "tpl_new", "tags": ["template", "gauss"]}) + + opener = FakeOpener([alias_missing, create_template, cache_template, fetch_template, fetch_template, share_template, alias_after]) + client = module.MorphClient("https://devbox.example.test", "token", opener=opener) + + result = module.publish_template( + client, + alias="demo", + template_path=template, + base_snapshot_id="snap_base", + timeout_seconds=1, + poll_seconds=0, + ) + + assert result["template_id"] == "tpl_new" + assert result["alias"] == "demo"