From 8f60eee9f910e67476d967f94d86dd0122750d0d Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Sat, 2 May 2026 13:09:14 -0300 Subject: [PATCH 01/13] test: cover publish payload --- .github/scripts/test_build_publish_payload.py | 216 ++++++++++++++++++ 1 file changed, 216 insertions(+) create mode 100644 .github/scripts/test_build_publish_payload.py diff --git a/.github/scripts/test_build_publish_payload.py b/.github/scripts/test_build_publish_payload.py new file mode 100644 index 00000000..30f07e7d --- /dev/null +++ b/.github/scripts/test_build_publish_payload.py @@ -0,0 +1,216 @@ +import tempfile +import unittest +from pathlib import Path + +from build_publish_payload import ( + build_payload, + derive_registry_function_name, + normalize_dependencies, + normalize_worker_interface, +) + + +class PublishPayloadTests(unittest.TestCase): + def test_normalize_dependencies_accepts_map(self): + self.assertEqual( + normalize_dependencies({"helper-worker": "^1.0.0"}), + [{"name": "helper-worker", "version": "^1.0.0"}], + ) + + def test_normalize_dependencies_accepts_wire_list(self): + deps = [{"name": "helper-worker", "version": "^1.0.0"}] + self.assertEqual(normalize_dependencies(deps), deps) + + def test_normalize_dependencies_rejects_scalar(self): + with self.assertRaisesRegex(ValueError, "dependencies"): + normalize_dependencies("helper-worker") + + def test_derive_registry_function_name_prefers_metadata_registry_name(self): + self.assertEqual( + derive_registry_function_name( + "image_resize::resize", + {"registry_name": "resize_image", "name": "ignored"}, + ), + "resize_image", + ) + + def test_derive_registry_function_name_falls_back_to_final_function_segment(self): + self.assertEqual(derive_registry_function_name("image_resize::resize", {}), "resize") + + def test_normalize_worker_interface_converts_function_schema_fields(self): + workers = { + "workers": [ + { + "id": "worker-1", + "name": "image-resize", + "functions": ["image_resize::resize", "image_resize::ping"], + } + ] + } + functions = { + "functions": [ + { + "function_id": "image_resize::resize", + "description": "Resize an image", + "request_format": {"type": "object"}, + "response_format": {"type": "object"}, + "metadata": {"tags": ["image", "transform"]}, + }, + { + "function_id": "image_resize::ping", + "description": None, + "request_format": None, + "response_format": None, + "metadata": None, + }, + { + "function_id": "other::fn", + "description": "Not this worker", + "request_format": {"type": "object"}, + "response_format": None, + "metadata": {}, + }, + ] + } + triggers = { + "triggers": [ + { + "id": "t1", + "trigger_type": "http", + "function_id": "image_resize::resize", + "config": {"api_path": "/resize"}, + "metadata": {"public": True}, + }, + { + "id": "t2", + "trigger_type": "http", + "function_id": "other::fn", + "config": {"api_path": "/other"}, + }, + ] + } + + interface = normalize_worker_interface( + worker_name="image-resize", + workers_json=workers, + functions_json=functions, + triggers_json=triggers, + ) + + self.assertEqual( + interface["functions"], + [ + { + "name": "resize", + "description": "Resize an image", + "request_schema": {"type": "object"}, + "response_schema": {"type": "object"}, + "metadata": {"tags": ["image", "transform"]}, + }, + { + "name": "ping", + "description": None, + "request_schema": None, + "response_schema": None, + "metadata": {}, + }, + ], + ) + self.assertEqual( + interface["triggers"], + [ + { + "id": "t1", + "trigger_type": "http", + "function_id": "image_resize::resize", + "config": {"api_path": "/resize"}, + "metadata": {"public": True}, + } + ], + ) + + def test_build_binary_payload_has_registry_shape_and_binaries(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "image-resize" + root.mkdir() + (root / "iii.worker.yaml").write_text( + "description: Resize images from a URL.\n" + "dependencies:\n" + " helper-worker: '^1.0.0'\n", + encoding="utf-8", + ) + (root / "README.md").write_text("# image-resize\n", encoding="utf-8") + (root / "config.yaml").write_text("{}\n", encoding="utf-8") + + payload = build_payload( + repo_root=Path(tmp), + worker="image-resize", + version="0.1.0", + registry_tag="latest", + deploy="binary", + repo_url="https://github.com/example/image-resize", + interface={ + "functions": [ + { + "name": "resize", + "description": "Resize", + "request_schema": {"type": "object"}, + "response_schema": None, + "metadata": {}, + } + ], + "triggers": [], + }, + binaries={ + "x86_64-unknown-linux-gnu": { + "url": "https://example.com/releases/image-resize-x86_64-unknown-linux-gnu.tar.gz", + "sha256": "b" * 64, + } + }, + image_tag="", + ) + + self.assertEqual(payload["worker_name"], "image-resize") + self.assertEqual(payload["version"], "0.1.0") + self.assertEqual(payload["tag"], "latest") + self.assertEqual(payload["type"], "binary") + self.assertEqual(payload["readme"], "# image-resize\n") + self.assertEqual(payload["repo"], "https://github.com/example/image-resize") + self.assertEqual(payload["description"], "Resize images from a URL.") + self.assertEqual( + payload["dependencies"], + [{"name": "helper-worker", "version": "^1.0.0"}], + ) + self.assertEqual(payload["config"], {}) + self.assertEqual(payload["functions"][0]["name"], "resize") + self.assertIn("request_schema", payload["functions"][0]) + self.assertIn("response_schema", payload["functions"][0]) + self.assertEqual(payload["triggers"], []) + self.assertIn("binaries", payload) + self.assertNotIn("image_tag", payload) + + def test_build_image_payload_has_image_tag(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "hello-worker" + root.mkdir() + (root / "iii.worker.yaml").write_text("description: Demo worker\n", encoding="utf-8") + + payload = build_payload( + repo_root=Path(tmp), + worker="hello-worker", + version="1.0.0", + registry_tag="latest", + deploy="image", + repo_url="https://github.com/example/workers", + interface={"functions": [], "triggers": []}, + binaries={}, + image_tag="ghcr.io/example/hello-worker:1.0.0", + ) + + self.assertEqual(payload["type"], "image") + self.assertEqual(payload["image_tag"], "ghcr.io/example/hello-worker:1.0.0") + self.assertNotIn("binaries", payload) + + +if __name__ == "__main__": + unittest.main() From f3ef6dadf0270c0cb91c5c3a76528e636174a002 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Sat, 2 May 2026 13:09:59 -0300 Subject: [PATCH 02/13] feat: build publish payload --- .github/scripts/build_publish_payload.py | 184 +++++++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 .github/scripts/build_publish_payload.py diff --git a/.github/scripts/build_publish_payload.py b/.github/scripts/build_publish_payload.py new file mode 100644 index 00000000..6148c68a --- /dev/null +++ b/.github/scripts/build_publish_payload.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +import argparse +import json +import pathlib +import sys +from typing import Any + +import yaml + + +def normalize_dependencies(raw_deps: Any) -> list[dict[str, Any]]: + if raw_deps in (None, ""): + return [] + if isinstance(raw_deps, dict): + return [{"name": name, "version": version} for name, version in raw_deps.items()] + if isinstance(raw_deps, list): + return raw_deps + raise ValueError(f"`dependencies` must be a map or list, got {type(raw_deps).__name__}") + + +def derive_registry_function_name(function_id: str, metadata: dict[str, Any] | None) -> str: + metadata = metadata or {} + for key in ("registry_name", "name"): + value = metadata.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + if "::" in function_id: + return function_id.rsplit("::", 1)[1] + return function_id + + +def _extract_array(payload: dict[str, Any], key: str) -> list[dict[str, Any]]: + value = payload.get(key, []) + if value is None: + return [] + if not isinstance(value, list): + raise ValueError(f"`{key}` must be an array") + return value + + +def normalize_worker_interface( + *, + worker_name: str, + workers_json: dict[str, Any], + functions_json: dict[str, Any], + triggers_json: dict[str, Any] | None = None, +) -> dict[str, list[dict[str, Any]]]: + workers = _extract_array(workers_json, "workers") + matches = [w for w in workers if w.get("name") == worker_name or w.get("id") == worker_name] + if len(matches) != 1: + raise ValueError(f"expected exactly one worker matching {worker_name!r}, found {len(matches)}") + + worker_function_ids = matches[0].get("functions") or [] + if not isinstance(worker_function_ids, list): + raise ValueError("worker `functions` must be an array") + + functions_by_id = { + f.get("function_id"): f + for f in _extract_array(functions_json, "functions") + if f.get("function_id") + } + + functions = [] + for function_id in worker_function_ids: + details = functions_by_id.get(function_id, {}) + metadata = details.get("metadata") or {} + functions.append( + { + "name": derive_registry_function_name(function_id, metadata), + "description": details.get("description"), + "request_schema": details.get("request_format"), + "response_schema": details.get("response_format"), + "metadata": metadata if isinstance(metadata, dict) else {}, + } + ) + + worker_ids = set(worker_function_ids) + triggers = [] + if triggers_json: + for trigger in _extract_array(triggers_json, "triggers"): + if trigger.get("function_id") not in worker_ids: + continue + metadata = trigger.get("metadata") or {} + triggers.append( + { + "id": trigger.get("id"), + "trigger_type": trigger.get("trigger_type"), + "function_id": trigger.get("function_id"), + "config": trigger.get("config") or {}, + "metadata": metadata if isinstance(metadata, dict) else {}, + } + ) + + return {"functions": functions, "triggers": triggers} + + +def build_payload( + *, + repo_root: pathlib.Path, + worker: str, + version: str, + registry_tag: str, + deploy: str, + repo_url: str, + interface: dict[str, Any], + binaries: dict[str, Any], + image_tag: str, +) -> dict[str, Any]: + root = repo_root / worker + meta = yaml.safe_load((root / "iii.worker.yaml").read_text(encoding="utf-8")) or {} + + readme_path = root / "README.md" + readme = readme_path.read_text(encoding="utf-8") if readme_path.exists() else "" + + config_path = root / "config.yaml" + config = yaml.safe_load(config_path.read_text(encoding="utf-8")) if config_path.exists() else {} + if config is None: + config = {} + + payload: dict[str, Any] = { + "worker_name": worker, + "version": version, + "tag": registry_tag or "latest", + "type": deploy, + "readme": readme, + "repo": repo_url, + "description": meta.get("description", ""), + "dependencies": normalize_dependencies(meta.get("dependencies")), + "config": config, + "functions": interface.get("functions") or [], + "triggers": interface.get("triggers") or [], + } + + if deploy == "binary": + if not binaries: + raise ValueError("deploy=binary requires non-empty binaries") + payload["binaries"] = binaries + elif deploy == "image": + if not image_tag: + raise ValueError("deploy=image requires image_tag") + payload["image_tag"] = image_tag + else: + raise ValueError(f"unsupported deploy={deploy}") + + return payload + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--worker", required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--registry-tag", default="latest") + parser.add_argument("--deploy", required=True, choices=["binary", "image"]) + parser.add_argument("--repo-url", required=True) + parser.add_argument("--interface-json", required=True) + parser.add_argument("--binaries-json", default="") + parser.add_argument("--image-tag", default="") + parser.add_argument("--repo-root", default=".") + parser.add_argument("--out", default="payload.json") + args = parser.parse_args() + + interface = json.loads(pathlib.Path(args.interface_json).read_text(encoding="utf-8")) + binaries = {} + if args.binaries_json: + binaries = json.loads(pathlib.Path(args.binaries_json).read_text(encoding="utf-8")) + + payload = build_payload( + repo_root=pathlib.Path(args.repo_root), + worker=args.worker, + version=args.version, + registry_tag=args.registry_tag, + deploy=args.deploy, + repo_url=args.repo_url, + interface=interface, + binaries=binaries, + image_tag=args.image_tag, + ) + pathlib.Path(args.out).write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") + print(json.dumps({k: v for k, v in payload.items() if k != "readme"}, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 0c5f59c891bd7325c04757a7895a24a35229b638 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Sat, 2 May 2026 13:10:41 -0300 Subject: [PATCH 03/13] feat: resolve binary artifacts --- .github/scripts/resolve_binary_artifacts.py | 76 +++++++++++++++++++ .../scripts/test_resolve_binary_artifacts.py | 32 ++++++++ 2 files changed, 108 insertions(+) create mode 100644 .github/scripts/resolve_binary_artifacts.py create mode 100644 .github/scripts/test_resolve_binary_artifacts.py diff --git a/.github/scripts/resolve_binary_artifacts.py b/.github/scripts/resolve_binary_artifacts.py new file mode 100644 index 00000000..f374a40e --- /dev/null +++ b/.github/scripts/resolve_binary_artifacts.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +import argparse +import json +import pathlib +import sys +import urllib.request +from collections.abc import Callable + + +DEFAULT_TARGETS = [ + "x86_64-apple-darwin", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc", + "i686-pc-windows-msvc", + "aarch64-pc-windows-msvc", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "aarch64-unknown-linux-gnu", + "armv7-unknown-linux-gnueabihf", +] + + +def read_checksum_url(url: str) -> str: + with urllib.request.urlopen(url, timeout=20) as response: + text = response.read().decode("utf-8").strip() + return text.split()[0] + + +def build_binary_artifact_map( + *, + repo_url: str, + tag: str, + bin_name: str, + targets: list[str], + read_checksum: Callable[[str], str], +) -> dict[str, dict[str, str]]: + base = f"{repo_url}/releases/download/{tag}" + binaries = {} + for target in targets: + ext = "zip" if "windows" in target else "tar.gz" + asset_url = f"{base}/{bin_name}-{target}.{ext}" + sha_url = f"{base}/{bin_name}-{target}.sha256" + try: + binaries[target] = { + "url": asset_url, + "sha256": read_checksum(sha_url), + } + except Exception as exc: + print(f"::warning::missing checksum for {target}: {exc}", file=sys.stderr) + if not binaries: + raise RuntimeError("no binary artefacts could be resolved") + return binaries + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--repo-url", required=True) + parser.add_argument("--tag", required=True) + parser.add_argument("--bin", required=True) + parser.add_argument("--out", default="binaries.json") + args = parser.parse_args() + + binaries = build_binary_artifact_map( + repo_url=args.repo_url, + tag=args.tag, + bin_name=args.bin, + targets=DEFAULT_TARGETS, + read_checksum=read_checksum_url, + ) + pathlib.Path(args.out).write_text(json.dumps(binaries, indent=2) + "\n", encoding="utf-8") + print(json.dumps(binaries, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/test_resolve_binary_artifacts.py b/.github/scripts/test_resolve_binary_artifacts.py new file mode 100644 index 00000000..ca3b9d86 --- /dev/null +++ b/.github/scripts/test_resolve_binary_artifacts.py @@ -0,0 +1,32 @@ +import unittest + +from resolve_binary_artifacts import build_binary_artifact_map + + +class ResolveBinaryArtifactsTests(unittest.TestCase): + def test_builds_binary_urls_and_sha_values(self): + checksums = { + "https://github.com/example/workers/releases/download/image-resize/v0.1.0/image-resize-x86_64-unknown-linux-gnu.sha256": "b" * 64 + } + + result = build_binary_artifact_map( + repo_url="https://github.com/example/workers", + tag="image-resize/v0.1.0", + bin_name="image-resize", + targets=["x86_64-unknown-linux-gnu"], + read_checksum=lambda url: checksums[url], + ) + + self.assertEqual( + result, + { + "x86_64-unknown-linux-gnu": { + "url": "https://github.com/example/workers/releases/download/image-resize/v0.1.0/image-resize-x86_64-unknown-linux-gnu.tar.gz", + "sha256": "b" * 64, + } + }, + ) + + +if __name__ == "__main__": + unittest.main() From f6298408da2cc89544b7faf0b6d17c570c90486b Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Sat, 2 May 2026 13:14:01 -0300 Subject: [PATCH 04/13] feat: collect worker interface --- .github/scripts/collect_worker_interface.py | 60 +++++++++++++++++++ .github/scripts/test_build_publish_payload.py | 36 +++++++++++ 2 files changed, 96 insertions(+) create mode 100644 .github/scripts/collect_worker_interface.py diff --git a/.github/scripts/collect_worker_interface.py b/.github/scripts/collect_worker_interface.py new file mode 100644 index 00000000..a946a34f --- /dev/null +++ b/.github/scripts/collect_worker_interface.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +import argparse +import json +import pathlib +import subprocess +import sys + +from build_publish_payload import normalize_worker_interface + + +def run_iii(function_id: str, payload: dict[str, object]) -> dict[str, object]: + completed = subprocess.run( + [ + "iii", + "trigger", + "--function-id", + function_id, + "--payload", + json.dumps(payload), + ], + check=True, + text=True, + capture_output=True, + ) + return json.loads(completed.stdout) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--worker", required=True) + parser.add_argument("--out", default="worker-interface.json") + parser.add_argument("--allow-missing-triggers", action="store_true") + args = parser.parse_args() + + workers_json = run_iii("engine::workers::list", {}) + functions_json = run_iii("engine::functions::list", {"include_internal": True}) + + triggers_json = None + try: + triggers_json = run_iii("engine::triggers::list", {"include_internal": True}) + except (subprocess.CalledProcessError, json.JSONDecodeError) as exc: + if not args.allow_missing_triggers: + raise RuntimeError( + "could not collect triggers; confirm the engine exposes " + "`engine::triggers::list` or pass --allow-missing-triggers" + ) from exc + + interface = normalize_worker_interface( + worker_name=args.worker, + workers_json=workers_json, + functions_json=functions_json, + triggers_json=triggers_json, + ) + pathlib.Path(args.out).write_text(json.dumps(interface, indent=2) + "\n", encoding="utf-8") + print(json.dumps(interface, indent=2)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/test_build_publish_payload.py b/.github/scripts/test_build_publish_payload.py index 30f07e7d..7bf3f377 100644 --- a/.github/scripts/test_build_publish_payload.py +++ b/.github/scripts/test_build_publish_payload.py @@ -129,6 +129,42 @@ def test_normalize_worker_interface_converts_function_schema_fields(self): ], ) + def test_normalize_worker_interface_rejects_missing_worker(self): + with self.assertRaisesRegex(ValueError, "expected exactly one worker"): + normalize_worker_interface( + worker_name="missing", + workers_json={"workers": []}, + functions_json={"functions": []}, + triggers_json={"triggers": []}, + ) + + def test_normalize_worker_interface_accepts_no_triggers_source(self): + interface = normalize_worker_interface( + worker_name="image-resize", + workers_json={ + "workers": [ + { + "id": "worker-1", + "name": "image-resize", + "functions": ["image_resize::resize"], + } + ] + }, + functions_json={ + "functions": [ + { + "function_id": "image_resize::resize", + "description": "Resize", + "request_format": None, + "response_format": None, + "metadata": None, + } + ] + }, + triggers_json=None, + ) + self.assertEqual(interface["triggers"], []) + def test_build_binary_payload_has_registry_shape_and_binaries(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp) / "image-resize" From de32b10ec54e6ae898a616ad04800102709b096f Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Sat, 2 May 2026 13:14:54 -0300 Subject: [PATCH 05/13] ci: build publish payload --- .github/workflows/_publish-registry.yml | 168 ++++++++++-------------- 1 file changed, 66 insertions(+), 102 deletions(-) diff --git a/.github/workflows/_publish-registry.yml b/.github/workflows/_publish-registry.yml index 40979183..0b7954f8 100644 --- a/.github/workflows/_publish-registry.yml +++ b/.github/workflows/_publish-registry.yml @@ -34,6 +34,16 @@ on: required: false type: string default: '' + collect_interface: + description: 'Collect functions/triggers from a running iii engine' + required: false + type: boolean + default: false + allow_missing_triggers: + description: 'Publish triggers=[] if engine trigger listing is unavailable' + required: false + type: boolean + default: true api_url: description: 'Workers registry base URL' required: false @@ -52,117 +62,71 @@ jobs: - name: Install pyyaml run: pip install --quiet pyyaml + - name: Collect worker interface + if: inputs.collect_interface == true + env: + WORKER: ${{ inputs.worker }} + ALLOW_MISSING_TRIGGERS: ${{ inputs.allow_missing_triggers }} + run: | + args=("--worker" "$WORKER" "--out" "worker-interface.json") + if [[ "$ALLOW_MISSING_TRIGGERS" == "true" ]]; then + args+=("--allow-missing-triggers") + fi + python3 .github/scripts/collect_worker_interface.py "${args[@]}" + + - name: Assert worker interface was collected + if: inputs.collect_interface == true + run: | + python3 - <<'PY' + import json + data = json.load(open("worker-interface.json")) + if not data.get("functions"): + raise SystemExit("collect_interface=true but no functions were collected") + PY + + - name: Create empty worker interface + if: inputs.collect_interface != true + run: | + printf '{ "functions": [], "triggers": [] }\n' > worker-interface.json + + - name: Resolve binary artifacts + if: inputs.deploy == 'binary' + env: + TAG: ${{ inputs.tag }} + BIN: ${{ inputs.bin }} + WORKER: ${{ inputs.worker }} + REPO_URL: ${{ format('https://github.com/{0}', github.repository) }} + run: | + python3 .github/scripts/resolve_binary_artifacts.py \ + --repo-url "$REPO_URL" \ + --tag "$TAG" \ + --bin "${BIN:-$WORKER}" \ + --out binaries.json + + - name: Create empty binary artifacts + if: inputs.deploy != 'binary' + run: | + printf '{}\n' > binaries.json + - name: Build payload - id: payload env: WORKER: ${{ inputs.worker }} VERSION: ${{ inputs.version }} DEPLOY: ${{ inputs.deploy }} REGISTRY_TAG: ${{ inputs.registry_tag }} - TAG: ${{ inputs.tag }} - BIN: ${{ inputs.bin }} IMAGE_TAG: ${{ inputs.image_tag }} REPO_URL: ${{ format('https://github.com/{0}', github.repository) }} run: | - python3 - <<'PY' - import json, os, pathlib, sys, urllib.request, yaml - - worker = os.environ["WORKER"] - version = os.environ["VERSION"] - deploy = os.environ["DEPLOY"] - reg_tag = os.environ["REGISTRY_TAG"] or "latest" - tag = os.environ["TAG"] - repo_url = os.environ["REPO_URL"] - bin_name = os.environ.get("BIN") or worker - image_tag = os.environ.get("IMAGE_TAG", "") - - root = pathlib.Path(worker) - meta = yaml.safe_load((root / "iii.worker.yaml").read_text()) or {} - - # Normalize `dependencies` into the array-of-objects wire shape the - # registry's /publish endpoint expects. Authors may write either: - # dependencies: - # math-worker: "^0.1.0" # map form (recommended) - # or: - # dependencies: - # - name: math-worker - # version: "^0.1.0" # explicit wire form - raw_deps = meta.get("dependencies") or [] - if isinstance(raw_deps, dict): - deps = [{"name": k, "version": v} for k, v in raw_deps.items()] - elif isinstance(raw_deps, list): - deps = raw_deps - else: - print( - f"::error::`dependencies` must be a map or list, got {type(raw_deps).__name__}" - ) - sys.exit(1) - - readme_path = root / "README.md" - readme = readme_path.read_text() if readme_path.exists() else "" - - config_path = root / "config.yaml" - config = {} - if config_path.exists(): - config = yaml.safe_load(config_path.read_text()) or {} - - payload = { - "worker_name": worker, - "version": version, - "tag": reg_tag, - "type": deploy, - "readme": readme, - "repo": repo_url, - "description": meta.get("description", ""), - "dependencies": deps, - "config": config, - } - - if deploy == "binary": - targets = [ - "x86_64-apple-darwin", - "aarch64-apple-darwin", - "x86_64-pc-windows-msvc", - "i686-pc-windows-msvc", - "aarch64-pc-windows-msvc", - "x86_64-unknown-linux-gnu", - "x86_64-unknown-linux-musl", - "aarch64-unknown-linux-gnu", - "armv7-unknown-linux-gnueabihf", - ] - base = f"{repo_url}/releases/download/{tag}" - binaries = {} - for t in targets: - ext = "zip" if "windows" in t else "tar.gz" - asset = f"{bin_name}-{t}.{ext}" - asset_url = f"{base}/{asset}" - # taiki-e/upload-rust-binary-action publishes the checksum - # as `-.sha256` (without the archive extension). - sha_url = f"{base}/{bin_name}-{t}.sha256" - try: - with urllib.request.urlopen(sha_url, timeout=20) as r: - sha_text = r.read().decode("utf-8").strip() - sha = sha_text.split()[0] - binaries[t] = {"url": asset_url, "sha256": sha} - except Exception as e: - print(f"::warning::missing checksum for {t}: {e}") - if not binaries: - print("::error::no binary artefacts could be resolved") - sys.exit(1) - payload["binaries"] = binaries - elif deploy == "image": - if not image_tag: - print("::error::deploy=image but no image_tag provided") - sys.exit(1) - payload["image_tag"] = image_tag - else: - print(f"::error::unsupported deploy={deploy}") - sys.exit(1) - - out = pathlib.Path("payload.json") - out.write_text(json.dumps(payload, indent=2)) - print(json.dumps({k: payload[k] for k in payload if k != "readme"}, indent=2)) - PY + python3 .github/scripts/build_publish_payload.py \ + --worker "$WORKER" \ + --version "$VERSION" \ + --registry-tag "$REGISTRY_TAG" \ + --deploy "$DEPLOY" \ + --repo-url "$REPO_URL" \ + --interface-json worker-interface.json \ + --binaries-json binaries.json \ + --image-tag "$IMAGE_TAG" \ + --out payload.json - name: POST /publish env: From c081070beb596889995729bfc66aa2d36617a9da Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Sat, 2 May 2026 13:22:17 -0300 Subject: [PATCH 06/13] fix: avoid yaml dependency for interface collection --- .github/scripts/build_publish_payload.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/scripts/build_publish_payload.py b/.github/scripts/build_publish_payload.py index 6148c68a..9ca7b8e1 100644 --- a/.github/scripts/build_publish_payload.py +++ b/.github/scripts/build_publish_payload.py @@ -5,8 +5,6 @@ import sys from typing import Any -import yaml - def normalize_dependencies(raw_deps: Any) -> list[dict[str, Any]]: if raw_deps in (None, ""): @@ -38,6 +36,12 @@ def _extract_array(payload: dict[str, Any], key: str) -> list[dict[str, Any]]: return value +def _read_yaml(path: pathlib.Path) -> Any: + import yaml + + return yaml.safe_load(path.read_text(encoding="utf-8")) + + def normalize_worker_interface( *, worker_name: str, @@ -107,13 +111,13 @@ def build_payload( image_tag: str, ) -> dict[str, Any]: root = repo_root / worker - meta = yaml.safe_load((root / "iii.worker.yaml").read_text(encoding="utf-8")) or {} + meta = _read_yaml(root / "iii.worker.yaml") or {} readme_path = root / "README.md" readme = readme_path.read_text(encoding="utf-8") if readme_path.exists() else "" config_path = root / "config.yaml" - config = yaml.safe_load(config_path.read_text(encoding="utf-8")) if config_path.exists() else {} + config = _read_yaml(config_path) if config_path.exists() else {} if config is None: config = {} From 0932a9638922b32d1c87b66b7a5459de4f02f13c Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Sat, 2 May 2026 19:14:51 -0300 Subject: [PATCH 07/13] fix: map live publish metadata to registry schema --- .github/scripts/build_publish_payload.py | 103 +++++++++++++++--- .github/scripts/test_build_publish_payload.py | 23 ++-- 2 files changed, 102 insertions(+), 24 deletions(-) diff --git a/.github/scripts/build_publish_payload.py b/.github/scripts/build_publish_payload.py index 9ca7b8e1..c174e98a 100644 --- a/.github/scripts/build_publish_payload.py +++ b/.github/scripts/build_publish_payload.py @@ -2,6 +2,7 @@ import argparse import json import pathlib +import re import sys from typing import Any @@ -42,6 +43,79 @@ def _read_yaml(path: pathlib.Path) -> Any: return yaml.safe_load(path.read_text(encoding="utf-8")) +def _schema_or_empty(value: Any) -> dict[str, Any]: + if value is None: + return {} + if isinstance(value, dict): + return value + raise ValueError("function schema fields must be objects or null") + + +def _metadata_or_empty(value: Any) -> dict[str, Any]: + return value if isinstance(value, dict) else {} + + +def _string_or_empty(value: Any) -> str: + return value if isinstance(value, str) else "" + + +def _slug(value: Any, fallback: str) -> str: + raw = value if isinstance(value, str) else fallback + slug = re.sub(r"[^a-z0-9]+", "-", raw.lower()).strip("-") + return slug or fallback + + +def _normalize_registry_function(function: dict[str, Any]) -> dict[str, Any]: + return { + "name": function.get("name"), + "description": _string_or_empty(function.get("description")), + "request_schema": _schema_or_empty(function.get("request_schema")), + "response_schema": _schema_or_empty(function.get("response_schema")), + "metadata": _metadata_or_empty(function.get("metadata")), + } + + +def _derive_trigger_name(trigger: dict[str, Any]) -> str: + metadata = _metadata_or_empty(trigger.get("metadata")) + for key in ("registry_name", "name"): + value = metadata.get(key) + if isinstance(value, str) and value.strip(): + return _slug(value, "trigger") + + config = trigger.get("config") if isinstance(trigger.get("config"), dict) else {} + api_path = config.get("api_path") + if isinstance(api_path, str) and api_path.strip(): + return _slug(api_path, "trigger") + + function_id = trigger.get("function_id") + if isinstance(function_id, str) and function_id.strip(): + return _slug(function_id.rsplit("::", 1)[-1], "trigger") + + return _slug(trigger.get("trigger_type") or trigger.get("name"), "trigger") + + +def _normalize_registry_trigger(trigger: dict[str, Any]) -> dict[str, Any]: + config = trigger.get("config") if isinstance(trigger.get("config"), dict) else {} + metadata = _metadata_or_empty(trigger.get("metadata")).copy() + for source_key, metadata_key in ( + ("id", "engine_id"), + ("trigger_type", "trigger_type"), + ("function_id", "function_id"), + ): + if trigger.get(source_key) is not None: + metadata.setdefault(metadata_key, trigger.get(source_key)) + if config: + metadata.setdefault("config", config) + + return { + "name": _derive_trigger_name(trigger), + "description": _string_or_empty(trigger.get("description")), + "invocation_schema": _schema_or_empty(trigger.get("invocation_schema")), + "return_schema": _schema_or_empty(trigger.get("return_schema")), + "metadata": metadata, + } + + def normalize_worker_interface( *, worker_name: str, @@ -71,10 +145,10 @@ def normalize_worker_interface( functions.append( { "name": derive_registry_function_name(function_id, metadata), - "description": details.get("description"), - "request_schema": details.get("request_format"), - "response_schema": details.get("response_format"), - "metadata": metadata if isinstance(metadata, dict) else {}, + "description": _string_or_empty(details.get("description")), + "request_schema": _schema_or_empty(details.get("request_format")), + "response_schema": _schema_or_empty(details.get("response_format")), + "metadata": _metadata_or_empty(metadata), } ) @@ -84,16 +158,7 @@ def normalize_worker_interface( for trigger in _extract_array(triggers_json, "triggers"): if trigger.get("function_id") not in worker_ids: continue - metadata = trigger.get("metadata") or {} - triggers.append( - { - "id": trigger.get("id"), - "trigger_type": trigger.get("trigger_type"), - "function_id": trigger.get("function_id"), - "config": trigger.get("config") or {}, - "metadata": metadata if isinstance(metadata, dict) else {}, - } - ) + triggers.append(_normalize_registry_trigger(trigger)) return {"functions": functions, "triggers": triggers} @@ -131,8 +196,14 @@ def build_payload( "description": meta.get("description", ""), "dependencies": normalize_dependencies(meta.get("dependencies")), "config": config, - "functions": interface.get("functions") or [], - "triggers": interface.get("triggers") or [], + "functions": [ + _normalize_registry_function(function) + for function in interface.get("functions") or [] + ], + "triggers": [ + _normalize_registry_trigger(trigger) + for trigger in interface.get("triggers") or [] + ], } if deploy == "binary": diff --git a/.github/scripts/test_build_publish_payload.py b/.github/scripts/test_build_publish_payload.py index 7bf3f377..6bd3a4df 100644 --- a/.github/scripts/test_build_publish_payload.py +++ b/.github/scripts/test_build_publish_payload.py @@ -109,9 +109,9 @@ def test_normalize_worker_interface_converts_function_schema_fields(self): }, { "name": "ping", - "description": None, - "request_schema": None, - "response_schema": None, + "description": "", + "request_schema": {}, + "response_schema": {}, "metadata": {}, }, ], @@ -120,11 +120,17 @@ def test_normalize_worker_interface_converts_function_schema_fields(self): interface["triggers"], [ { - "id": "t1", - "trigger_type": "http", - "function_id": "image_resize::resize", - "config": {"api_path": "/resize"}, - "metadata": {"public": True}, + "name": "resize", + "description": "", + "invocation_schema": {}, + "return_schema": {}, + "metadata": { + "public": True, + "engine_id": "t1", + "trigger_type": "http", + "function_id": "image_resize::resize", + "config": {"api_path": "/resize"}, + }, } ], ) @@ -221,6 +227,7 @@ def test_build_binary_payload_has_registry_shape_and_binaries(self): self.assertEqual(payload["functions"][0]["name"], "resize") self.assertIn("request_schema", payload["functions"][0]) self.assertIn("response_schema", payload["functions"][0]) + self.assertEqual(payload["functions"][0]["response_schema"], {}) self.assertEqual(payload["triggers"], []) self.assertIn("binaries", payload) self.assertNotIn("image_tag", payload) From a9dbf563df5e50bdabf7b19d25097ed773ee42b8 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Sat, 2 May 2026 19:29:22 -0300 Subject: [PATCH 08/13] ci: collect worker interface during release publish --- .github/scripts/collect_worker_interface.py | 27 +++++++- .../scripts/test_collect_worker_interface.py | 45 +++++++++++++ .github/workflows/_publish-registry.yml | 65 ++++++++++++++++++- .github/workflows/release.yml | 1 + 4 files changed, 136 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/test_collect_worker_interface.py diff --git a/.github/scripts/collect_worker_interface.py b/.github/scripts/collect_worker_interface.py index a946a34f..f4b0f6c7 100644 --- a/.github/scripts/collect_worker_interface.py +++ b/.github/scripts/collect_worker_interface.py @@ -4,10 +4,23 @@ import pathlib import subprocess import sys +import time from build_publish_payload import normalize_worker_interface +def count_worker_matches(workers_json: dict[str, object], worker_name: str) -> int: + workers = workers_json.get("workers", []) + if not isinstance(workers, list): + return 0 + return sum( + 1 + for worker in workers + if isinstance(worker, dict) + and (worker.get("name") == worker_name or worker.get("id") == worker_name) + ) + + def run_iii(function_id: str, payload: dict[str, object]) -> dict[str, object]: completed = subprocess.run( [ @@ -25,14 +38,26 @@ def run_iii(function_id: str, payload: dict[str, object]) -> dict[str, object]: return json.loads(completed.stdout) +def wait_for_worker(worker_name: str, wait_seconds: int) -> dict[str, object]: + deadline = time.monotonic() + wait_seconds + workers_json = run_iii("engine::workers::list", {}) + + while count_worker_matches(workers_json, worker_name) != 1 and time.monotonic() < deadline: + time.sleep(2) + workers_json = run_iii("engine::workers::list", {}) + + return workers_json + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--worker", required=True) parser.add_argument("--out", default="worker-interface.json") parser.add_argument("--allow-missing-triggers", action="store_true") + parser.add_argument("--wait-seconds", type=int, default=0) args = parser.parse_args() - workers_json = run_iii("engine::workers::list", {}) + workers_json = wait_for_worker(args.worker, args.wait_seconds) functions_json = run_iii("engine::functions::list", {"include_internal": True}) triggers_json = None diff --git a/.github/scripts/test_collect_worker_interface.py b/.github/scripts/test_collect_worker_interface.py new file mode 100644 index 00000000..313d9567 --- /dev/null +++ b/.github/scripts/test_collect_worker_interface.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +import unittest + +import collect_worker_interface + + +class CollectWorkerInterfaceTests(unittest.TestCase): + def test_count_worker_matches_by_name_or_id(self): + workers_json = { + "workers": [ + {"name": "image-resize", "id": "worker-1"}, + {"name": "other", "id": "mcp"}, + {"name": "ignored"}, + ] + } + + self.assertEqual( + collect_worker_interface.count_worker_matches(workers_json, "image-resize"), + 1, + ) + self.assertEqual( + collect_worker_interface.count_worker_matches(workers_json, "mcp"), + 1, + ) + self.assertEqual( + collect_worker_interface.count_worker_matches(workers_json, "missing"), + 0, + ) + + def test_wait_for_worker_zero_wait_returns_latest_snapshot(self): + original_run_iii = collect_worker_interface.run_iii + try: + snapshot = {"workers": [{"name": "mcp"}]} + collect_worker_interface.run_iii = lambda _function_id, _payload: snapshot + + self.assertIs( + collect_worker_interface.wait_for_worker("mcp", 0), + snapshot, + ) + finally: + collect_worker_interface.run_iii = original_run_iii + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/_publish-registry.yml b/.github/workflows/_publish-registry.yml index 0b7954f8..b61e1d5a 100644 --- a/.github/workflows/_publish-registry.yml +++ b/.github/workflows/_publish-registry.yml @@ -62,13 +62,62 @@ jobs: - name: Install pyyaml run: pip install --quiet pyyaml + - name: Install iii CLI + if: inputs.collect_interface == true + run: | + set -euo pipefail + curl -fsSL https://install.iii.dev/iii/main/install.sh -o /tmp/install-iii.sh + sh /tmp/install-iii.sh + { + echo "$HOME/.local/bin" + echo "$HOME/.iii/bin" + } >> "$GITHUB_PATH" + export PATH="$HOME/.local/bin:$HOME/.iii/bin:$PATH" + iii --version + + - name: Start III engine + if: inputs.collect_interface == true + run: | + set -euo pipefail + printf 'workers: []\n' > config.yaml + iii --config config.yaml --no-update-check > iii-engine.log 2>&1 & + echo "$!" > iii-engine.pid + + for _ in {1..60}; do + if ! kill -0 "$(cat iii-engine.pid)" 2>/dev/null; then + echo "::error::iii engine exited before becoming ready" + cat iii-engine.log + exit 1 + fi + + if iii trigger --function-id='engine::workers::list' --payload='{}' >/tmp/iii-workers.json 2>/tmp/iii-trigger.err; then + cat /tmp/iii-workers.json + exit 0 + fi + + sleep 1 + done + + echo "::error::iii engine did not become ready" + cat /tmp/iii-trigger.err || true + cat iii-engine.log || true + exit 1 + + - name: Add local worker to III + if: inputs.collect_interface == true + env: + WORKER: ${{ inputs.worker }} + run: | + set -euo pipefail + iii worker add "./$WORKER" --force --reset-config --wait + - name: Collect worker interface if: inputs.collect_interface == true env: WORKER: ${{ inputs.worker }} ALLOW_MISSING_TRIGGERS: ${{ inputs.allow_missing_triggers }} run: | - args=("--worker" "$WORKER" "--out" "worker-interface.json") + args=("--worker" "$WORKER" "--out" "worker-interface.json" "--wait-seconds" "120") if [[ "$ALLOW_MISSING_TRIGGERS" == "true" ]]; then args+=("--allow-missing-triggers") fi @@ -148,3 +197,17 @@ jobs: echo "::error::publish failed with HTTP $http" exit 1 fi + + - name: Dump III logs + if: failure() && inputs.collect_interface == true + run: | + echo "::group::iii engine log" + tail -n 200 iii-engine.log || true + echo "::endgroup::" + + - name: Stop III engine + if: always() && inputs.collect_interface == true + run: | + if [[ -f iii-engine.pid ]]; then + kill "$(cat iii-engine.pid)" 2>/dev/null || true + fi diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 029341a4..a4daf68b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -197,4 +197,5 @@ jobs: tag: ${{ needs.setup.outputs.tag }} bin: ${{ needs.setup.outputs.bin }} image_tag: ${{ needs.container-build.outputs.image_tag }} + collect_interface: true secrets: inherit From 55ab094d7d04ee58efaa5a67aa85f6bf968df876 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Sat, 2 May 2026 19:45:10 -0300 Subject: [PATCH 09/13] ci: make trigger collection best effort --- .github/scripts/collect_worker_interface.py | 23 ++++++++++--------- .../scripts/test_collect_worker_interface.py | 17 ++++++++++++++ .github/workflows/_publish-registry.yml | 9 -------- 3 files changed, 29 insertions(+), 20 deletions(-) diff --git a/.github/scripts/collect_worker_interface.py b/.github/scripts/collect_worker_interface.py index f4b0f6c7..8b5edbcd 100644 --- a/.github/scripts/collect_worker_interface.py +++ b/.github/scripts/collect_worker_interface.py @@ -49,26 +49,27 @@ def wait_for_worker(worker_name: str, wait_seconds: int) -> dict[str, object]: return workers_json +def collect_triggers() -> dict[str, object] | None: + try: + return run_iii("engine::triggers::list", {"include_internal": True}) + except (subprocess.CalledProcessError, json.JSONDecodeError) as exc: + print( + f"::warning::could not collect triggers; publishing triggers=[]: {exc}", + file=sys.stderr, + ) + return None + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--worker", required=True) parser.add_argument("--out", default="worker-interface.json") - parser.add_argument("--allow-missing-triggers", action="store_true") parser.add_argument("--wait-seconds", type=int, default=0) args = parser.parse_args() workers_json = wait_for_worker(args.worker, args.wait_seconds) functions_json = run_iii("engine::functions::list", {"include_internal": True}) - - triggers_json = None - try: - triggers_json = run_iii("engine::triggers::list", {"include_internal": True}) - except (subprocess.CalledProcessError, json.JSONDecodeError) as exc: - if not args.allow_missing_triggers: - raise RuntimeError( - "could not collect triggers; confirm the engine exposes " - "`engine::triggers::list` or pass --allow-missing-triggers" - ) from exc + triggers_json = collect_triggers() interface = normalize_worker_interface( worker_name=args.worker, diff --git a/.github/scripts/test_collect_worker_interface.py b/.github/scripts/test_collect_worker_interface.py index 313d9567..b0e0688a 100644 --- a/.github/scripts/test_collect_worker_interface.py +++ b/.github/scripts/test_collect_worker_interface.py @@ -1,4 +1,7 @@ #!/usr/bin/env python3 +import contextlib +import io +import subprocess import unittest import collect_worker_interface @@ -40,6 +43,20 @@ def test_wait_for_worker_zero_wait_returns_latest_snapshot(self): finally: collect_worker_interface.run_iii = original_run_iii + def test_collect_triggers_returns_none_when_engine_listing_fails(self): + original_run_iii = collect_worker_interface.run_iii + try: + collect_worker_interface.run_iii = lambda _function_id, _payload: (_ for _ in ()).throw( + subprocess.CalledProcessError(1, ["iii"]) + ) + + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + self.assertIsNone(collect_worker_interface.collect_triggers()) + self.assertIn("publishing triggers=[]", stderr.getvalue()) + finally: + collect_worker_interface.run_iii = original_run_iii + if __name__ == "__main__": unittest.main() diff --git a/.github/workflows/_publish-registry.yml b/.github/workflows/_publish-registry.yml index b61e1d5a..f2cf2b3e 100644 --- a/.github/workflows/_publish-registry.yml +++ b/.github/workflows/_publish-registry.yml @@ -39,11 +39,6 @@ on: required: false type: boolean default: false - allow_missing_triggers: - description: 'Publish triggers=[] if engine trigger listing is unavailable' - required: false - type: boolean - default: true api_url: description: 'Workers registry base URL' required: false @@ -115,12 +110,8 @@ jobs: if: inputs.collect_interface == true env: WORKER: ${{ inputs.worker }} - ALLOW_MISSING_TRIGGERS: ${{ inputs.allow_missing_triggers }} run: | args=("--worker" "$WORKER" "--out" "worker-interface.json" "--wait-seconds" "120") - if [[ "$ALLOW_MISSING_TRIGGERS" == "true" ]]; then - args+=("--allow-missing-triggers") - fi python3 .github/scripts/collect_worker_interface.py "${args[@]}" - name: Assert worker interface was collected From 0468e7b1ae6209606a8aa6d6d46fb680e45be5da Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Sat, 2 May 2026 19:52:29 -0300 Subject: [PATCH 10/13] ci: always collect publish interface --- .github/workflows/_publish-registry.yml | 21 +++------------------ .github/workflows/release.yml | 1 - 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/.github/workflows/_publish-registry.yml b/.github/workflows/_publish-registry.yml index f2cf2b3e..9aa55ab5 100644 --- a/.github/workflows/_publish-registry.yml +++ b/.github/workflows/_publish-registry.yml @@ -34,11 +34,6 @@ on: required: false type: string default: '' - collect_interface: - description: 'Collect functions/triggers from a running iii engine' - required: false - type: boolean - default: false api_url: description: 'Workers registry base URL' required: false @@ -58,7 +53,6 @@ jobs: run: pip install --quiet pyyaml - name: Install iii CLI - if: inputs.collect_interface == true run: | set -euo pipefail curl -fsSL https://install.iii.dev/iii/main/install.sh -o /tmp/install-iii.sh @@ -71,7 +65,6 @@ jobs: iii --version - name: Start III engine - if: inputs.collect_interface == true run: | set -euo pipefail printf 'workers: []\n' > config.yaml @@ -99,7 +92,6 @@ jobs: exit 1 - name: Add local worker to III - if: inputs.collect_interface == true env: WORKER: ${{ inputs.worker }} run: | @@ -107,7 +99,6 @@ jobs: iii worker add "./$WORKER" --force --reset-config --wait - name: Collect worker interface - if: inputs.collect_interface == true env: WORKER: ${{ inputs.worker }} run: | @@ -115,20 +106,14 @@ jobs: python3 .github/scripts/collect_worker_interface.py "${args[@]}" - name: Assert worker interface was collected - if: inputs.collect_interface == true run: | python3 - <<'PY' import json data = json.load(open("worker-interface.json")) if not data.get("functions"): - raise SystemExit("collect_interface=true but no functions were collected") + raise SystemExit("no worker functions were collected") PY - - name: Create empty worker interface - if: inputs.collect_interface != true - run: | - printf '{ "functions": [], "triggers": [] }\n' > worker-interface.json - - name: Resolve binary artifacts if: inputs.deploy == 'binary' env: @@ -190,14 +175,14 @@ jobs: fi - name: Dump III logs - if: failure() && inputs.collect_interface == true + if: failure() run: | echo "::group::iii engine log" tail -n 200 iii-engine.log || true echo "::endgroup::" - name: Stop III engine - if: always() && inputs.collect_interface == true + if: always() run: | if [[ -f iii-engine.pid ]]; then kill "$(cat iii-engine.pid)" 2>/dev/null || true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a4daf68b..029341a4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -197,5 +197,4 @@ jobs: tag: ${{ needs.setup.outputs.tag }} bin: ${{ needs.setup.outputs.bin }} image_tag: ${{ needs.container-build.outputs.image_tag }} - collect_interface: true secrets: inherit From 9d789199f49e17780d8b423b1c4c5bb438e63eec Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Sat, 2 May 2026 20:09:09 -0300 Subject: [PATCH 11/13] test: remove collect interface script test --- .../scripts/test_collect_worker_interface.py | 62 ------------------- 1 file changed, 62 deletions(-) delete mode 100644 .github/scripts/test_collect_worker_interface.py diff --git a/.github/scripts/test_collect_worker_interface.py b/.github/scripts/test_collect_worker_interface.py deleted file mode 100644 index b0e0688a..00000000 --- a/.github/scripts/test_collect_worker_interface.py +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env python3 -import contextlib -import io -import subprocess -import unittest - -import collect_worker_interface - - -class CollectWorkerInterfaceTests(unittest.TestCase): - def test_count_worker_matches_by_name_or_id(self): - workers_json = { - "workers": [ - {"name": "image-resize", "id": "worker-1"}, - {"name": "other", "id": "mcp"}, - {"name": "ignored"}, - ] - } - - self.assertEqual( - collect_worker_interface.count_worker_matches(workers_json, "image-resize"), - 1, - ) - self.assertEqual( - collect_worker_interface.count_worker_matches(workers_json, "mcp"), - 1, - ) - self.assertEqual( - collect_worker_interface.count_worker_matches(workers_json, "missing"), - 0, - ) - - def test_wait_for_worker_zero_wait_returns_latest_snapshot(self): - original_run_iii = collect_worker_interface.run_iii - try: - snapshot = {"workers": [{"name": "mcp"}]} - collect_worker_interface.run_iii = lambda _function_id, _payload: snapshot - - self.assertIs( - collect_worker_interface.wait_for_worker("mcp", 0), - snapshot, - ) - finally: - collect_worker_interface.run_iii = original_run_iii - - def test_collect_triggers_returns_none_when_engine_listing_fails(self): - original_run_iii = collect_worker_interface.run_iii - try: - collect_worker_interface.run_iii = lambda _function_id, _payload: (_ for _ in ()).throw( - subprocess.CalledProcessError(1, ["iii"]) - ) - - stderr = io.StringIO() - with contextlib.redirect_stderr(stderr): - self.assertIsNone(collect_worker_interface.collect_triggers()) - self.assertIn("publishing triggers=[]", stderr.getvalue()) - finally: - collect_worker_interface.run_iii = original_run_iii - - -if __name__ == "__main__": - unittest.main() From 25158b42c91c7100f2c6d874348117e18da65fc6 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Sat, 2 May 2026 21:22:19 -0300 Subject: [PATCH 12/13] ci: start rust workers for publish interface collection --- .github/scripts/test_build_publish_payload.py | 259 ------------------ .../scripts/test_resolve_binary_artifacts.py | 32 --- .github/workflows/_publish-registry.yml | 76 ++++- 3 files changed, 74 insertions(+), 293 deletions(-) delete mode 100644 .github/scripts/test_build_publish_payload.py delete mode 100644 .github/scripts/test_resolve_binary_artifacts.py diff --git a/.github/scripts/test_build_publish_payload.py b/.github/scripts/test_build_publish_payload.py deleted file mode 100644 index 6bd3a4df..00000000 --- a/.github/scripts/test_build_publish_payload.py +++ /dev/null @@ -1,259 +0,0 @@ -import tempfile -import unittest -from pathlib import Path - -from build_publish_payload import ( - build_payload, - derive_registry_function_name, - normalize_dependencies, - normalize_worker_interface, -) - - -class PublishPayloadTests(unittest.TestCase): - def test_normalize_dependencies_accepts_map(self): - self.assertEqual( - normalize_dependencies({"helper-worker": "^1.0.0"}), - [{"name": "helper-worker", "version": "^1.0.0"}], - ) - - def test_normalize_dependencies_accepts_wire_list(self): - deps = [{"name": "helper-worker", "version": "^1.0.0"}] - self.assertEqual(normalize_dependencies(deps), deps) - - def test_normalize_dependencies_rejects_scalar(self): - with self.assertRaisesRegex(ValueError, "dependencies"): - normalize_dependencies("helper-worker") - - def test_derive_registry_function_name_prefers_metadata_registry_name(self): - self.assertEqual( - derive_registry_function_name( - "image_resize::resize", - {"registry_name": "resize_image", "name": "ignored"}, - ), - "resize_image", - ) - - def test_derive_registry_function_name_falls_back_to_final_function_segment(self): - self.assertEqual(derive_registry_function_name("image_resize::resize", {}), "resize") - - def test_normalize_worker_interface_converts_function_schema_fields(self): - workers = { - "workers": [ - { - "id": "worker-1", - "name": "image-resize", - "functions": ["image_resize::resize", "image_resize::ping"], - } - ] - } - functions = { - "functions": [ - { - "function_id": "image_resize::resize", - "description": "Resize an image", - "request_format": {"type": "object"}, - "response_format": {"type": "object"}, - "metadata": {"tags": ["image", "transform"]}, - }, - { - "function_id": "image_resize::ping", - "description": None, - "request_format": None, - "response_format": None, - "metadata": None, - }, - { - "function_id": "other::fn", - "description": "Not this worker", - "request_format": {"type": "object"}, - "response_format": None, - "metadata": {}, - }, - ] - } - triggers = { - "triggers": [ - { - "id": "t1", - "trigger_type": "http", - "function_id": "image_resize::resize", - "config": {"api_path": "/resize"}, - "metadata": {"public": True}, - }, - { - "id": "t2", - "trigger_type": "http", - "function_id": "other::fn", - "config": {"api_path": "/other"}, - }, - ] - } - - interface = normalize_worker_interface( - worker_name="image-resize", - workers_json=workers, - functions_json=functions, - triggers_json=triggers, - ) - - self.assertEqual( - interface["functions"], - [ - { - "name": "resize", - "description": "Resize an image", - "request_schema": {"type": "object"}, - "response_schema": {"type": "object"}, - "metadata": {"tags": ["image", "transform"]}, - }, - { - "name": "ping", - "description": "", - "request_schema": {}, - "response_schema": {}, - "metadata": {}, - }, - ], - ) - self.assertEqual( - interface["triggers"], - [ - { - "name": "resize", - "description": "", - "invocation_schema": {}, - "return_schema": {}, - "metadata": { - "public": True, - "engine_id": "t1", - "trigger_type": "http", - "function_id": "image_resize::resize", - "config": {"api_path": "/resize"}, - }, - } - ], - ) - - def test_normalize_worker_interface_rejects_missing_worker(self): - with self.assertRaisesRegex(ValueError, "expected exactly one worker"): - normalize_worker_interface( - worker_name="missing", - workers_json={"workers": []}, - functions_json={"functions": []}, - triggers_json={"triggers": []}, - ) - - def test_normalize_worker_interface_accepts_no_triggers_source(self): - interface = normalize_worker_interface( - worker_name="image-resize", - workers_json={ - "workers": [ - { - "id": "worker-1", - "name": "image-resize", - "functions": ["image_resize::resize"], - } - ] - }, - functions_json={ - "functions": [ - { - "function_id": "image_resize::resize", - "description": "Resize", - "request_format": None, - "response_format": None, - "metadata": None, - } - ] - }, - triggers_json=None, - ) - self.assertEqual(interface["triggers"], []) - - def test_build_binary_payload_has_registry_shape_and_binaries(self): - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) / "image-resize" - root.mkdir() - (root / "iii.worker.yaml").write_text( - "description: Resize images from a URL.\n" - "dependencies:\n" - " helper-worker: '^1.0.0'\n", - encoding="utf-8", - ) - (root / "README.md").write_text("# image-resize\n", encoding="utf-8") - (root / "config.yaml").write_text("{}\n", encoding="utf-8") - - payload = build_payload( - repo_root=Path(tmp), - worker="image-resize", - version="0.1.0", - registry_tag="latest", - deploy="binary", - repo_url="https://github.com/example/image-resize", - interface={ - "functions": [ - { - "name": "resize", - "description": "Resize", - "request_schema": {"type": "object"}, - "response_schema": None, - "metadata": {}, - } - ], - "triggers": [], - }, - binaries={ - "x86_64-unknown-linux-gnu": { - "url": "https://example.com/releases/image-resize-x86_64-unknown-linux-gnu.tar.gz", - "sha256": "b" * 64, - } - }, - image_tag="", - ) - - self.assertEqual(payload["worker_name"], "image-resize") - self.assertEqual(payload["version"], "0.1.0") - self.assertEqual(payload["tag"], "latest") - self.assertEqual(payload["type"], "binary") - self.assertEqual(payload["readme"], "# image-resize\n") - self.assertEqual(payload["repo"], "https://github.com/example/image-resize") - self.assertEqual(payload["description"], "Resize images from a URL.") - self.assertEqual( - payload["dependencies"], - [{"name": "helper-worker", "version": "^1.0.0"}], - ) - self.assertEqual(payload["config"], {}) - self.assertEqual(payload["functions"][0]["name"], "resize") - self.assertIn("request_schema", payload["functions"][0]) - self.assertIn("response_schema", payload["functions"][0]) - self.assertEqual(payload["functions"][0]["response_schema"], {}) - self.assertEqual(payload["triggers"], []) - self.assertIn("binaries", payload) - self.assertNotIn("image_tag", payload) - - def test_build_image_payload_has_image_tag(self): - with tempfile.TemporaryDirectory() as tmp: - root = Path(tmp) / "hello-worker" - root.mkdir() - (root / "iii.worker.yaml").write_text("description: Demo worker\n", encoding="utf-8") - - payload = build_payload( - repo_root=Path(tmp), - worker="hello-worker", - version="1.0.0", - registry_tag="latest", - deploy="image", - repo_url="https://github.com/example/workers", - interface={"functions": [], "triggers": []}, - binaries={}, - image_tag="ghcr.io/example/hello-worker:1.0.0", - ) - - self.assertEqual(payload["type"], "image") - self.assertEqual(payload["image_tag"], "ghcr.io/example/hello-worker:1.0.0") - self.assertNotIn("binaries", payload) - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/scripts/test_resolve_binary_artifacts.py b/.github/scripts/test_resolve_binary_artifacts.py deleted file mode 100644 index ca3b9d86..00000000 --- a/.github/scripts/test_resolve_binary_artifacts.py +++ /dev/null @@ -1,32 +0,0 @@ -import unittest - -from resolve_binary_artifacts import build_binary_artifact_map - - -class ResolveBinaryArtifactsTests(unittest.TestCase): - def test_builds_binary_urls_and_sha_values(self): - checksums = { - "https://github.com/example/workers/releases/download/image-resize/v0.1.0/image-resize-x86_64-unknown-linux-gnu.sha256": "b" * 64 - } - - result = build_binary_artifact_map( - repo_url="https://github.com/example/workers", - tag="image-resize/v0.1.0", - bin_name="image-resize", - targets=["x86_64-unknown-linux-gnu"], - read_checksum=lambda url: checksums[url], - ) - - self.assertEqual( - result, - { - "x86_64-unknown-linux-gnu": { - "url": "https://github.com/example/workers/releases/download/image-resize/v0.1.0/image-resize-x86_64-unknown-linux-gnu.tar.gz", - "sha256": "b" * 64, - } - }, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/.github/workflows/_publish-registry.yml b/.github/workflows/_publish-registry.yml index 9aa55ab5..9a0b537a 100644 --- a/.github/workflows/_publish-registry.yml +++ b/.github/workflows/_publish-registry.yml @@ -91,12 +91,72 @@ jobs: cat iii-engine.log || true exit 1 - - name: Add local worker to III + - name: Start local worker for interface collection env: WORKER: ${{ inputs.worker }} + BIN: ${{ inputs.bin }} run: | set -euo pipefail - iii worker add "./$WORKER" --force --reset-config --wait + mode=$(python3 - <<'PY' + import os + import yaml + + worker = os.environ["WORKER"] + manifest_path = f"{worker}/iii.worker.yaml" + manifest = yaml.safe_load(open(manifest_path, encoding="utf-8").read()) or {} + scripts = manifest.get("scripts") or {} + runtime = manifest.get("runtime") or {} + has_scripts_start = bool(str(scripts.get("start") or "").strip()) + has_runtime = bool(runtime.get("kind") or runtime.get("language")) + language = str(manifest.get("language") or "").strip().lower() + + if has_scripts_start or has_runtime: + print("iii-add") + elif language == "rust": + print("cargo-run") + else: + print("unsupported") + PY + ) + + case "$mode" in + iii-add) + iii worker add "./$WORKER" --force --reset-config --wait + ;; + cargo-run) + if [[ ! -f "$WORKER/Cargo.toml" ]]; then + echo "::error::$WORKER is a Rust worker but $WORKER/Cargo.toml was not found" + exit 1 + fi + + worker_log="worker-$WORKER.log" + pushd "$WORKER" >/dev/null + cargo_args=(run) + if [[ -n "${BIN:-}" ]]; then + cargo_args+=(--bin "$BIN") + fi + cargo_args+=(--) + if [[ "$WORKER" == "mcp" ]]; then + cargo_args+=(--no-stdio) + fi + + echo "Starting local worker with: (cd $WORKER && ${cargo_args[*]})" + "${cargo_args[@]}" > "../$worker_log" 2>&1 & + echo "$!" > ../worker.pid + popd >/dev/null + + sleep 2 + if ! kill -0 "$(cat worker.pid)" 2>/dev/null; then + echo "::error::$WORKER exited before interface collection" + cat "$worker_log" || true + exit 1 + fi + ;; + *) + echo "::error::$WORKER iii.worker.yaml has no local runtime/scripts.start and is not a Rust worker; cannot collect interface" + exit 1 + ;; + esac - name: Collect worker interface env: @@ -180,6 +240,18 @@ jobs: echo "::group::iii engine log" tail -n 200 iii-engine.log || true echo "::endgroup::" + if [[ -f worker-${{ inputs.worker }}.log ]]; then + echo "::group::worker log" + tail -n 200 worker-${{ inputs.worker }}.log || true + echo "::endgroup::" + fi + + - name: Stop local worker + if: always() + run: | + if [[ -f worker.pid ]]; then + kill "$(cat worker.pid)" 2>/dev/null || true + fi - name: Stop III engine if: always() From 9ef41a4001f80a8cfb72783029cdee2041527ba9 Mon Sep 17 00:00:00 2001 From: Guilherme Beira Date: Sun, 3 May 2026 11:00:39 -0300 Subject: [PATCH 13/13] fix: address coderabbit review feedback - normalize_dependencies now converts list-form string entries to {name, version} dicts and validates each item - normalize_worker_interface fails fast when worker_function_ids reference missing function details, instead of publishing blank schemas - run_iii has a 30s subprocess timeout, and collect_triggers handles TimeoutExpired alongside the existing soft-fail paths - pin the iii CLI installer to v0.11.5 via the VERSION env var so the publish job no longer tracks main --- .github/scripts/build_publish_payload.py | 21 +++++++++++++++++++-- .github/scripts/collect_worker_interface.py | 7 ++++++- .github/workflows/_publish-registry.yml | 2 ++ 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/.github/scripts/build_publish_payload.py b/.github/scripts/build_publish_payload.py index c174e98a..8aad641c 100644 --- a/.github/scripts/build_publish_payload.py +++ b/.github/scripts/build_publish_payload.py @@ -13,7 +13,17 @@ def normalize_dependencies(raw_deps: Any) -> list[dict[str, Any]]: if isinstance(raw_deps, dict): return [{"name": name, "version": version} for name, version in raw_deps.items()] if isinstance(raw_deps, list): - return raw_deps + normalized: list[dict[str, Any]] = [] + for dep in raw_deps: + if isinstance(dep, str): + normalized.append({"name": dep, "version": ""}) + elif isinstance(dep, dict) and isinstance(dep.get("name"), str): + normalized.append({"name": dep["name"], "version": dep.get("version") or ""}) + else: + raise ValueError( + "dependency list entries must be strings or {name, version} objects" + ) + return normalized raise ValueError(f"`dependencies` must be a map or list, got {type(raw_deps).__name__}") @@ -138,9 +148,16 @@ def normalize_worker_interface( if f.get("function_id") } + missing_function_ids = [fid for fid in worker_function_ids if fid not in functions_by_id] + if missing_function_ids: + raise ValueError( + "missing function details for worker functions: " + + ", ".join(str(fid) for fid in missing_function_ids) + ) + functions = [] for function_id in worker_function_ids: - details = functions_by_id.get(function_id, {}) + details = functions_by_id[function_id] metadata = details.get("metadata") or {} functions.append( { diff --git a/.github/scripts/collect_worker_interface.py b/.github/scripts/collect_worker_interface.py index 8b5edbcd..50f0c2bb 100644 --- a/.github/scripts/collect_worker_interface.py +++ b/.github/scripts/collect_worker_interface.py @@ -34,6 +34,7 @@ def run_iii(function_id: str, payload: dict[str, object]) -> dict[str, object]: check=True, text=True, capture_output=True, + timeout=30, ) return json.loads(completed.stdout) @@ -52,7 +53,11 @@ def wait_for_worker(worker_name: str, wait_seconds: int) -> dict[str, object]: def collect_triggers() -> dict[str, object] | None: try: return run_iii("engine::triggers::list", {"include_internal": True}) - except (subprocess.CalledProcessError, json.JSONDecodeError) as exc: + except ( + subprocess.CalledProcessError, + subprocess.TimeoutExpired, + json.JSONDecodeError, + ) as exc: print( f"::warning::could not collect triggers; publishing triggers=[]: {exc}", file=sys.stderr, diff --git a/.github/workflows/_publish-registry.yml b/.github/workflows/_publish-registry.yml index 9a0b537a..20c4dc81 100644 --- a/.github/workflows/_publish-registry.yml +++ b/.github/workflows/_publish-registry.yml @@ -53,6 +53,8 @@ jobs: run: pip install --quiet pyyaml - name: Install iii CLI + env: + VERSION: '0.11.5' run: | set -euo pipefail curl -fsSL https://install.iii.dev/iii/main/install.sh -o /tmp/install-iii.sh