From 54ac2c925a55d44a458b0deaa0a416b2b36144dc Mon Sep 17 00:00:00 2001 From: Dev Punia Date: Sat, 30 May 2026 12:33:04 +0530 Subject: [PATCH 01/13] chore(matrix): scaffold install/update matrix harness --- package.json | 5 ++++- test-matrix/README.md | 12 ++++++++++++ test-matrix/harness-ext/noop.js | 3 +++ test-matrix/harness-ext/package.json | 11 +++++++++++ tests/matrix/__init__.py | 0 tests/matrix/fixtures/.gitkeep | 0 6 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 test-matrix/README.md create mode 100644 test-matrix/harness-ext/noop.js create mode 100644 test-matrix/harness-ext/package.json create mode 100644 tests/matrix/__init__.py create mode 100644 tests/matrix/fixtures/.gitkeep diff --git a/package.json b/package.json index 55e3fd324..03e43c8b1 100644 --- a/package.json +++ b/package.json @@ -1209,7 +1209,10 @@ "compile": "tsc -p ./", "docker:deploy": "./docker-setup/deploy.sh", "docker:logs": "cd docker-setup && docker compose logs -f || docker-compose logs -f", - "docker:stop": "cd docker-setup && docker compose down || docker-compose down" + "docker:stop": "cd docker-setup && docker compose down || docker-compose down", + "matrix:vscode": "node test-matrix/vscode-cell.mjs", + "matrix:aggregate": "python3 test-matrix/aggregate.py", + "matrix:test-aggregate": "python3 -m pytest tests/matrix/ -q" }, "devDependencies": { "@istanbuljs/nyc-config-typescript": "^1.0.2", diff --git a/test-matrix/README.md b/test-matrix/README.md new file mode 100644 index 000000000..b1a28cfb6 --- /dev/null +++ b/test-matrix/README.md @@ -0,0 +1,12 @@ +# Install / Update Test Matrix (P1) + +Each cell emits a RESULT_JSON (schema in docs/superpowers/plans/2026-05-30-extension-install-update-matrix-p1.md). + +- `vscode-cell.mjs` — real VSCode/Insiders via @vscode/test-electron (fresh + upgrade) +- `codeserver-cell.sh` — wraps docker-setup/vsix-smoke.sh +- `aggregate.py` — renders the install + update matrices, Slack payload, and the blocking gate exit code + +Run one cell locally: +npm run build && npm run compile +bash test-matrix/setup-dbt-env.sh +node test-matrix/vscode-cell.mjs --mode fresh --target latest --out /tmp/r.json diff --git a/test-matrix/harness-ext/noop.js b/test-matrix/harness-ext/noop.js new file mode 100644 index 000000000..6e901df8d --- /dev/null +++ b/test-matrix/harness-ext/noop.js @@ -0,0 +1,3 @@ +function activate() {} +function deactivate() {} +module.exports = { activate, deactivate }; diff --git a/test-matrix/harness-ext/package.json b/test-matrix/harness-ext/package.json new file mode 100644 index 000000000..b72f9a91b --- /dev/null +++ b/test-matrix/harness-ext/package.json @@ -0,0 +1,11 @@ +{ + "name": "matrix-harness", + "publisher": "altimate-internal", + "version": "0.0.0", + "engines": { + "vscode": "^1.95.0" + }, + "main": "./noop.js", + "activationEvents": [], + "contributes": {} +} diff --git a/tests/matrix/__init__.py b/tests/matrix/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/matrix/fixtures/.gitkeep b/tests/matrix/fixtures/.gitkeep new file mode 100644 index 000000000..e69de29bb From 88eeff2428c6f8ff6c8e4c7a2e800104ba530e1b Mon Sep 17 00:00:00 2001 From: Dev Punia Date: Sat, 30 May 2026 12:39:21 +0530 Subject: [PATCH 02/13] feat(matrix): aggregator renders install/update matrices + blocking gate --- test-matrix/aggregate.py | 167 +++++++++++++++++++++++++++++++++ tests/matrix/test_aggregate.py | 73 ++++++++++++++ 2 files changed, 240 insertions(+) create mode 100644 test-matrix/aggregate.py create mode 100644 tests/matrix/test_aggregate.py diff --git a/test-matrix/aggregate.py b/test-matrix/aggregate.py new file mode 100644 index 000000000..87312fb57 --- /dev/null +++ b/test-matrix/aggregate.py @@ -0,0 +1,167 @@ +#!/usr/bin/env python3 +"""Aggregate per-cell RESULT_JSON files into the install + update matrices, +a Slack payload, and a blocking-gate exit code. + +Usage: + python3 aggregate.py --results-dir --out-dir [--target ] [--trigger pr|schedule|release] +""" +from __future__ import annotations + +import argparse +import glob +import json +import os +import sys + +# Only these runtimes can fail a release. Everything else is informational. +BLOCKING_RUNTIMES = {"vscode"} + +# Stable display order; runtimes not listed are appended alphabetically. +RUNTIME_ORDER = ["vscode", "vscode-insiders", "cursor", "windsurf", "kiro", "code-server"] +OS_ORDER = ["linux", "windows", "macos"] + + +def _cell_symbol(cell: dict) -> str: + if cell.get("status") == "skip": + return "⏭️" + if cell.get("status") == "pass": + return "✅" + # failed + return "❌" if cell.get("runtime") in BLOCKING_RUNTIMES else "⚠️" + + +def _runtime_sort_key(rt: str): + return (RUNTIME_ORDER.index(rt) if rt in RUNTIME_ORDER else len(RUNTIME_ORDER), rt) + + +def _os_sort_key(os_name: str): + return (OS_ORDER.index(os_name) if os_name in OS_ORDER else len(OS_ORDER), os_name) + + +def build_matrices(results: list[dict]) -> dict: + install = [r for r in results if r.get("scenario") == "fresh"] + upgrade = [r for r in results if r.get("scenario") == "upgrade"] + + has_blocking_failure = any( + r.get("status") == "fail" and r.get("runtime") in BLOCKING_RUNTIMES + for r in results + ) + + install_md = _render_install(install) + update_md = _render_update(upgrade) + slack_blocks = _render_slack(results, has_blocking_failure) + + return { + "install_md": install_md, + "update_md": update_md, + "slack_blocks": slack_blocks, + "has_blocking_failure": has_blocking_failure, + } + + +def _render_install(cells: list[dict]) -> str: + runtimes = sorted({c["runtime"] for c in cells}, key=_runtime_sort_key) + oses = sorted({c["os"] for c in cells}, key=_os_sort_key) + by = {(c["runtime"], c["os"]): c for c in cells} + + lines = ["### Install matrix (fresh install of target)", ""] + lines.append("| Runtime | " + " | ".join(oses) + " |") + lines.append("|---|" + "---|" * len(oses)) + for rt in runtimes: + row = [rt] + for os_name in oses: + cell = by.get((rt, os_name)) + row.append(_cell_symbol(cell) if cell else "—") + lines.append("| " + " | ".join(row) + " |") + lines.append("") + return "\n".join(lines) + + +def _render_update(cells: list[dict]) -> str: + runtimes = sorted({c["runtime"] for c in cells}, key=_runtime_sort_key) + baselines = sorted({c.get("from") for c in cells if c.get("from")}) + by = {(c["runtime"], c.get("from")): c for c in cells} + + lines = ["### Update matrix (upgrade baseline → target)", ""] + if not cells: + lines.append("_no upgrade cells in this run_") + lines.append("") + return "\n".join(lines) + lines.append("| Runtime | " + " | ".join(f"from {b}" for b in baselines) + " |") + lines.append("|---|" + "---|" * len(baselines)) + for rt in runtimes: + row = [rt] + for b in baselines: + cell = by.get((rt, b)) + row.append(_cell_symbol(cell) if cell else "—") + lines.append("| " + " | ".join(row) + " |") + lines.append("") + return "\n".join(lines) + + +def _render_slack(results: list[dict], has_blocking_failure: bool) -> list[dict]: + total = len(results) + passed = sum(1 for r in results if r.get("status") == "pass") + failed = [r for r in results if r.get("status") == "fail"] + skipped = [r for r in results if r.get("status") == "skip"] + headline = "❌ Install/Update matrix: BLOCKING failure" if has_blocking_failure else ( + "⚠️ Install/Update matrix: non-blocking issues" if failed else "✅ Install/Update matrix: all green" + ) + detail = f"{passed}/{total} cells passed" + if failed: + detail += "\nFailures:\n" + "\n".join( + f"• {r.get('runtime','?')}/{r.get('os','?')}/{r.get('scenario','?')}" + + (f" (from {r['from']})" if r.get("from") else "") + + f": {r.get('reason') or 'failed'}" + for r in failed + ) + if skipped: + detail += "\nSkipped:\n" + "\n".join( + f"• {r.get('runtime','?')}/{r.get('os','?')}: {r.get('reason') or 'skipped'}" for r in skipped + ) + return [ + {"type": "section", "text": {"type": "mrkdwn", "text": f"*{headline}*"}}, + {"type": "section", "text": {"type": "mrkdwn", "text": detail}}, + ] + + +def _load_results(results_dir: str) -> list[dict]: + out = [] + for path in sorted(glob.glob(os.path.join(results_dir, "**", "*.json"), recursive=True)): + with open(path) as f: + out.append(json.load(f)) + return out + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--results-dir", required=True) + ap.add_argument("--out-dir", required=True) + ap.add_argument("--target", default="") + ap.add_argument("--trigger", default="manual") + args = ap.parse_args() + + results = _load_results(args.results_dir) + out = build_matrices(results) + os.makedirs(args.out_dir, exist_ok=True) + + header = f"## VSIX Install + Update Matrix — target `{args.target or 'latest'}` ({args.trigger})\n\n" + combined = header + out["install_md"] + "\n" + out["update_md"] + with open(os.path.join(args.out_dir, "install-matrix.md"), "w") as f: + f.write(out["install_md"]) + with open(os.path.join(args.out_dir, "update-matrix.md"), "w") as f: + f.write(out["update_md"]) + with open(os.path.join(args.out_dir, "matrix.md"), "w") as f: + f.write(combined) + with open(os.path.join(args.out_dir, "slack.json"), "w") as f: + json.dump({"blocks": out["slack_blocks"]}, f, indent=2) + + print(combined) + if out["has_blocking_failure"]: + print("::error::Blocking-lane cell(s) failed — see matrix above") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/matrix/test_aggregate.py b/tests/matrix/test_aggregate.py new file mode 100644 index 000000000..7e4ff1686 --- /dev/null +++ b/tests/matrix/test_aggregate.py @@ -0,0 +1,73 @@ +import importlib.util +import pathlib + +_spec = importlib.util.spec_from_file_location( + "aggregate", pathlib.Path(__file__).resolve().parents[2] / "test-matrix" / "aggregate.py" +) +aggregate = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(aggregate) + + +def _cell(**kw): + base = dict( + runtime="vscode", os="linux", scenario="fresh", **{"from": None}, + to="0.61.5", install_ok=True, deps_resolved={}, activation_ok=True, + dbt_flow_ok=True, status="pass", reason="", duration_s=10, log_artifact="x.log", + ) + base.update(kw) + return base + + +def test_blocking_failure_when_vscode_fails(): + results = [_cell(status="fail", activation_ok=False, reason="no activate")] + out = aggregate.build_matrices(results) + assert out["has_blocking_failure"] is True + + +def test_no_blocking_failure_when_only_codeserver_fails(): + results = [ + _cell(), + _cell(runtime="code-server", status="fail", dbt_flow_ok=False, reason="boom"), + ] + out = aggregate.build_matrices(results) + assert out["has_blocking_failure"] is False + + +def test_insiders_is_non_blocking(): + results = [_cell(runtime="vscode-insiders", status="fail", reason="upstream churn")] + out = aggregate.build_matrices(results) + assert out["has_blocking_failure"] is False + + +def test_install_matrix_has_a_row_per_runtime_os(): + results = [ + _cell(os="linux"), _cell(os="windows"), _cell(os="macos"), + _cell(runtime="code-server", os="linux"), + ] + md = aggregate.build_matrices(results)["install_md"] + assert "vscode" in md and "code-server" in md + assert "linux" in md and "windows" in md and "macos" in md + assert "✅" in md + + +def test_update_matrix_groups_by_baseline(): + results = [ + _cell(scenario="upgrade", **{"from": "0.61.4"}), + _cell(scenario="upgrade", **{"from": "0.55.5"}, status="fail", dbt_flow_ok=False), + ] + md = aggregate.build_matrices(results)["update_md"] + assert "0.61.4" in md and "0.55.5" in md + assert "✅" in md and "❌" in md + + +def test_fork_failure_renders_warning_not_cross(): + # In P1 there are no forks, but a non-blocking runtime failure must render as ⚠️ + results = [_cell(runtime="code-server", status="fail", reason="x")] + md = aggregate.build_matrices(results)["install_md"] + assert "⚠️" in md + + +def test_skip_cell_renders_skip_symbol(): + results = [_cell(status="skip", reason="not applicable")] + md = aggregate.build_matrices(results)["install_md"] + assert "⏭️" in md From b889c8df715380ed2d2feaadd4a0649b848ab7dc Mon Sep 17 00:00:00 2001 From: Dev Punia Date: Sat, 30 May 2026 12:50:09 +0530 Subject: [PATCH 03/13] test(matrix): cover aggregate CLI files + exit code --- tests/matrix/test_aggregate.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/matrix/test_aggregate.py b/tests/matrix/test_aggregate.py index 7e4ff1686..06950d7c0 100644 --- a/tests/matrix/test_aggregate.py +++ b/tests/matrix/test_aggregate.py @@ -1,5 +1,8 @@ import importlib.util +import json import pathlib +import subprocess +import sys _spec = importlib.util.spec_from_file_location( "aggregate", pathlib.Path(__file__).resolve().parents[2] / "test-matrix" / "aggregate.py" @@ -71,3 +74,26 @@ def test_skip_cell_renders_skip_symbol(): results = [_cell(status="skip", reason="not applicable")] md = aggregate.build_matrices(results)["install_md"] assert "⏭️" in md + + +def test_cli_writes_files_and_exit_code(tmp_path): + rdir = tmp_path / "results" + rdir.mkdir() + (rdir / "a.json").write_text(json.dumps(_cell())) + (rdir / "b.json").write_text( + json.dumps( + _cell(runtime="vscode", os="windows", status="fail", activation_ok=False, reason="x") + ) + ) + odir = tmp_path / "out" + root = pathlib.Path(__file__).resolve().parents[2] + proc = subprocess.run( + [sys.executable, str(root / "test-matrix" / "aggregate.py"), + "--results-dir", str(rdir), "--out-dir", str(odir), "--target", "0.61.5"], + capture_output=True, text=True, + ) + assert proc.returncode == 1 # a blocking vscode cell failed + assert (odir / "matrix.md").exists() + assert (odir / "slack.json").exists() + slack = json.loads((odir / "slack.json").read_text()) + assert "blocks" in slack and len(slack["blocks"]) == 2 From 779dcec03c52d34986fef94be7fbb2559acf4db0 Mon Sep 17 00:00:00 2001 From: Dev Punia Date: Sat, 30 May 2026 12:52:21 +0530 Subject: [PATCH 04/13] feat(matrix): hermetic dbt-duckdb env setup for the fixture --- .gitignore | 3 +++ test-matrix/setup-dbt-env.sh | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100755 test-matrix/setup-dbt-env.sh diff --git a/.gitignore b/.gitignore index 667c53f9b..2edbff47b 100755 --- a/.gitignore +++ b/.gitignore @@ -50,3 +50,6 @@ docker-setup/.env # GitHub issues dashboard cache monitoring/github_issues/.cache/ + +# Install/update matrix local env +.matrix-venv/ diff --git a/test-matrix/setup-dbt-env.sh b/test-matrix/setup-dbt-env.sh new file mode 100755 index 000000000..8c9044548 --- /dev/null +++ b/test-matrix/setup-dbt-env.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Creates a hermetic dbt env for the dbt-core-sample-duckdb fixture and prints, +# on the LAST line, the python interpreter path (for dbt.dbtPythonPathOverride). +# Also writes a profiles dir with a tmp duckdb path and exports DBT_PROFILES_DIR. +# +# eval "$(bash test-matrix/setup-dbt-env.sh)" # exports PY_INTERP + DBT_PROFILES_DIR +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +FIXTURE="$REPO_ROOT/test-fixtures/dbt-core-sample-duckdb" +VENV="${MATRIX_VENV:-$REPO_ROOT/.matrix-venv}" +PROFILES_DIR="${MATRIX_PROFILES_DIR:-$(mktemp -d)}" + +python3 -m venv "$VENV" +# shellcheck disable=SC1091 +"$VENV/bin/pip" install --quiet --upgrade pip +"$VENV/bin/pip" install --quiet "dbt-core==1.9.6" "dbt-duckdb==1.9.3" + +# Hermetic profiles.yml: same profile name as the fixture, duckdb at a tmp path. +cat > "$PROFILES_DIR/profiles.yml" </dev/null 2>&1 || true ) + +# Emit shell-eval-able exports. +echo "export PY_INTERP='$VENV/bin/python'" +echo "export DBT_PROFILES_DIR='$PROFILES_DIR'" From 93ff0d8c422516b922138293a19896ad915a1353 Mon Sep 17 00:00:00 2001 From: Dev Punia Date: Sat, 30 May 2026 12:53:07 +0530 Subject: [PATCH 05/13] feat(matrix): in-host activation + dbt-init assertion suite --- src/test/matrix/activation.test.ts | 98 ++++++++++++++++++++++++++++++ src/test/matrix/index.ts | 19 ++++++ 2 files changed, 117 insertions(+) create mode 100644 src/test/matrix/activation.test.ts create mode 100644 src/test/matrix/index.ts diff --git a/src/test/matrix/activation.test.ts b/src/test/matrix/activation.test.ts new file mode 100644 index 000000000..9f40b743b --- /dev/null +++ b/src/test/matrix/activation.test.ts @@ -0,0 +1,98 @@ +import * as assert from "assert"; +import * as fs from "fs"; +import * as path from "path"; +import * as vscode from "vscode"; + +const EXTENSION_ID = "innoverio.vscode-dbt-power-user"; +// A command this extension contributes (proves contributions loaded), from package.json contributes.commands. +const STABLE_COMMAND = "dbtPowerUser.openInsights"; +// Success/failure markers emitted to the file-backed "Log - dbt" LogOutputChannel and the exthost log. +const INIT_OK = "Initialized dbt project"; +const INIT_FAIL = "Unable to register dbt project"; + +function sleep(ms: number) { + return new Promise((r) => setTimeout(r, ms)); +} + +function readAllLogs(uddDir: string): string { + // VSCode writes LogOutputChannels + console output under /logs/**. + const out: string[] = []; + const stack = [path.join(uddDir, "logs")]; + while (stack.length) { + const dir = stack.pop()!; + let entries: fs.Dirent[] = []; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + continue; + } + for (const e of entries) { + const p = path.join(dir, e.name); + if (e.isDirectory()) { + stack.push(p); + } else if (e.name.endsWith(".log")) { + try { + out.push(fs.readFileSync(p, "utf8")); + } catch { + /* ignore unreadable rotating log */ + } + } + } + } + return out.join("\n"); +} + +suite("Matrix: installed VSIX activation + dbt project init", function () { + this.timeout(120_000); + + test("extension is installed and activates", async function () { + const ext = vscode.extensions.getExtension(EXTENSION_ID); + assert.ok( + ext, + `${EXTENSION_ID} should be installed in the test extensions-dir`, + ); + await ext!.activate(); + assert.strictEqual(ext!.isActive, true, "extension should be active"); + }); + + test("contributed command is registered", async function () { + const cmds = await vscode.commands.getCommands(true); + assert.ok( + cmds.includes(STABLE_COMMAND), + `command ${STABLE_COMMAND} should be registered`, + ); + }); + + test("dbt fixture workspace is open", function () { + const folders = vscode.workspace.workspaceFolders ?? []; + assert.ok(folders.length > 0, "a workspace folder should be open"); + assert.ok( + folders.some((f) => + fs.existsSync(path.join(f.uri.fsPath, "dbt_project.yml")), + ), + "the open workspace should contain dbt_project.yml", + ); + }); + + test("dbt project initializes (log shows 'Initialized dbt project')", async function () { + const uddDir = process.env.MATRIX_UDD; + assert.ok(uddDir, "MATRIX_UDD env must point at the --user-data-dir"); + const deadline = Date.now() + 90_000; + let logs = ""; + while (Date.now() < deadline) { + logs = readAllLogs(uddDir!); + if (logs.includes(INIT_FAIL)) { + assert.fail( + `dbt project registration failed: found '${INIT_FAIL}' in logs`, + ); + } + if (logs.includes(INIT_OK)) { + return; // success + } + await sleep(2000); + } + assert.fail( + `did not observe '${INIT_OK}' within 90s (dbt project did not initialize)`, + ); + }); +}); diff --git a/src/test/matrix/index.ts b/src/test/matrix/index.ts new file mode 100644 index 000000000..f5e6c592d --- /dev/null +++ b/src/test/matrix/index.ts @@ -0,0 +1,19 @@ +import { glob } from "glob"; +import Mocha from "mocha"; +import * as path from "path"; + +export async function run(): Promise { + const mocha = new Mocha({ ui: "tdd", timeout: 120_000, color: true }); + const testsRoot = path.resolve(__dirname); + const files = await glob("**/*.test.js", { cwd: testsRoot }); + for (const file of files) { + mocha.addFile(path.resolve(testsRoot, file)); + } + return new Promise((resolve, reject) => { + mocha.run((failures) => + failures > 0 + ? reject(new Error(`${failures} test(s) failed.`)) + : resolve(), + ); + }); +} From 569f7c48a5e093d6082d483eecc18fb9d9d548b5 Mon Sep 17 00:00:00 2001 From: Dev Punia Date: Sat, 30 May 2026 12:54:19 +0530 Subject: [PATCH 06/13] feat(matrix): vscode cell driver (fresh + upgrade install + dbt activation) --- test-matrix/vscode-cell.mjs | 133 ++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100644 test-matrix/vscode-cell.mjs diff --git a/test-matrix/vscode-cell.mjs b/test-matrix/vscode-cell.mjs new file mode 100644 index 000000000..26bd2a2fd --- /dev/null +++ b/test-matrix/vscode-cell.mjs @@ -0,0 +1,133 @@ +#!/usr/bin/env node +// Driver for one VSCode/Insiders matrix cell. Downloads a real editor, installs +// the extension (+ dependencies) via the editor CLI, launches it headless against +// the dbt-core-sample-duckdb fixture, and writes a RESULT_JSON. +// +// Usage: +// node test-matrix/vscode-cell.mjs --mode fresh|upgrade --target \ +// [--from ] [--vscode-version stable|insiders|x.y.z] --out +import { execFileSync } from "node:child_process"; +import { existsSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + downloadAndUnzipVSCode, + resolveCliArgsFromVSCodeExecutablePath, + runTests, +} from "@vscode/test-electron"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const EXTENSION_ID = "innoverio.vscode-dbt-power-user"; +const DEPS = ["samuelcolvin.jinjahtml", "ms-python.python", "altimateai.vscode-altimate-mcp-server"]; + +function arg(name, def = undefined) { + const i = process.argv.indexOf(`--${name}`); + if (i === -1) return def; + const v = process.argv[i + 1]; + return v && !v.startsWith("--") ? v : true; +} + +function osLabel() { + if (process.platform === "darwin") return "macos"; + if (process.platform === "win32") return "windows"; + return "linux"; +} + +async function main() { + const mode = arg("mode", "fresh"); + const target = arg("target", "latest"); // vsix path or "latest" + const fromVersion = arg("from", null); + const vscodeVersion = arg("vscode-version", "stable"); // stable | insiders | x.y.z + const outPath = resolve(arg("out", "/tmp/result.json")); + const repoRoot = resolve(join(HERE, "..")); + const fixture = join(repoRoot, "test-fixtures", "dbt-core-sample-duckdb"); + const runtime = vscodeVersion === "insiders" ? "vscode-insiders" : "vscode"; + + const result = { + runtime, os: osLabel(), scenario: mode, from: fromVersion || null, + to: target === "latest" ? "latest" : "pr-build", install_ok: false, + deps_resolved: {}, activation_ok: false, dbt_flow_ok: false, + status: "fail", reason: "", duration_s: 0, log_artifact: outPath, + }; + const started = Date.now(); + + const extDir = mkdtempSync(join(tmpdir(), "matrix-ext-")); + const uddDir = mkdtempSync(join(tmpdir(), "matrix-udd-")); + + try { + // 1. Download the editor + resolve its CLI. + const exe = await downloadAndUnzipVSCode(vscodeVersion); + const [cli, ...baseArgs] = resolveCliArgsFromVSCodeExecutablePath(exe); + const cliRun = (extraArgs) => + execFileSync(cli, [...baseArgs, "--extensions-dir", extDir, "--user-data-dir", uddDir, ...extraArgs], + { stdio: "pipe", encoding: "utf8" }); + + // 2. Install dependencies (real VSCode resolves ms-python.python from the MS marketplace). + for (const dep of DEPS) { + try { + cliRun(["--install-extension", dep, "--force"]); + result.deps_resolved[dep] = true; + } catch { + result.deps_resolved[dep] = false; + } + } + + // 3. Upgrade scenario: install the baseline first. + if (mode === "upgrade") { + if (!fromVersion) throw new Error("--from required for upgrade mode"); + cliRun(["--install-extension", `${EXTENSION_ID}@${fromVersion}`, "--force"]); + } + + // 4. Install the target (a built .vsix path, or latest from marketplace). + const targetArg = target === "latest" ? EXTENSION_ID : resolve(target); + if (target !== "latest" && !existsSync(targetArg)) throw new Error(`vsix not found: ${targetArg}`); + cliRun(["--install-extension", targetArg, "--force"]); + + // 5. Verify the target is present. + const listed = cliRun(["--list-extensions", "--show-versions"]); + if (!listed.toLowerCase().includes(EXTENSION_ID.toLowerCase())) { + throw new Error(`extension not present after install:\n${listed}`); + } + result.install_ok = true; + + // 6. Launch headless against the fixture; the in-host suite asserts activation + dbt init. + await runTests({ + vscodeExecutablePath: exe, + extensionDevelopmentPath: join(repoRoot, "test-matrix", "harness-ext"), + extensionTestsPath: join(repoRoot, "out", "test", "matrix", "index"), + launchArgs: [ + "--extensions-dir", extDir, + "--user-data-dir", uddDir, + "--log", "trace", + "--disable-workspace-trust", + fixture, + ], + extensionTestsEnv: { + ...process.env, + MATRIX_UDD: uddDir, + DBT_PROFILES_DIR: process.env.DBT_PROFILES_DIR ?? "", + }, + }); + + // runTests resolves only if all in-host tests passed. + result.activation_ok = true; + result.dbt_flow_ok = true; + result.status = "pass"; + } catch (err) { + result.reason = String(err && err.message ? err.message : err).slice(0, 500); + // Distinguish install failure from activation/dbt failure for the report. + if (result.install_ok && !result.activation_ok) { + result.activation_ok = false; + result.dbt_flow_ok = false; + } + } finally { + result.duration_s = Math.round((Date.now() - started) / 1000); + writeFileSync(outPath, JSON.stringify(result, null, 2)); + console.log(`[matrix] ${runtime}/${result.os}/${mode} -> ${result.status}: ${result.reason || "ok"}`); + } + + process.exit(result.status === "pass" ? 0 : 1); +} + +main(); From cc8b08bb3613d3cb926b3345a02280ec7182790c Mon Sep 17 00:00:00 2001 From: Dev Punia Date: Sat, 30 May 2026 12:55:13 +0530 Subject: [PATCH 07/13] feat(matrix): code-server cell adapter emits RESULT_JSON --- test-matrix/codeserver-cell.sh | 46 ++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 test-matrix/codeserver-cell.sh diff --git a/test-matrix/codeserver-cell.sh b/test-matrix/codeserver-cell.sh new file mode 100755 index 000000000..4fb27587b --- /dev/null +++ b/test-matrix/codeserver-cell.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Runs the existing docker-setup/vsix-smoke.sh and converts its PASS/FAIL into a +# matrix RESULT_JSON. Usage: +# bash test-matrix/codeserver-cell.sh --mode fresh|upgrade [--from ] \ +# [--vsix-file |--target latest] --out +set -uo pipefail +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +MODE="fresh"; FROM=""; VSIX=""; TARGET="latest"; OUT="/tmp/codeserver.json" +while [ $# -gt 0 ]; do + case "$1" in + --mode) MODE="$2"; shift 2;; + --from) FROM="$2"; shift 2;; + --vsix-file) VSIX="$2"; shift 2;; + --target) TARGET="$2"; shift 2;; + --out) OUT="$2"; shift 2;; + *) shift;; + esac +done + +args=() +[ -n "$VSIX" ] && args+=(--vsix-file "$VSIX") +[ "$MODE" = "upgrade" ] && [ -n "$FROM" ] && args+=(--from-version "$FROM") + +start=$(date +%s) +if VSIX_SMOKE_REPORT=/tmp/cs-smoke.md bash "$REPO_ROOT/docker-setup/vsix-smoke.sh" "${args[@]}"; then + STATUS="pass"; REASON=""; OKS=true +else + STATUS="fail"; REASON="$(tail -3 /tmp/cs-smoke.md 2>/dev/null | tr '\n' ' ' | sed 's/"/'"'"'/g')"; OKS=false +fi +end=$(date +%s) + +python3 - "$OUT" "$MODE" "$FROM" "$TARGET" "$STATUS" "$REASON" "$OKS" $((end-start)) <<'PY' +import json, sys +out, mode, frm, target, status, reason, oks, dur = sys.argv[1:9] +ok = oks == "true" +json.dump({ + "runtime": "code-server", "os": "linux", "scenario": mode, + "from": frm or None, "to": "pr-build" if target != "latest" else "latest", + "install_ok": ok, "deps_resolved": {}, "activation_ok": ok, "dbt_flow_ok": ok, + "status": status, "reason": reason, "duration_s": int(dur), + "log_artifact": "codeserver.json", +}, open(out, "w"), indent=2) +print("wrote", out, status) +PY +[ "$STATUS" = "pass" ] From 7a971b473e97873fe23265f01d2da24b82574078 Mon Sep 17 00:00:00 2001 From: Dev Punia Date: Sat, 30 May 2026 12:57:34 +0530 Subject: [PATCH 08/13] fix(matrix): seed dbt python override + telemetry-off in cell user settings --- test-matrix/vscode-cell.mjs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/test-matrix/vscode-cell.mjs b/test-matrix/vscode-cell.mjs index 26bd2a2fd..b8a0fa73e 100644 --- a/test-matrix/vscode-cell.mjs +++ b/test-matrix/vscode-cell.mjs @@ -7,7 +7,7 @@ // node test-matrix/vscode-cell.mjs --mode fresh|upgrade --target \ // [--from ] [--vscode-version stable|insiders|x.y.z] --out import { execFileSync } from "node:child_process"; -import { existsSync, mkdtempSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; @@ -55,6 +55,27 @@ async function main() { const extDir = mkdtempSync(join(tmpdir(), "matrix-ext-")); const uddDir = mkdtempSync(join(tmpdir(), "matrix-udd-")); + // Pre-seed user settings so the extension finds dbt (via the hermetic venv's + // python) and so the test run does NOT emit real telemetry to App Insights. + const userDir = join(uddDir, "User"); + mkdirSync(userDir, { recursive: true }); + const pyInterp = process.env.PY_INTERP || ""; + writeFileSync( + join(userDir, "settings.json"), + JSON.stringify( + { + "dbt.dbtIntegration": "core", + ...(pyInterp ? { "dbt.dbtPythonPathOverride": pyInterp } : {}), + "dbt.altimateAiKey": "", + "telemetry.telemetryLevel": "off", + "redhat.telemetry.enabled": false, + "workbench.startupEditor": "none", + }, + null, + 2, + ), + ); + try { // 1. Download the editor + resolve its CLI. const exe = await downloadAndUnzipVSCode(vscodeVersion); From dc14e208d53c9a0567de2aba7f21ad0d79bbaae1 Mon Sep 17 00:00:00 2001 From: Dev Punia Date: Sat, 30 May 2026 12:59:37 +0530 Subject: [PATCH 09/13] ci(matrix): install/update matrix workflow (PR + daily) with Slack + sticky comment --- .github/workflows/install-update-matrix.yml | 201 ++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 .github/workflows/install-update-matrix.yml diff --git a/.github/workflows/install-update-matrix.yml b/.github/workflows/install-update-matrix.yml new file mode 100644 index 000000000..548e09546 --- /dev/null +++ b/.github/workflows/install-update-matrix.yml @@ -0,0 +1,201 @@ +name: Install/Update Matrix + +on: + pull_request: + branches: ["*"] + push: + tags: ["*"] + schedule: + - cron: "30 6 * * *" # daily, 06:30 UTC (offset from vsix-smoke's 06:00) + workflow_dispatch: + inputs: + target: + description: "vsix target: 'latest' or 'pr-build'" + type: string + default: "pr-build" + +permissions: + contents: read + pull-requests: write + statuses: write + +concurrency: + group: install-update-matrix-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Build one platform-specific VSIX per OS (native deps make a linux vsix + # uninstallable on mac/windows), each on its matching runner. + build: + strategy: + fail-fast: false + matrix: + include: + - { os: ubuntu-latest, target: linux-x64 } + - { os: macos-latest, target: darwin-arm64 } + - { os: windows-latest, target: win32-x64 } + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/common-build + with: + vsce-target: ${{ matrix.target }} + - name: Package VSIX + run: npx @vscode/vsce package --target ${{ matrix.target }} -o pu-${{ matrix.target }}.vsix + env: + NODE_OPTIONS: --max-old-space-size=8192 + - uses: actions/upload-artifact@v4 + with: + name: vsix-${{ matrix.target }} + path: pu-${{ matrix.target }}.vsix + retention-days: 3 + + vscode-cells: + needs: build + strategy: + fail-fast: false + matrix: + include: + # Blocking lane: stock VSCode, fresh + full upgrade baselines on Linux + - { os: ubuntu-latest, osl: linux, target: linux-x64, vscode: stable, mode: fresh, from: "" } + - { os: ubuntu-latest, osl: linux, target: linux-x64, vscode: stable, mode: upgrade, from: "0.61.4" } + - { os: ubuntu-latest, osl: linux, target: linux-x64, vscode: stable, mode: upgrade, from: "0.60.7" } + - { os: ubuntu-latest, osl: linux, target: linux-x64, vscode: stable, mode: upgrade, from: "0.59.5" } + - { os: ubuntu-latest, osl: linux, target: linux-x64, vscode: stable, mode: upgrade, from: "0.55.5" } + # Blocking lane: macOS + Windows, fresh + one representative upgrade + - { os: macos-latest, osl: macos, target: darwin-arm64, vscode: stable, mode: fresh, from: "" } + - { os: macos-latest, osl: macos, target: darwin-arm64, vscode: stable, mode: upgrade, from: "0.61.4" } + - { os: windows-latest, osl: windows, target: win32-x64, vscode: stable, mode: fresh, from: "" } + - { os: windows-latest, osl: windows, target: win32-x64, vscode: stable, mode: upgrade, from: "0.61.4" } + # Non-blocking: Insiders fresh on Linux + - { os: ubuntu-latest, osl: linux, target: linux-x64, vscode: insiders, mode: fresh, from: "" } + runs-on: ${{ matrix.os }} + timeout-minutes: 40 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + - name: Install deps + compile test suite + run: | + npm ci + npm run compile + - uses: actions/download-artifact@v4 + with: + name: vsix-${{ matrix.target }} + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Run cell + shell: bash + run: | + set -e + eval "$(bash test-matrix/setup-dbt-env.sh)" + vsixes=( pu-*.vsix ); VSIX="${vsixes[0]}" + OUT="result-${{ matrix.osl }}-${{ matrix.vscode }}-${{ matrix.mode }}-${{ matrix.from }}.json" + ARGS=(--mode "${{ matrix.mode }}" --target "$PWD/$VSIX" --vscode-version "${{ matrix.vscode }}" --out "$OUT") + [ -n "${{ matrix.from }}" ] && ARGS+=(--from "${{ matrix.from }}") + if [ "${{ matrix.osl }}" = "linux" ]; then + sudo apt-get update && sudo apt-get install -y xvfb + xvfb-run -a node test-matrix/vscode-cell.mjs "${ARGS[@]}" || true + else + node test-matrix/vscode-cell.mjs "${ARGS[@]}" || true + fi + - uses: actions/upload-artifact@v4 + with: + name: result-${{ matrix.osl }}-${{ matrix.vscode }}-${{ matrix.mode }}-${{ matrix.from }} + path: result-*.json + if-no-files-found: warn + + codeserver-cell: + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: vsix-linux-x64 + - name: code-server fresh + upgrade + run: | + vsixes=( pu-*.vsix ); VSIX="${vsixes[0]}" + bash test-matrix/codeserver-cell.sh --mode fresh --vsix-file "$PWD/$VSIX" --out result-codeserver-fresh.json || true + bash test-matrix/codeserver-cell.sh --mode upgrade --from 0.61.4 --vsix-file "$PWD/$VSIX" --out result-codeserver-upgrade.json || true + - uses: actions/upload-artifact@v4 + with: + name: result-codeserver + path: result-codeserver-*.json + if-no-files-found: warn + + aggregate: + needs: [vscode-cells, codeserver-cell] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + pattern: result-* + path: results + merge-multiple: true + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Aggregate + id: agg + run: | + set +e + python3 test-matrix/aggregate.py --results-dir results --out-dir agg \ + --target "${{ github.event_name == 'pull_request' && 'pr-build' || 'latest' }}" \ + --trigger "${{ github.event_name }}" + code=$? + echo "blocking_failed=$([ $code -ne 0 ] && echo true || echo false)" >> "$GITHUB_OUTPUT" + exit 0 + - uses: actions/upload-artifact@v4 + with: + name: matrix-report + path: agg/ + - name: Post sticky PR comment + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const body = '\n' + fs.readFileSync('agg/matrix.md', 'utf8'); + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number }); + const existing = comments.find(c => c.body.includes('')); + if (existing) await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: existing.id, body }); + else await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, body }); + - name: Set commit status (blocking gate) + if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + uses: actions/github-script@v7 + with: + script: | + const failed = '${{ steps.agg.outputs.blocking_failed }}' === 'true'; + await github.rest.repos.createCommitStatus({ + owner: context.repo.owner, repo: context.repo.repo, + sha: context.payload.pull_request.head.sha, + state: failed ? 'failure' : 'success', + description: failed ? 'Blocking-lane cell failed' : 'Install/Update matrix green', + context: 'Install/Update Matrix', + target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`, + }); + - name: Slack notify + if: always() + env: + SLACK_WEBHOOK_URL: ${{ secrets.MATRIX_SLACK_WEBHOOK }} + run: | + if [ -n "$SLACK_WEBHOOK_URL" ]; then + curl -sf -X POST -H 'Content-type: application/json' --data @agg/slack.json "$SLACK_WEBHOOK_URL" || echo "slack post failed (non-fatal)" + else + echo "MATRIX_SLACK_WEBHOOK not set; skipping Slack (channel ID pending)" + fi + - name: Fail job if blocking lane failed + if: steps.agg.outputs.blocking_failed == 'true' + run: | + echo "::error::Blocking-lane matrix cell(s) failed" + exit 1 From 0d9fa4896a0dc0c2df1400979e735628080a9363 Mon Sep 17 00:00:00 2001 From: Dev Punia Date: Sat, 30 May 2026 13:01:34 +0530 Subject: [PATCH 10/13] ci(matrix): gate marketplace publish on Install/Update Matrix; set status on tags --- .github/workflows/ci.yml | 28 +++++++++++++++++++-- .github/workflows/install-update-matrix.yml | 10 ++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74cbb88c0..6534f3ba6 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -201,8 +201,32 @@ jobs: retention-days: 14 if-no-files-found: ignore + # Release gate: block marketplace publish until the Install/Update Matrix + # reports green for this tag commit. The matrix workflow (triggered by the + # same tag push) sets the 'Install/Update Matrix' commit status; we poll it. + matrix-gate: + runs-on: ubuntu-latest + if: success() && startsWith(github.ref, 'refs/tags/') + timeout-minutes: 25 + steps: + - name: Wait for Install/Update Matrix status on this commit + uses: actions/github-script@v7 + with: + script: | + const ref = context.sha; + for (let i = 0; i < 40; i++) { + const { data } = await github.rest.repos.getCombinedStatusForRef({ + owner: context.repo.owner, repo: context.repo.repo, ref }); + const s = data.statuses.find((x) => x.context === 'Install/Update Matrix'); + if (s && s.state === 'success') { core.info('Install/Update Matrix green'); return; } + if (s && s.state === 'failure') { core.setFailed('Install/Update Matrix failed — blocking release'); return; } + core.info('waiting for Install/Update Matrix status...'); + await new Promise((r) => setTimeout(r, 30000)); + } + core.setFailed('Timed out waiting for Install/Update Matrix status'); + release-vsstudio-marketplace: - needs: [build, release-smoke] + needs: [build, release-smoke, matrix-gate] runs-on: ubuntu-latest if: success() && startsWith( github.ref, 'refs/tags/') strategy: @@ -256,7 +280,7 @@ jobs: text: "Tag: ${{ github.ref_name }} release to Visual Studio Marketplace status: ${{ needs.release-vsstudio-marketplace.result == 'success' && 'succeeded' || 'failed' }} :${{ needs.release-vsstudio-marketplace.result == 'success' && 'tada' || 'disappointed' }}:" release-openvsx-marketplace: - needs: build + needs: [build, matrix-gate] runs-on: ubuntu-latest if: success() && startsWith( github.ref, 'refs/tags/') strategy: diff --git a/.github/workflows/install-update-matrix.yml b/.github/workflows/install-update-matrix.yml index 548e09546..2d5abd450 100644 --- a/.github/workflows/install-update-matrix.yml +++ b/.github/workflows/install-update-matrix.yml @@ -171,14 +171,20 @@ jobs: if (existing) await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: existing.id, body }); else await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, body }); - name: Set commit status (blocking gate) - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + if: github.event_name == 'pull_request' || startsWith(github.ref, 'refs/tags/') uses: actions/github-script@v7 with: script: | + const pr = context.payload.pull_request; + if (pr && pr.head.repo.full_name !== context.payload.repository.full_name) { + core.info('fork PR — cannot set commit status, skipping'); + return; + } const failed = '${{ steps.agg.outputs.blocking_failed }}' === 'true'; + const sha = pr ? pr.head.sha : context.sha; await github.rest.repos.createCommitStatus({ owner: context.repo.owner, repo: context.repo.repo, - sha: context.payload.pull_request.head.sha, + sha, state: failed ? 'failure' : 'success', description: failed ? 'Blocking-lane cell failed' : 'Install/Update matrix green', context: 'Install/Update Matrix', From c01cdd2d5647af58b8bc3a402a6fd38ff2eaf76c Mon Sep 17 00:00:00 2001 From: Dev Punia Date: Sat, 30 May 2026 13:06:30 +0530 Subject: [PATCH 11/13] ci(matrix): match multi-segment base branches (** not *) so PR runs trigger --- .github/workflows/install-update-matrix.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/install-update-matrix.yml b/.github/workflows/install-update-matrix.yml index 2d5abd450..6f831c0ef 100644 --- a/.github/workflows/install-update-matrix.yml +++ b/.github/workflows/install-update-matrix.yml @@ -2,7 +2,8 @@ name: Install/Update Matrix on: pull_request: - branches: ["*"] + # "**" matches multi-segment base branches (e.g. feat/x); "*" would not. + branches: ["**"] push: tags: ["*"] schedule: From 90003a8ca616d8d7994289b29efa0548e3e24678 Mon Sep 17 00:00:00 2001 From: Dev Punia Date: Sat, 30 May 2026 13:21:36 +0530 Subject: [PATCH 12/13] fix(matrix): run editor CLI via shell on Windows (.cmd needs it) --- test-matrix/vscode-cell.mjs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test-matrix/vscode-cell.mjs b/test-matrix/vscode-cell.mjs index b8a0fa73e..30d410c59 100644 --- a/test-matrix/vscode-cell.mjs +++ b/test-matrix/vscode-cell.mjs @@ -80,9 +80,11 @@ async function main() { // 1. Download the editor + resolve its CLI. const exe = await downloadAndUnzipVSCode(vscodeVersion); const [cli, ...baseArgs] = resolveCliArgsFromVSCodeExecutablePath(exe); + // On Windows the resolved CLI is a `.cmd` (code.cmd); execFileSync cannot run + // a batch file without a shell, so enable shell there (per @vscode/test-electron). const cliRun = (extraArgs) => execFileSync(cli, [...baseArgs, "--extensions-dir", extDir, "--user-data-dir", uddDir, ...extraArgs], - { stdio: "pipe", encoding: "utf8" }); + { stdio: "pipe", encoding: "utf8", shell: process.platform === "win32" }); // 2. Install dependencies (real VSCode resolves ms-python.python from the MS marketplace). for (const dep of DEPS) { From f3a79d5dc6d98ab806ac30a6f2d5373e7aa3ead6 Mon Sep 17 00:00:00 2001 From: Dev Punia Date: Sat, 30 May 2026 13:48:16 +0530 Subject: [PATCH 13/13] =?UTF-8?q?fix(matrix):=20address=20final=20review?= =?UTF-8?q?=20=E2=80=94=20gate=20on=20empty/corrupt=20results,=20OS-split?= =?UTF-8?q?=20update=20matrix,=20widen=20release-gate=20poll=20+=20error?= =?UTF-8?q?=20state,=20capture=20CLI=20stderr?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/ci.yml | 7 ++- .github/workflows/install-update-matrix.yml | 7 +-- test-matrix/aggregate.py | 61 ++++++++++++++++----- test-matrix/vscode-cell.mjs | 12 ++-- tests/matrix/test_aggregate.py | 39 +++++++++++++ 5 files changed, 98 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6534f3ba6..67008d03d 100755 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -207,19 +207,22 @@ jobs: matrix-gate: runs-on: ubuntu-latest if: success() && startsWith(github.ref, 'refs/tags/') - timeout-minutes: 25 + timeout-minutes: 60 steps: - name: Wait for Install/Update Matrix status on this commit uses: actions/github-script@v7 with: script: | const ref = context.sha; - for (let i = 0; i < 40; i++) { + // Poll up to ~50 min (100 x 30s): the matrix builds 3 platforms + + // runs 12 cells, which on a cold cache can exceed 30 min. + for (let i = 0; i < 100; i++) { const { data } = await github.rest.repos.getCombinedStatusForRef({ owner: context.repo.owner, repo: context.repo.repo, ref }); const s = data.statuses.find((x) => x.context === 'Install/Update Matrix'); if (s && s.state === 'success') { core.info('Install/Update Matrix green'); return; } if (s && s.state === 'failure') { core.setFailed('Install/Update Matrix failed — blocking release'); return; } + if (s && s.state === 'error') { core.setFailed('Install/Update Matrix status errored — blocking release'); return; } core.info('waiting for Install/Update Matrix status...'); await new Promise((r) => setTimeout(r, 30000)); } diff --git a/.github/workflows/install-update-matrix.yml b/.github/workflows/install-update-matrix.yml index 6f831c0ef..c9f4661d7 100644 --- a/.github/workflows/install-update-matrix.yml +++ b/.github/workflows/install-update-matrix.yml @@ -8,12 +8,7 @@ on: tags: ["*"] schedule: - cron: "30 6 * * *" # daily, 06:30 UTC (offset from vsix-smoke's 06:00) - workflow_dispatch: - inputs: - target: - description: "vsix target: 'latest' or 'pr-build'" - type: string - default: "pr-build" + workflow_dispatch: {} permissions: contents: read diff --git a/test-matrix/aggregate.py b/test-matrix/aggregate.py index 87312fb57..fe9b52889 100644 --- a/test-matrix/aggregate.py +++ b/test-matrix/aggregate.py @@ -78,21 +78,26 @@ def _render_install(cells: list[dict]) -> str: def _render_update(cells: list[dict]) -> str: - runtimes = sorted({c["runtime"] for c in cells}, key=_runtime_sort_key) - baselines = sorted({c.get("from") for c in cells if c.get("from")}) - by = {(c["runtime"], c.get("from")): c for c in cells} - lines = ["### Update matrix (upgrade baseline → target)", ""] if not cells: lines.append("_no upgrade cells in this run_") lines.append("") return "\n".join(lines) - lines.append("| Runtime | " + " | ".join(f"from {b}" for b in baselines) + " |") + # Row = (runtime, os) so OS-specific upgrade cells (e.g. linux vs windows + # both upgrading from 0.61.4) don't collide into one row. + rows = sorted( + {(c["runtime"], c["os"]) for c in cells}, + key=lambda k: (_runtime_sort_key(k[0]), _os_sort_key(k[1])), + ) + baselines = sorted({c.get("from") for c in cells if c.get("from")}) + by = {(c["runtime"], c["os"], c.get("from")): c for c in cells} + + lines.append("| Runtime / OS | " + " | ".join(f"from {b}" for b in baselines) + " |") lines.append("|---|" + "---|" * len(baselines)) - for rt in runtimes: - row = [rt] + for rt, os_name in rows: + row = [f"{rt} ({os_name})"] for b in baselines: - cell = by.get((rt, b)) + cell = by.get((rt, os_name, b)) row.append(_cell_symbol(cell) if cell else "—") lines.append("| " + " | ".join(row) + " |") lines.append("") @@ -125,12 +130,18 @@ def _render_slack(results: list[dict], has_blocking_failure: bool) -> list[dict] ] -def _load_results(results_dir: str) -> list[dict]: +def _load_results(results_dir: str): + """Load every RESULT_JSON. Returns (results, errors) where errors is a list + of (path, message) for files that could not be parsed.""" out = [] + errors = [] for path in sorted(glob.glob(os.path.join(results_dir, "**", "*.json"), recursive=True)): - with open(path) as f: - out.append(json.load(f)) - return out + try: + with open(path) as f: + out.append(json.load(f)) + except (json.JSONDecodeError, OSError) as e: + errors.append((path, str(e))) + return out, errors def main() -> int: @@ -141,10 +152,28 @@ def main() -> int: ap.add_argument("--trigger", default="manual") args = ap.parse_args() - results = _load_results(args.results_dir) - out = build_matrices(results) + results, load_errors = _load_results(args.results_dir) os.makedirs(args.out_dir, exist_ok=True) + for path, msg in load_errors: + print(f"::warning::unreadable result file {path}: {msg}") + + if not results: + # No cell produced a result (e.g. every runner died before writing one). + # We cannot certify the matrix, so block and still emit a visible board. + note = "No result files found — treating as a blocking failure" + print(f"::error::{note}") + board = f"## VSIX Install + Update Matrix\n\n:x: {note}\n" + for name in ("install-matrix.md", "update-matrix.md", "matrix.md"): + with open(os.path.join(args.out_dir, name), "w") as f: + f.write(board) + with open(os.path.join(args.out_dir, "slack.json"), "w") as f: + json.dump( + {"blocks": [{"type": "section", "text": {"type": "mrkdwn", "text": f"*❌ {note}*"}}]}, + f, indent=2, + ) + return 1 + out = build_matrices(results) header = f"## VSIX Install + Update Matrix — target `{args.target or 'latest'}` ({args.trigger})\n\n" combined = header + out["install_md"] + "\n" + out["update_md"] with open(os.path.join(args.out_dir, "install-matrix.md"), "w") as f: @@ -160,6 +189,10 @@ def main() -> int: if out["has_blocking_failure"]: print("::error::Blocking-lane cell(s) failed — see matrix above") return 1 + if load_errors: + # A result file existed but was corrupt — we can't confirm that cell, so block. + print("::error::Some result files were unreadable — cannot certify the matrix") + return 1 return 0 diff --git a/test-matrix/vscode-cell.mjs b/test-matrix/vscode-cell.mjs index 30d410c59..50c3bc71f 100644 --- a/test-matrix/vscode-cell.mjs +++ b/test-matrix/vscode-cell.mjs @@ -138,12 +138,12 @@ async function main() { result.dbt_flow_ok = true; result.status = "pass"; } catch (err) { - result.reason = String(err && err.message ? err.message : err).slice(0, 500); - // Distinguish install failure from activation/dbt failure for the report. - if (result.install_ok && !result.activation_ok) { - result.activation_ok = false; - result.dbt_flow_ok = false; - } + // Capture the CLI's stderr/stdout too — execFileSync errors otherwise hide + // the actual reason (e.g. dependency resolution failures). install_ok / + // activation_ok already encode which phase failed. + const base = String(err && err.message ? err.message : err); + const std = `${(err && err.stderr) || ""}${(err && err.stdout) || ""}`.trim(); + result.reason = (std ? `${base} | ${std}` : base).slice(0, 600); } finally { result.duration_s = Math.round((Date.now() - started) / 1000); writeFileSync(outPath, JSON.stringify(result, null, 2)); diff --git a/tests/matrix/test_aggregate.py b/tests/matrix/test_aggregate.py index 06950d7c0..bbe9fcb7b 100644 --- a/tests/matrix/test_aggregate.py +++ b/tests/matrix/test_aggregate.py @@ -97,3 +97,42 @@ def test_cli_writes_files_and_exit_code(tmp_path): assert (odir / "slack.json").exists() slack = json.loads((odir / "slack.json").read_text()) assert "blocks" in slack and len(slack["blocks"]) == 2 + + +def _run_cli(rdir, odir): + root = pathlib.Path(__file__).resolve().parents[2] + return subprocess.run( + [sys.executable, str(root / "test-matrix" / "aggregate.py"), + "--results-dir", str(rdir), "--out-dir", str(odir)], + capture_output=True, text=True, + ) + + +def test_cli_empty_results_blocks(tmp_path): + rdir = tmp_path / "results" + rdir.mkdir() + odir = tmp_path / "out" + proc = _run_cli(rdir, odir) + assert proc.returncode == 1 # no results == cannot certify == block + assert (odir / "matrix.md").exists() + assert (odir / "slack.json").exists() + + +def test_cli_malformed_result_blocks_even_with_a_passing_cell(tmp_path): + rdir = tmp_path / "results" + rdir.mkdir() + (rdir / "good.json").write_text(json.dumps(_cell())) # a passing vscode cell + (rdir / "bad.json").write_text("{ this is not valid json") + odir = tmp_path / "out" + proc = _run_cli(rdir, odir) + assert proc.returncode == 1 # corrupt file blocks despite the good cell passing + + +def test_update_matrix_separates_os_rows(): + results = [ + _cell(scenario="upgrade", os="linux", **{"from": "0.61.4"}), + _cell(scenario="upgrade", os="windows", **{"from": "0.61.4"}, status="fail", dbt_flow_ok=False), + ] + md = aggregate.build_matrices(results)["update_md"] + assert "vscode (linux)" in md and "vscode (windows)" in md + assert "✅" in md and "❌" in md