Install/Update test matrix (P1: blocking VSCode 3-OS lane + board)

.github/workflows/ci.yml

@@ -201,8 +201,35 @@ jobs:
           retention-days: 14
           if-no-files-found: ignore
 
+  # Release gate: block marketplace publish until the Install/Update Matrix
+  # reports green for this tag commit. The matrix workflow (triggered by the
+  # same tag push) sets the 'Install/Update Matrix' commit status; we poll it.
+  matrix-gate:
+    runs-on: ubuntu-latest
+    if: success() && startsWith(github.ref, 'refs/tags/')
+    timeout-minutes: 60
+    steps:
+      - name: Wait for Install/Update Matrix status on this commit
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const ref = context.sha;
+            // Poll up to ~50 min (100 x 30s): the matrix builds 3 platforms +
+            // runs 12 cells, which on a cold cache can exceed 30 min.
+            for (let i = 0; i < 100; i++) {
+              const { data } = await github.rest.repos.getCombinedStatusForRef({
+                owner: context.repo.owner, repo: context.repo.repo, ref });
+              const s = data.statuses.find((x) => x.context === 'Install/Update Matrix');
+              if (s && s.state === 'success') { core.info('Install/Update Matrix green'); return; }
+              if (s && s.state === 'failure') { core.setFailed('Install/Update Matrix failed — blocking release'); return; }
+              if (s && s.state === 'error') { core.setFailed('Install/Update Matrix status errored — blocking release'); return; }
+              core.info('waiting for Install/Update Matrix status...');
+              await new Promise((r) => setTimeout(r, 30000));
+            }
+            core.setFailed('Timed out waiting for Install/Update Matrix status');
+
   release-vsstudio-marketplace:
-    needs: [build, release-smoke]
+    needs: [build, release-smoke, matrix-gate]
     runs-on: ubuntu-latest
     if: success() && startsWith( github.ref, 'refs/tags/')
     strategy:
@@ -256,7 +283,7 @@ jobs:
           text: "Tag: ${{  github.ref_name }} release to Visual Studio Marketplace status: ${{ needs.release-vsstudio-marketplace.result == 'success' && 'succeeded' || 'failed' }} :${{ needs.release-vsstudio-marketplace.result == 'success' && 'tada' || 'disappointed' }}:"
 
   release-openvsx-marketplace:
-    needs: build
+    needs: [build, matrix-gate]
     runs-on: ubuntu-latest
     if: success() && startsWith( github.ref, 'refs/tags/')
     strategy:

.github/workflows/install-update-matrix.yml

@@ -0,0 +1,203 @@
+name: Install/Update Matrix
+
+on:
+  pull_request:
+    # "**" matches multi-segment base branches (e.g. feat/x); "*" would not.
+    branches: ["**"]
+  push:
+    tags: ["*"]
+  schedule:
+    - cron: "30 6 * * *" # daily, 06:30 UTC (offset from vsix-smoke's 06:00)
+  workflow_dispatch: {}
+
+permissions:
+  contents: read
+  pull-requests: write
+  statuses: write
+
+concurrency:
+  group: install-update-matrix-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # Build one platform-specific VSIX per OS (native deps make a linux vsix
+  # uninstallable on mac/windows), each on its matching runner.
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { os: ubuntu-latest, target: linux-x64 }
+          - { os: macos-latest, target: darwin-arm64 }
+          - { os: windows-latest, target: win32-x64 }
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 30
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/common-build
+        with:
+          vsce-target: ${{ matrix.target }}
+      - name: Package VSIX
+        run: npx @vscode/vsce package --target ${{ matrix.target }} -o pu-${{ matrix.target }}.vsix
+        env:
+          NODE_OPTIONS: --max-old-space-size=8192
+      - uses: actions/upload-artifact@v4
+        with:
+          name: vsix-${{ matrix.target }}
+          path: pu-${{ matrix.target }}.vsix
+          retention-days: 3
+
+  vscode-cells:
+    needs: build
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Blocking lane: stock VSCode, fresh + full upgrade baselines on Linux
+          - { os: ubuntu-latest,  osl: linux,   target: linux-x64,    vscode: stable,   mode: fresh,   from: "" }
+          - { os: ubuntu-latest,  osl: linux,   target: linux-x64,    vscode: stable,   mode: upgrade, from: "0.61.4" }
+          - { os: ubuntu-latest,  osl: linux,   target: linux-x64,    vscode: stable,   mode: upgrade, from: "0.60.7" }
+          - { os: ubuntu-latest,  osl: linux,   target: linux-x64,    vscode: stable,   mode: upgrade, from: "0.59.5" }
+          - { os: ubuntu-latest,  osl: linux,   target: linux-x64,    vscode: stable,   mode: upgrade, from: "0.55.5" }
+          # Blocking lane: macOS + Windows, fresh + one representative upgrade
+          - { os: macos-latest,   osl: macos,   target: darwin-arm64, vscode: stable,   mode: fresh,   from: "" }
+          - { os: macos-latest,   osl: macos,   target: darwin-arm64, vscode: stable,   mode: upgrade, from: "0.61.4" }
+          - { os: windows-latest, osl: windows, target: win32-x64,    vscode: stable,   mode: fresh,   from: "" }
+          - { os: windows-latest, osl: windows, target: win32-x64,    vscode: stable,   mode: upgrade, from: "0.61.4" }
+          # Non-blocking: Insiders fresh on Linux
+          - { os: ubuntu-latest,  osl: linux,   target: linux-x64,    vscode: insiders, mode: fresh,   from: "" }
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 40
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: "npm"
+      - name: Install deps + compile test suite
+        run: |
+          npm ci
+          npm run compile
+      - uses: actions/download-artifact@v4
+        with:
+          name: vsix-${{ matrix.target }}
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Run cell
+        shell: bash
+        run: |
+          set -e
+          eval "$(bash test-matrix/setup-dbt-env.sh)"
+          vsixes=( pu-*.vsix ); VSIX="${vsixes[0]}"
+          OUT="result-${{ matrix.osl }}-${{ matrix.vscode }}-${{ matrix.mode }}-${{ matrix.from }}.json"
+          ARGS=(--mode "${{ matrix.mode }}" --target "$PWD/$VSIX" --vscode-version "${{ matrix.vscode }}" --out "$OUT")
+          [ -n "${{ matrix.from }}" ] && ARGS+=(--from "${{ matrix.from }}")
+          if [ "${{ matrix.osl }}" = "linux" ]; then
+            sudo apt-get update && sudo apt-get install -y xvfb
+            xvfb-run -a node test-matrix/vscode-cell.mjs "${ARGS[@]}" || true
+          else
+            node test-matrix/vscode-cell.mjs "${ARGS[@]}" || true
+          fi
+      - uses: actions/upload-artifact@v4
+        with:
+          name: result-${{ matrix.osl }}-${{ matrix.vscode }}-${{ matrix.mode }}-${{ matrix.from }}
+          path: result-*.json
+          if-no-files-found: warn
+
+  codeserver-cell:
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    continue-on-error: true
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          name: vsix-linux-x64
+      - name: code-server fresh + upgrade
+        run: |
+          vsixes=( pu-*.vsix ); VSIX="${vsixes[0]}"
+          bash test-matrix/codeserver-cell.sh --mode fresh   --vsix-file "$PWD/$VSIX" --out result-codeserver-fresh.json || true
+          bash test-matrix/codeserver-cell.sh --mode upgrade --from 0.61.4 --vsix-file "$PWD/$VSIX" --out result-codeserver-upgrade.json || true
+      - uses: actions/upload-artifact@v4
+        with:
+          name: result-codeserver
+          path: result-codeserver-*.json
+          if-no-files-found: warn
+
+  aggregate:
+    needs: [vscode-cells, codeserver-cell]
+    if: always()
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: result-*
+          path: results
+          merge-multiple: true
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Aggregate
+        id: agg
+        run: |
+          set +e
+          python3 test-matrix/aggregate.py --results-dir results --out-dir agg \
+            --target "${{ github.event_name == 'pull_request' && 'pr-build' || 'latest' }}" \
+            --trigger "${{ github.event_name }}"
+          code=$?
+          echo "blocking_failed=$([ $code -ne 0 ] && echo true || echo false)" >> "$GITHUB_OUTPUT"
+          exit 0
+      - uses: actions/upload-artifact@v4
+        with:
+          name: matrix-report
+          path: agg/
+      - name: Post sticky PR comment
+        if: github.event_name == 'pull_request'
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            const body = '<!-- install-update-matrix -->\n' + fs.readFileSync('agg/matrix.md', 'utf8');
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number });
+            const existing = comments.find(c => c.body.includes('<!-- install-update-matrix -->'));
+            if (existing) await github.rest.issues.updateComment({ owner: context.repo.owner, repo: context.repo.repo, comment_id: existing.id, body });
+            else await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, body });
+      - name: Set commit status (blocking gate)
+        if: github.event_name == 'pull_request' || startsWith(github.ref, 'refs/tags/')
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const pr = context.payload.pull_request;
+            if (pr && pr.head.repo.full_name !== context.payload.repository.full_name) {
+              core.info('fork PR — cannot set commit status, skipping');
+              return;
+            }
+            const failed = '${{ steps.agg.outputs.blocking_failed }}' === 'true';
+            const sha = pr ? pr.head.sha : context.sha;
+            await github.rest.repos.createCommitStatus({
+              owner: context.repo.owner, repo: context.repo.repo,
+              sha,
+              state: failed ? 'failure' : 'success',
+              description: failed ? 'Blocking-lane cell failed' : 'Install/Update matrix green',
+              context: 'Install/Update Matrix',
+              target_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`,
+            });
+      - name: Slack notify
+        if: always()
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.MATRIX_SLACK_WEBHOOK }}
+        run: |
+          if [ -n "$SLACK_WEBHOOK_URL" ]; then
+            curl -sf -X POST -H 'Content-type: application/json' --data @agg/slack.json "$SLACK_WEBHOOK_URL" || echo "slack post failed (non-fatal)"
+          else
+            echo "MATRIX_SLACK_WEBHOOK not set; skipping Slack (channel ID pending)"
+          fi
+      - name: Fail job if blocking lane failed
+        if: steps.agg.outputs.blocking_failed == 'true'
+        run: |
+          echo "::error::Blocking-lane matrix cell(s) failed"
+          exit 1

.gitignore

@@ -50,3 +50,6 @@ docker-setup/.env
 
 # GitHub issues dashboard cache
 monitoring/github_issues/.cache/
+
+# Install/update matrix local env
+.matrix-venv/

package.json

@@ -1209,7 +1209,10 @@
     "compile": "tsc -p ./",
     "docker:deploy": "./docker-setup/deploy.sh",
     "docker:logs": "cd docker-setup && docker compose logs -f || docker-compose logs -f",
-    "docker:stop": "cd docker-setup && docker compose down || docker-compose down"
+    "docker:stop": "cd docker-setup && docker compose down || docker-compose down",
+    "matrix:vscode": "node test-matrix/vscode-cell.mjs",
+    "matrix:aggregate": "python3 test-matrix/aggregate.py",
+    "matrix:test-aggregate": "python3 -m pytest tests/matrix/ -q"
   },
   "devDependencies": {
     "@istanbuljs/nyc-config-typescript": "^1.0.2",

src/test/matrix/activation.test.ts

@@ -0,0 +1,98 @@
+import * as assert from "assert";
+import * as fs from "fs";
+import * as path from "path";
+import * as vscode from "vscode";
+
+const EXTENSION_ID = "innoverio.vscode-dbt-power-user";
+// A command this extension contributes (proves contributions loaded), from package.json contributes.commands.
+const STABLE_COMMAND = "dbtPowerUser.openInsights";
+// Success/failure markers emitted to the file-backed "Log - dbt" LogOutputChannel and the exthost log.
+const INIT_OK = "Initialized dbt project";
+const INIT_FAIL = "Unable to register dbt project";
+
+function sleep(ms: number) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+function readAllLogs(uddDir: string): string {
+  // VSCode writes LogOutputChannels + console output under <user-data-dir>/logs/**.
+  const out: string[] = [];
+  const stack = [path.join(uddDir, "logs")];
+  while (stack.length) {
+    const dir = stack.pop()!;
+    let entries: fs.Dirent[] = [];
+    try {
+      entries = fs.readdirSync(dir, { withFileTypes: true });
+    } catch {
+      continue;
+    }
+    for (const e of entries) {
+      const p = path.join(dir, e.name);
+      if (e.isDirectory()) {
+        stack.push(p);
+      } else if (e.name.endsWith(".log")) {
+        try {
+          out.push(fs.readFileSync(p, "utf8"));
+        } catch {
+          /* ignore unreadable rotating log */
+        }
+      }
+    }
+  }
+  return out.join("\n");
+}
+
+suite("Matrix: installed VSIX activation + dbt project init", function () {
+  this.timeout(120_000);
+
+  test("extension is installed and activates", async function () {
+    const ext = vscode.extensions.getExtension(EXTENSION_ID);
+    assert.ok(
+      ext,
+      `${EXTENSION_ID} should be installed in the test extensions-dir`,
+    );
+    await ext!.activate();
+    assert.strictEqual(ext!.isActive, true, "extension should be active");
+  });
+
+  test("contributed command is registered", async function () {
+    const cmds = await vscode.commands.getCommands(true);
+    assert.ok(
+      cmds.includes(STABLE_COMMAND),
+      `command ${STABLE_COMMAND} should be registered`,
+    );
+  });
+
+  test("dbt fixture workspace is open", function () {
+    const folders = vscode.workspace.workspaceFolders ?? [];
+    assert.ok(folders.length > 0, "a workspace folder should be open");
+    assert.ok(
+      folders.some((f) =>
+        fs.existsSync(path.join(f.uri.fsPath, "dbt_project.yml")),
+      ),
+      "the open workspace should contain dbt_project.yml",
+    );
+  });
+
+  test("dbt project initializes (log shows 'Initialized dbt project')", async function () {
+    const uddDir = process.env.MATRIX_UDD;
+    assert.ok(uddDir, "MATRIX_UDD env must point at the --user-data-dir");
+    const deadline = Date.now() + 90_000;
+    let logs = "";
+    while (Date.now() < deadline) {
+      logs = readAllLogs(uddDir!);
+      if (logs.includes(INIT_FAIL)) {
+        assert.fail(
+          `dbt project registration failed: found '${INIT_FAIL}' in logs`,
+        );
+      }
+      if (logs.includes(INIT_OK)) {
+        return; // success
+      }
+      await sleep(2000);
+    }
+    assert.fail(
+      `did not observe '${INIT_OK}' within 90s (dbt project did not initialize)`,
+    );
+  });
+});

src/test/matrix/index.ts

@@ -0,0 +1,19 @@
+import { glob } from "glob";
+import Mocha from "mocha";
+import * as path from "path";
+
+export async function run(): Promise<void> {
+  const mocha = new Mocha({ ui: "tdd", timeout: 120_000, color: true });
+  const testsRoot = path.resolve(__dirname);
+  const files = await glob("**/*.test.js", { cwd: testsRoot });
+  for (const file of files) {
+    mocha.addFile(path.resolve(testsRoot, file));
+  }
+  return new Promise<void>((resolve, reject) => {
+    mocha.run((failures) =>
+      failures > 0
+        ? reject(new Error(`${failures} test(s) failed.`))
+        : resolve(),
+    );
+  });
+}