ci.yml

name: CI

on:
  push:
    branches:
      - main
    tags:
      - "v*"
  pull_request:
  workflow_dispatch:

# Test selection: PRs and main pushes only run the moon/git-compat tests
# affected by the change (tools/select-affected-tests.mjs, driven by the
# module -> git-test map in tools/flaker-affected-rules.mjs). Full runs are
# the pre-release check: trigger via workflow_dispatch or a v* tag push.

jobs:
  select:
    runs-on: ubuntu-latest
    outputs:
      full: ${{ steps.sel.outputs.full }}
      cmd_bit: ${{ steps.sel.outputs.cmd_bit }}
      moon_targets: ${{ steps.sel.outputs.moon_targets }}
      git_test_count: ${{ steps.sel.outputs.git_test_count }}
      git_tests: ${{ steps.sel.outputs.git_tests }}
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          fetch-depth: 0
      - name: Select affected tests
        id: sel
        run: |
          set -euo pipefail
          full=false
          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then full=true; fi
          case "${{ github.ref }}" in refs/tags/*) full=true ;; esac
          if [ "$full" = "true" ]; then
            echo "Full run (pre-release check trigger)"
            node tools/select-affected-tests.mjs --full --github
          else
            base="${{ github.event.pull_request.base.sha || github.event.before }}"
            if [ -z "$base" ] || [ "$base" = "0000000000000000000000000000000000000000" ]; then
              base="origin/main"
            fi
            echo "Selecting affected tests vs base $base"
            node tools/select-affected-tests.mjs --base "$base" --head HEAD --github
          fi

  test:
    runs-on: ubuntu-latest
    needs: select
    if: needs.select.outputs.moon_targets != ''
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          submodules: true
      - name: Install pkfire + pkspec
        uses: mizchi/pkfire@v0.12.1
        with:
          version: 0.12.1
      - name: Install pkspec
        uses: mizchi/pkspec@v0.4.1
        with:
          version: 0.4.1
          pkl-version: none
      - name: Install MoonBit CLI
        uses: ./.github/actions/setup-moonbit
      - name: Moon update
        run: moon update
      - name: pkf run check
        run: pkf run check
      - name: Build native binary for wbtests
        run: |
          moon build --target native --release modules/bit
          cp _build/native/release/build/mizchi/bit/bit.exe tools/git-shim/moon
          chmod +x tools/git-shim/moon
          # Set system git as fallback for wbtests (third_party/git/git is not built in test job)
          which git > tools/git-shim/real-git-path
          # Allow file:// protocol for submodule wbtests (git 2.38+ defaults to user)
          git config --global protocol.file.allow always
      - name: Run JS and WASM tests
        run: |
          set -euo pipefail
          full="${{ needs.select.outputs.full }}"
          targets=" ${{ needs.select.outputs.moon_targets }} "
          affected() { [ "$full" = "true" ] || case "$targets" in *" $1 "*) true ;; *) false ;; esac; }
          if affected mizchi/bit/tests || affected mizchi/bit_lib; then
            moon test --target js -p mizchi/bit -p mizchi/bit_lib
          fi
          if affected mizchi/bit_runtime; then
            moon test --target wasm -p mizchi/bit_runtime -f storage_runtime_wbtest.mbt
          fi
          if affected mizchi/bit_diff3; then
            moon test --target wasm -p mizchi/bit_diff3
          fi
          if affected mizchi/bit_repo; then
            moon test --target wasm -p mizchi/bit_repo
          fi
          if affected mizchi/bit_grep; then
            moon test --target wasm -p mizchi/bit_grep
          fi
          # Kill any orphan moon processes and remove stale lock before native tests
          pkill -f "moon" || true
          rm -f _build/.moon-lock
      - name: Run native tests (non-cmd modules)
        timeout-minutes: 30
        run: |
          set -euo pipefail
          full="${{ needs.select.outputs.full }}"
          if [ "$full" = "true" ]; then
            # Test every extracted bit_* / bitx_* module + mizchi/bit (without cmd).
            # cmd/bit native tests run in the separate cmd-native-test job because
            # they dominate runtime (~50 min for the linker pass).
            modules=$(awk '/members = \[/,/\]/' moon.work | grep -oE '"[^"]+"' | tr -d '"' | sed 's|^\./||')
            for m in $modules; do
              case "$m" in
                modules/bit) continue ;; # cmd lives here — handled separately
              esac
              mod_name="mizchi/$(basename "$m")"
              echo "::group::moon test --target native -p $mod_name"
              moon test --target native -p "$mod_name"
              echo "::endgroup::"
            done
            moon test --target native -p mizchi/bit/tests
            moon test --target native -p mizchi/bit/fuzz_tests || true
          else
            for t in ${{ needs.select.outputs.moon_targets }}; do
              echo "::group::moon test --target native -p $t"
              if [ "$t" = "mizchi/bit/tests" ]; then
                moon test --target native -p mizchi/bit/tests
                moon test --target native -p mizchi/bit/fuzz_tests || true
              else
                moon test --target native -p "$t"
              fi
              echo "::endgroup::"
            done
          fi

  cmd-native-test:
    runs-on: ubuntu-latest
    needs: select
    if: needs.select.outputs.cmd_bit == 'true'
    # The bundled `tcc` backend hangs compiling cmd/bit's large test binary, so
    # override the native C compiler with gcc (MOON_CC). gcc at -O0 (the debug
    # default) compiles it quickly and never hangs, unlike tcc; using gcc -O
    # (--release) also works but is far slower.
    env:
      MOON_CC: gcc
    strategy:
      fail-fast: false
      matrix:
        # cmd/bit's whitebox tests are split into 6 file-based shards: a single
        # `moon test <files>` only compiles a test driver for the selected files
        # and runs their tests, so each shard's compile and run is ~1/6 of the
        # whole. This keeps every job well under the timeout (the unsharded run
        # exceeded 150 min even with gcc). cmd/git-bit is small, run whole.
        include:
          - { name: cmd-git-bit, kind: pkg,   sel: mizchi/bit/cmd/git-bit, timeout: 30 }
          - { name: cmd-bit-0,   kind: shard, sel: "0",                     timeout: 60 }
          - { name: cmd-bit-1,   kind: shard, sel: "1",                     timeout: 60 }
          - { name: cmd-bit-2,   kind: shard, sel: "2",                     timeout: 60 }
          - { name: cmd-bit-3,   kind: shard, sel: "3",                     timeout: 60 }
          - { name: cmd-bit-4,   kind: shard, sel: "4",                     timeout: 60 }
          - { name: cmd-bit-5,   kind: shard, sel: "5",                     timeout: 60 }
    name: cmd-native-test (${{ matrix.name }})
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          submodules: true
      - name: Install pkfire + pkspec
        uses: mizchi/pkfire@v0.12.1
        with:
          version: 0.12.1
      - name: Install pkspec
        uses: mizchi/pkspec@v0.4.1
        with:
          version: 0.4.1
          pkl-version: none
      - name: Install MoonBit CLI
        uses: ./.github/actions/setup-moonbit
      - name: Moon update
        run: moon update
      - name: Build native binary for wbtests
        run: |
          set -euo pipefail
          moon --version
          moon build --target native modules/bit
          bin_path=$(find _build/native -maxdepth 8 -type f -name 'bit.exe' | head -1)
          if [ -z "$bin_path" ]; then
            echo "bit.exe not found after moon build"
            find _build/native -maxdepth 8 -type f | head -20 || true
            exit 1
          fi
          echo "bit.exe found at: $bin_path"
          cp "$bin_path" tools/git-shim/moon
          chmod +x tools/git-shim/moon
          which git > tools/git-shim/real-git-path
          git config --global protocol.file.allow always
      - name: Clear stale processes and moon lock
        run: |
          pkill -f "moon" || true
          pkill -f "clang" || true
          sleep 1
          rm -f _build/.moon-lock
      - name: Run ${{ matrix.name }} native tests
        timeout-minutes: ${{ matrix.timeout }}
        run: |
          set -euo pipefail
          if [ "${{ matrix.kind }}" = "pkg" ]; then
            moon test --target native -p ${{ matrix.sel }}
          else
            files=$(ls modules/bit/cmd/bit/*.mbt | sort | awk "(NR-1) % 6 == ${{ matrix.sel }}")
            echo "cmd-bit shard ${{ matrix.sel }}/6 — files:"
            echo "$files"
            # shellcheck disable=SC2086
            moon test --target native $files
          fi

  distributed-test:
    runs-on: ubuntu-latest
    needs: select
    if: >-
      needs.select.outputs.full == 'true' ||
      contains(needs.select.outputs.moon_targets, 'bitx_hub') ||
      contains(needs.select.outputs.moon_targets, 'bitx_kv')
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          submodules: true
      - name: Install MoonBit CLI
        uses: ./.github/actions/setup-moonbit
      - name: Moon update
        run: moon update
      - name: Distributed tests
        run: |
          moon test --target native -p mizchi/bitx_hub
          moon test --target native -p mizchi/bitx_hub/native
          moon test --target native -p mizchi/bitx_kv

  nix-build:
    runs-on: ubuntu-latest
    needs: select
    if: needs.select.outputs.moon_targets != ''
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@v5
      - uses: ./.github/actions/setup-nix
      - name: Build
        run: nix build
      - name: Smoke test
        run: test -x ./result/bin/bit

  git-compat:
    runs-on: ubuntu-latest
    needs: select
    if: needs.select.outputs.git_test_count != '0'
    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
    steps:
      - name: Checkout
        uses: actions/checkout@v5
        with:
          submodules: true
          fetch-depth: 0

      - name: Compute shard test list
        id: shardtests
        run: |
          set -euo pipefail
          printf '%s\n' "${{ needs.select.outputs.git_tests }}" > /tmp/selected-git-tests.txt
          tests=$(tools/select-git-tests.sh "${{ matrix.shard }}" 10 /tmp/selected-git-tests.txt)
          count=$(echo "$tests" | wc -w | tr -d ' ')
          echo "count=$count" >> "$GITHUB_OUTPUT"
          {
            echo "tests<<SHARD_TESTS_EOF"
            echo "$tests"
            echo "SHARD_TESTS_EOF"
          } >> "$GITHUB_OUTPUT"
          echo "Shard ${{ matrix.shard }}: $count of ${{ needs.select.outputs.git_test_count }} selected tests"

      - name: Apply git test patches
        if: steps.shardtests.outputs.count != '0'
        run: tools/apply-git-test-patches.sh

      - name: Install dependencies
        if: steps.shardtests.outputs.count != '0'
        run: |
          sudo apt-get update
          sudo apt-get install -y gettext libcurl4-openssl-dev libexpat1-dev

      - name: Build git from source
        if: steps.shardtests.outputs.count != '0'
        run: |
          cd third_party/git
          make -j$(nproc)

      - name: Install MoonBit CLI
        if: steps.shardtests.outputs.count != '0'
        uses: ./.github/actions/setup-moonbit

      - name: Moon update
        if: steps.shardtests.outputs.count != '0'
        run: moon update

      - name: Build bit
        if: steps.shardtests.outputs.count != '0'
        run: |
          set -euo pipefail
          moon --version
          moon build --target native --release modules/bit
          bin_path=$(find _build/native -maxdepth 8 -type f -name 'bit.exe' | head -1)
          if [ -z "$bin_path" ]; then
            echo "bit.exe not found after moon build"
            find _build/native -maxdepth 8 -type f | head -20 || true
            exit 1
          fi
          echo "bit.exe found at: $bin_path"
          cp "$bin_path" tools/git-shim/moon
          chmod +x tools/git-shim/moon

      - name: Run git compatibility tests
        if: steps.shardtests.outputs.count != '0'
        run: |
          real_git="$(pwd)/third_party/git/git"
          exec_path="$(pwd)/third_party/git"
          shim_dir="$(pwd)/tools/git-shim/bin"
          echo "$real_git" > tools/git-shim/real-git-path

          tests="${{ steps.shardtests.outputs.tests }}"
          echo "Shard ${{ matrix.shard }}: $(echo "$tests" | wc -w | tr -d ' ') tests"

          export SHIM_REAL_GIT="$real_git" SHIM_EXEC_PATH="$exec_path" \
          SHIM_MOON="$(pwd)/tools/git-shim/moon" \
          SHIM_CMDS="init add diff diff-files diff-index ls-files tag branch checkout switch commit log show reflog reset update-ref update-index status merge rebase clone push fetch pull mv notes stash rm submodule worktree config show-ref for-each-ref rev-parse symbolic-ref cherry-pick remote cat-file hash-object ls-tree write-tree commit-tree receive-pack upload-pack pack-objects index-pack format-patch describe gc clean sparse-checkout restore blame grep shell rev-list bisect diff-tree read-tree fsck am apply bundle cherry revert prune pack-refs mktree shortlog verify-pack unpack-objects maintenance range-diff show-branch repack multi-pack-index pack-redundant send-pack request-pull merge-base var stripspace ls-remote fmt-merge-msg patch-id count-objects name-rev update-server-info check-ref-format mktag interpret-trailers column merge-tree merge-file fast-import fast-export verify-tag fetch-pack difftool rerere mailinfo archive check-attr check-ignore show-index get-tar-commit-id verify-commit annotate last-modified history" SHIM_STRICT=1 \
          GIT_TEST_INSTALLED="$shim_dir" GIT_TEST_EXEC_PATH="$exec_path" \
          GIT_TEST_DEFAULT_HASH=sha1

          failed_tests=""
          passed=0
          failed=0
          timing_log="test-timing-shard-${{ matrix.shard }}.tsv"
          echo "# test	seconds" > "$timing_log"
          cd third_party/git/t
          test_output="/tmp/git-test-output-$$.log"
          for t in $tests; do
            # Strip third_party/git/t/ prefix if present
            t_base="${t#third_party/git/t/}"
            t_start=$SECONDS
            # Per-test timeout: 3× the recorded median runtime weight,
            # floored at 120s. Recorded weights cap at the previous 120s
            # timeout, so tests whose true runtime exceeds it (e.g.
            # t1400-update-ref) get enough headroom to finish.
            t_weight=$(awk -F'\t' -v t="$t_base" '$1==t {print $2}' "$GITHUB_WORKSPACE/tools/git-test-runtime-seconds.tsv")
            t_timeout=$(( ${t_weight:-40} * 3 ))
            if [ "$t_timeout" -lt 120 ]; then t_timeout=120; fi
            if timeout "$t_timeout" bash "./$t_base" > "$test_output" 2>&1; then
              tail -5 "$test_output"
              passed=$((passed + 1))
            else
              exit_code=$?
              echo "::group::FAIL $t_base (exit=$exit_code)"
              grep -E "^not ok" "$test_output" | head -30
              tail -5 "$test_output"
              echo "::endgroup::"
              if [ "$exit_code" -eq 124 ]; then
                echo "TIMEOUT: $t_base exceeded ${t_timeout}s"
              fi
              failed=$((failed + 1))
              failed_tests="$failed_tests $t_base"
            fi
            elapsed=$((SECONDS - t_start))
            echo "$t_base	$elapsed" >> "$GITHUB_WORKSPACE/$timing_log"
            echo "::timing::$t_base	${elapsed}s"
            rm -f "$test_output"
          done
          cd "$GITHUB_WORKSPACE"
          echo "=== Summary ==="
          echo "Passed: $passed, Failed: $failed"
          echo "=== Per-test timing ==="
          cat "$timing_log"
          if [ -n "$failed_tests" ]; then
            echo "Failed tests:$failed_tests"
          fi
          # Regression check (full runs only — the absolute baseline is
          # meaningless for a selected subset).
          # Baseline: 191 passes across 10 shards (~19 per shard).
          # Per-shard minimum: 12 (allows natural variance across shards).
          if [ "${{ needs.select.outputs.full }}" = "true" ]; then
            shard_min=12
            if [ "$passed" -lt "$shard_min" ]; then
              echo "REGRESSION: only $passed tests passed (minimum: $shard_min)"
              exit 1
            fi
          fi

      - name: Upload test timing
        if: always() && steps.shardtests.outputs.count != '0'
        # Informational artifact: a flaky artifact-service 403 must not fail
        # the shard after its tests already passed.
        continue-on-error: true
        uses: actions/upload-artifact@v5
        with:
          name: test-timing-shard-${{ matrix.shard }}
          path: test-timing-shard-${{ matrix.shard }}.tsv
          overwrite: true

      - name: Generate compat table
        if: always() && steps.shardtests.outputs.count != '0'
        run: |
          bash tools/generate-compat-table.sh > "compat-results-shard-${{ matrix.shard }}.md"
          cat "compat-results-shard-${{ matrix.shard }}.md"

      - name: Upload compat results
        if: always() && steps.shardtests.outputs.count != '0'
        # Informational artifact: a flaky artifact-service 403 must not fail
        # the shard after its tests already passed.
        continue-on-error: true
        uses: actions/upload-artifact@v5
        with:
          name: compat-results-shard-${{ matrix.shard }}
          path: compat-results-shard-${{ matrix.shard }}.md
          overwrite: true