ci.yml

name: CI

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}

on:
  push:
    branches: ["develop"]
  pull_request: {}
  workflow_dispatch: {}

permissions:
  actions: read
  contents: read
  checks: write  # audit-check creates checks
  issues: write  # audit-check creates issues

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1

jobs:
  lint-toml:
    runs-on: ubuntu-latest
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v5
      - uses: spiraldb/actions/.github/actions/lint-toml@0.15.0

  validate-workflow-yaml:
    runs-on: ubuntu-latest
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v5
      - name: Validate YAML file
        run: |
          # Lint the workflows and yamllint's configuration file.
          yamllint \
            --strict \
            -c yamllint-config.yaml \
            yamllint-config.yaml \
            .github/

  python-lint:
    name: "Python (lint)"
    runs-on: ubuntu-latest
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v5
      - name: Install uv
        uses: spiraldb/actions/.github/actions/setup-uv@0.15.0
        with:
          sync: false
          prune-cache: false
      - name: Python Lint - Format
        run: uv run ruff format --check .
      - name: Python Lint - Ruff
        run: uv run ruff check .
      - name: Python Lint - PyRight
        run: uv run basedpyright vortex-python

  python-test:
    name: "Python (test)"
    runs-on: ubuntu-latest
    timeout-minutes: 120
    env:
      RUST_LOG: "info,uv=debug"
    steps:
      - uses: actions/checkout@v5
      - uses: ./.github/actions/setup-rust
      - name: Install uv
        uses: spiraldb/actions/.github/actions/setup-uv@0.15.0
        with:
          sync: false
          prune-cache: false

      - name: Pytest - Vortex
        run: |
          uv run --all-packages pytest --benchmark-disable test/
        working-directory: vortex-python/

      - name: Doctest - PyVortex
        run: |
          uv run --all-packages make doctest
        working-directory: docs/

      - name: Ensure docs build - PyVortex
        run: |
          uv run --all-packages make html
        working-directory: docs/

      - uses: mlugg/setup-zig@v2
      - name: Ensure wheel and sdist can be built on Linux - PyVortex
        shell: bash
        run: |
          echo "Clearing wheel target directory"
          rm -rf ../target/wheels/

          uv venv
          uv pip install "maturin[zig]"
          uv tool run maturin build --interpreter python3.11 --zig
          uv tool run maturin build --interpreter python3.11 --zig --sdist

          file_count=$(ls -1 ../target/wheels/ | wc -l)

          if [[ $file_count -ne 2 ]]; then
            echo "Unexpected number of files detected ${file_count}:"
            ls ../target/wheels/
            exit 1
          else
            echo "Generated two files"
          fi
        working-directory: vortex-python/

  rust-docs:
    name: "Rust (docs)"
    timeout-minutes: 120
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=m7i+m7i-flex+m7a
      - cpu=8
      - image=ubuntu24-full-x64
      - extras=s3-cache
      - tag=rust-docs
    steps:
      - uses: runs-on/action@v2
        with:
          sccache: s3
      - uses: actions/checkout@v5
      - uses: ./.github/actions/setup-rust
      - name: Docs
        run: |
          RUSTDOCFLAGS="-D warnings" cargo doc --no-deps
          # nextest doesn't support doc tests, so we run it here
          cargo test --doc --workspace --all-features --exclude vortex-cxx --exclude vortex-jni --exclude vortex-ffi --exclude xtask --no-fail-fast

  build-rust:
    name: "Rust build (${{matrix.config.name}})"
    timeout-minutes: 120
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=m7i+m7i-flex+m7a
      - cpu=8
      - image=ubuntu24-full-x64
      - extras=s3-cache
      - tag=${{ matrix.config.name }}
    env:
      # disable lints for build, they will be caught in Rust lint job.
      RUSTFLAGS: "-A warnings"
    strategy:
      fail-fast: false
      matrix:
        config:
          - name: "all-features"
            command: "build"
            args: "--all-features --all-targets"
          - name: "default features"
            command: "build"
            args: "--all-targets"
          - name: "with tokio dispatcher"
            command: "build"
            args: "--no-default-features --features tokio --all-targets --ignore-unknown-features"
          - name: "with compio dispatcher"
            command: "build"
            args: "--no-default-features --features compio --all-targets --ignore-unknown-features"
          - name: "with tokio+compio dispatcher"
            command: "build"
            args: "--no-default-features --features tokio,compio --all-targets --ignore-unknown-features"
          - name: "wasm32 with default features"
            command: "build"
            target: wasm32-unknown-unknown
            env:
              rustflags: "RUSTFLAGS='-A warnings --cfg getrandom_backend=\"wasm_js\"'"
            args: "--target wasm32-unknown-unknown --exclude vortex --exclude vortex-datafusion --exclude vortex-duckdb --exclude vortex-tui --exclude vortex-zstd"
    steps:
      - uses: runs-on/action@v2
        with:
          sccache: s3
      - uses: actions/checkout@v5
      - uses: ./.github/actions/setup-rust
      - name: Install wasm32 target
        if: ${{ matrix.config.target == 'wasm32-unknown-unknown' }}
        run: rustup target add wasm32-unknown-unknown
      - name: Install cargo-hack
        uses: taiki-e/install-action@cargo-hack
      - name: Rust Build (${{matrix.config.name}})
        run: ${{matrix.config.env.rustflags}} cargo hack ${{matrix.config.command}} --locked ${{matrix.config.args}} --ignore-private
      - name: "Make sure no files changed after build"
        run: |
          git status --porcelain
          test -z "$(git status --porcelain)"

  check-min-deps:
    name: "Check build with minimal dependencies"
    timeout-minutes: 120
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=m7i+m7i-flex+m7a
      - cpu=8
      - image=ubuntu24-full-x64
      - extras=s3-cache
      - tag=rust-min-deps
    steps:
      - uses: runs-on/action@v2
        with:
          sccache: s3
      - uses: actions/checkout@v5
      - uses: ./.github/actions/setup-rust
      - uses: taiki-e/install-action@cargo-hack
      - uses: taiki-e/install-action@cargo-minimal-versions
      - run: cargo minimal-versions check --direct --workspace --ignore-private

  rust-lint:
    name: "Rust (lint)"
    timeout-minutes: 120
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=m7i+m7i-flex+m7a
      - cpu=16
      - image=ubuntu24-full-x64
      - extras=s3-cache
      - tag=rust-lint
    steps:
      - uses: runs-on/action@v2
        with:
          sccache: s3
      - uses: actions/checkout@v5
      - uses: ./.github/actions/setup-rust
        with:
          toolchain: nightly
      - name: Rust Lint - Format
        run: cargo +nightly fmt --all --check
      - name: Rustc check
        run: cargo check --locked --all-features --all-targets
      - name: Rust Lint - Clippy All Features
        run: cargo clippy --locked --all-features --all-targets -- -D warnings
      - name: Rust Lint - Clippy Default Features
        run: cargo clippy --locked --all-targets -- -D warnings

  rust-lint-no-default:
    name: "Rust (lint, no default)"
    timeout-minutes: 120
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=m7i+m7i-flex+m7a
      - cpu=16
      - image=ubuntu24-full-x64
      - extras=s3-cache
      - tag=rust-lint-no-default
    steps:
      - uses: runs-on/action@v2
        with:
          sccache: s3
      - uses: actions/checkout@v5
      - uses: ./.github/actions/setup-rust
      - name: Install cargo-hack
        uses: taiki-e/install-action@cargo-hack
      - name: Rust Lint - Clippy No Default Features
        shell: bash
        run: |
          # https://spiraldb.slack.com/archives/C07BV3GKAJ2/p1732736281946729
          cargo hack clippy --no-default-features

  rust-coverage:
    name: "Rust tests (coverage)"
    timeout-minutes: 120
    permissions:
      id-token: write
    strategy:
      matrix:
        suite:
          - tests
          - tpc-h
          - ffi
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=m7i+m7i-flex+m7a
      - cpu=8
      - image=ubuntu24-full-x64
      - extras=s3-cache
      - tag=rust-coverage
      - tag=suite-${{ matrix.suite }}
    env:
      RUSTFLAGS: "-Cinstrument-coverage -A warnings"
      CARGO_INCREMENTAL: 0  # Disable incremental compilation to get accurate coverage
      LLVM_PROFILE_FILE: "target/coverage/vortex-%p-%m.profraw"
      GRCOV_OUTPUT_FILE: "target/coverage/vortex.lcov"
    steps:
      - uses: runs-on/action@v2
        with:
          sccache: s3
      - uses: actions/checkout@v5
      - uses: ./.github/actions/setup-rust
      - name: Install grcov
        uses: taiki-e/install-action@grcov
      - name: Install nextest
        uses: taiki-e/install-action@v2
        with:
          tool: nextest@0.9.98
      - name: Rust Tests
        if: ${{ matrix.suite == 'tests' }}
        run: |
          cargo +nightly nextest run --locked --workspace --all-features --no-fail-fast
      - name: Run TPC-H
        if: ${{ matrix.suite == 'tpc-h' }}
        run: |
          cargo run --bin query_bench -- tpch -i2 --scale-factor 0.1
      - name: Run FFI Example
        if: ${{ matrix.suite == 'ffi' }}
        run: |
          cargo build -p vortex-ffi
          cargo run -p vortex-ffi --example hello_vortex
      - name: Generate coverage report
        run: |
          rustup component add llvm-tools-preview
          grcov . --binary-path target/debug/ -s . -t lcov --llvm --ignore-not-existing \
            --ignore '../*' --ignore '/*' --ignore 'fuzz/*' --ignore 'bench-vortex/*' \
            --ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \
            --ignore 'vortex-python/*' --ignore 'vortex-jni/*' --ignore 'vortex-flatbuffers/*' \
            --ignore 'vortex-proto/*' --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \
            --ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' --ignore 'vortex-cxx/*' \
            -o ${{ env.GRCOV_OUTPUT_FILE }}
      - name: Codecov
        uses: codecov/codecov-action@v5
        with:
          name: run-${{ matrix.suite }}
          files: ${{ env.GRCOV_OUTPUT_FILE }}
          disable_search: true
          flags: ${{ matrix.suite }}
          use_oidc: true

  rust-test:
    name: "Rust tests (sanitizer)"
    timeout-minutes: 120
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=m7i+m7i-flex+m7a
      - cpu=8
      - image=ubuntu24-full-x64
      - extras=s3-cache
      - tag=rust-test-sanitizer
    env:
      # Add debug symbols and enable ASAN/LSAN with better output
      RUSTFLAGS: "-A warnings -Zsanitizer=address -Zsanitizer=leak --cfg disable_loom -C debuginfo=2 -C opt-level=0 -C strip=none"
      ASAN_OPTIONS: "symbolize=1:print_stats=1:check_initialization_order=1:detect_leaks=1:halt_on_error=0:verbosity=1:leak_check_at_exit=1"
      LSAN_OPTIONS: "verbosity=1:report_objects=1"
      ASAN_SYMBOLIZER_PATH: "/usr/bin/llvm-symbolizer"
      # Keep frame pointers for better stack traces
      CARGO_PROFILE_DEV_DEBUG: "true"
      CARGO_PROFILE_TEST_DEBUG: "true"
    steps:
      - uses: runs-on/action@v2
        with:
          sccache: s3
      - uses: actions/checkout@v5
      - name: Install llvm
        uses: aminya/setup-cpp@v1
        with:
          compiler: llvm
      - uses: ./.github/actions/setup-rust
        with:
          toolchain: nightly
          components: "rust-src, rustfmt, clippy, llvm-tools-preview"
      - name: Install nextest
        uses: taiki-e/install-action@v2
        with:
          tool: nextest@0.9.98
      - name: Rust Tests
        run: |
          # Build with full debug info first (helps with caching)
          cargo +nightly build --locked --workspace --all-features --target x86_64-unknown-linux-gnu
          # Run tests with sanitizers and debug output
          cargo +nightly nextest run \
            --locked \
            --workspace \
            --all-features \
            --no-fail-fast \
            --target x86_64-unknown-linux-gnu \
            --verbose

  build-java:
    name: "Java"
    runs-on: ubuntu-latest
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v5
      - uses: actions/setup-java@v5
        with:
          distribution: "corretto"
          java-version: "17"
      - uses: ./.github/actions/setup-rust
      - run: ./gradlew test --parallel
        working-directory: ./java

  bench-codspeed:
    strategy:
      matrix:
        shard: [1, 2]
    name: "Benchmark with Codspeed (Shard #${{ matrix.shard }})"
    timeout-minutes: 120
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=c6id.8xlarge
      - extras=s3-cache
      - image=ubuntu24-full-x64
      - tag=bench-codspeed
    steps:
      - uses: runs-on/action@v2
        with:
          sccache: s3
      - uses: actions/checkout@v5
      - uses: ./.github/actions/setup-rust

      - name: Install Codspeed
        shell: bash
        run: cargo install --force cargo-codspeed --locked

      - name: Build benchmarks (shard 1)
        env:
          RUSTFLAGS: "-C target-feature=+avx2"
        if: ${{ matrix.shard == 1 }}
        run: |
          cargo codspeed build --features test-harness \
            -p vortex-buffer \
            -p vortex-array \
            -p vortex \
            -p vortex-fastlanes \
            --profile bench

      - name: Build benchmarks (shard 2)
        env:
          RUSTFLAGS: "-C target-feature=+avx2"
        if: ${{ matrix.shard == 2 }}
        run: |
          cargo codspeed build --features test-harness \
            --exclude bench-vortex \
            --exclude vortex-datafusion \
            --exclude vortex-duckdb \
            --exclude vortex-fuzz \
            --exclude vortex-python \
            --exclude vortex-tui \
            --exclude xtask \
            --exclude vortex-buffer \
            --exclude vortex-array \
            --exclude vortex \
            --exclude vortex-fastlanes \
            --workspace \
            --profile bench

      - name: Run benchmarks
        uses: CodSpeedHQ/action@v3
        with:
          run: cargo codspeed run
          runner-version: 3.4.1-beta.1
          token: ${{ secrets.CODSPEED_TOKEN }}

  license-check-and-audit-check:
    name: License Check and Audit Check
    runs-on: ubuntu-latest
    timeout-minutes: 120
    strategy:
      matrix:
        checks:
          - advisories
          - bans licenses sources
    # Prevent sudden announcement of a new advisory from failing ci:
    continue-on-error: ${{ matrix.checks == 'advisories' }}
    steps:
      - uses: actions/checkout@v5
      - uses: EmbarkStudios/cargo-deny-action@v2
        with:
          command: check ${{ matrix.checks }}

  cxx-test:
    name: "C++ build"
    timeout-minutes: 120
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=m7i+m7i-flex+m7a
      - cpu=8
      - image=ubuntu24-full-x64
      - extras=s3-cache
      - tag=cxx-build
    steps:
      - uses: actions/checkout@v5
      - uses: ./.github/actions/setup-rust
      - name: Build and run C++ unit tests
        working-directory: vortex-cxx/
        run: |
          mkdir -p build
          cd build
          cmake -DVORTEX_ENABLE_TESTING=ON -DVORTEX_ENABLE_ASAN=ON ..
          cmake --build . --parallel $(nproc)
          ctest -V
      - name: Build and run the example in release mode
        working-directory: vortex-cxx/examples
        run: |
          mkdir -p build
          cd build
          cmake -DCMAKE_BUILD_TYPE=Release ..
          cmake --build . --parallel $(nproc)
          ./hello-vortex ../goldenfiles/example.vortex

  wasm-integration:
    name: "wasm-integration"
    runs-on: ubuntu-latest
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v5
      - uses: ./.github/actions/setup-rust
        with:
          toolchain: nightly-2025-06-26
          targets: "wasm32-wasip1"
          components: "rust-src"
      - name: Setup Wasmer
        uses: wasmerio/setup-wasmer@v3.1
        # there is a compiler bug in nightly (but not in nightly-2025-06-26)
      - run: cargo +nightly-2025-06-26 -Zbuild-std=panic_abort,std build --target wasm32-wasip1
        working-directory: ./wasm-test
      - run: wasmer run ./target/wasm32-wasip1/debug/wasm-test.wasm
        working-directory: ./wasm-test

  miri:
    name: "Rust tests (miri)"
    runs-on: ubuntu-latest
    timeout-minutes: 120
    env:
      MIRIFLAGS: -Zmiri-strict-provenance -Zmiri-symbolic-alignment-check -Zmiri-disable-isolation -Zmiri-env-forward=RUST_BACKTRACE
      RUSTFLAGS: "-A warnings"
      RUST_BACKTRACE: full
    steps:
      - uses: actions/checkout@v5
      - uses: ./.github/actions/setup-rust
        with:
          toolchain: nightly
          components: "rust-src, rustfmt, clippy, miri"
      - uses: taiki-e/install-action@v2
        with:
          tool: nextest@0.9.98
      - name: Run Miri
        run: cargo +nightly miri nextest run --no-fail-fast -p vortex-buffer -p vortex-ffi

  generated-files:
    name: "Check generated source files are up to date"
    runs-on: ubuntu-latest
    timeout-minutes: 120
    steps:
      - uses: actions/checkout@v5
      - uses: ./.github/actions/setup-rust
        with:
          toolchain: nightly
      - uses: ./.github/actions/setup-flatc
      - name: Install Protoc
        uses: arduino/setup-protoc@v3
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: "regenerate all .fbs/.proto Rust code"
        run: |
          cargo xtask generate-fbs
          cargo xtask generate-proto
      - name: "regenerate FFI header file"
        run: |
          cargo +nightly build -p vortex-ffi
      - name: "Make sure no files changed after regenerating"
        run: |
          git status --porcelain
          test -z "$(git status --porcelain)"

      - name: "Checkout develop flatbuffers"
        working-directory: vortex-flatbuffers/
        run: |
          cp -R flatbuffers flatbuffers.HEAD
          git fetch origin develop --depth 1
          git checkout origin/develop -- flatbuffers
      - name: "Verify flatbuffer back-compat"
        working-directory: vortex-flatbuffers/
        run: |
          find flatbuffers/ -type f -name "*.fbs" | sed 's/^flatbuffers\///' | xargs -I{} -n1 flatc -I flatbuffers.HEAD --conform-includes flatbuffers --conform flatbuffers/{} flatbuffers.HEAD/{}