Support cross-compilation for macOS #510
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build macOS arm64 | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| TAG_NAME: | |
| description: 'Release Version Tag' | |
| required: true | |
| release: | |
| types: [created] | |
| push: | |
| branches: | |
| - main | |
| paths-ignore: | |
| - '**/*.md' | |
| pull_request: | |
| types: [opened, synchronize, reopened, ready_for_review] | |
| branches: | |
| - main | |
| paths-ignore: | |
| - '**/*.md' | |
| jobs: | |
| build_universal_wheel_on_linux: | |
| name: Build on Linux (cross-compile for macOS arm64) | |
| runs-on: GH-Linux-ARM64 | |
| # if: ${{ !github.event.pull_request.draft }} | |
| timeout-minutes: 600 | |
| steps: | |
| - name: Install Python build dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y make build-essential libssl-dev zlib1g-dev \ | |
| libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm \ | |
| libncursesw5-dev xz-utils tk-dev libxml2-dev libxmlsec1-dev \ | |
| libffi-dev liblzma-dev p7zip-full | |
| - name: Upgrade Rust toolchain | |
| run: | | |
| rustup toolchain install nightly-2025-07-07 | |
| rustup default nightly-2025-07-07 | |
| rustup component add rust-src | |
| rustc --version | |
| cargo --version | |
| - name: Install clang++ for Ubuntu | |
| run: | | |
| pwd | |
| uname -a | |
| wget https://apt.llvm.org/llvm.sh | |
| chmod +x llvm.sh | |
| sudo ./llvm.sh 19 | |
| which clang++-19 | |
| clang++-19 --version | |
| sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget | |
| # Install WebAssembly linker (wasm-ld) | |
| sudo apt-get install -y lld-19 | |
| # Create symlink for wasm-ld | |
| if ! command -v wasm-ld &> /dev/null; then | |
| sudo ln -sf /usr/bin/wasm-ld-19 /usr/bin/wasm-ld || true | |
| fi | |
| which wasm-ld || echo "wasm-ld not found in PATH" | |
| ccache -s | |
| - name: Update git | |
| run: | | |
| sudo add-apt-repository ppa:git-core/ppa -y | |
| sudo apt-get update | |
| sudo apt-get install -y git | |
| git --version | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Update submodules | |
| run: | | |
| git submodule update --init --recursive --jobs 4 | |
| - name: ccache | |
| uses: hendrikmuhs/[email protected] | |
| with: | |
| key: ubuntu-24.04-aarch64-cross-compile | |
| max-size: 5G | |
| append-timestamp: true | |
| - name: remove old clang and link clang-19 to clang | |
| run: | | |
| sudo rm -f /usr/bin/clang || true | |
| sudo ln -s /usr/bin/clang-19 /usr/bin/clang | |
| sudo rm -f /usr/bin/clang++ || true | |
| sudo ln -s /usr/bin/clang++-19 /usr/bin/clang++ | |
| which clang++ | |
| clang++ --version | |
| - name: Run chdb/build_mac_on_linux.sh | |
| timeout-minutes: 600 | |
| run: | | |
| source ~/.cargo/env | |
| bash ./chdb/build_mac_on_linux.sh arm64 | |
| continue-on-error: false | |
| - name: Run chdb/build/build_static_lib_mac_on_linux.sh | |
| timeout-minutes: 600 | |
| run: | | |
| source ~/.cargo/env | |
| bash ./chdb/build/build_static_lib_mac_on_linux.sh arm64 | |
| continue-on-error: false | |
| - name: Check ccache statistics | |
| run: | | |
| ccache -s | |
| ls -lh chdb | |
| df -h | |
| - name: Keep killall ccache and wait for ccache to finish | |
| if: always() | |
| run: | | |
| sleep 60 | |
| while ps -ef | grep ccache | grep -v grep; do \ | |
| killall ccache; \ | |
| sleep 10; \ | |
| done | |
| - name: Upload build artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: macos-arm64-build-artifacts | |
| path: | | |
| ./libchdb.so | |
| ./libchdb.a | |
| ./chdb/_chdb.abi3.so | |
| ./chdb/libpybind11nonlimitedapi_stubs.dylib | |
| ./chdb/libpybind11nonlimitedapi_chdb_3.8.dylib | |
| ./chdb/libpybind11nonlimitedapi_chdb_3.9.dylib | |
| ./chdb/libpybind11nonlimitedapi_chdb_3.10.dylib | |
| ./chdb/libpybind11nonlimitedapi_chdb_3.11.dylib | |
| ./chdb/libpybind11nonlimitedapi_chdb_3.12.dylib | |
| ./chdb/libpybind11nonlimitedapi_chdb_3.13.dylib | |
| ./chdb/libpybind11nonlimitedapi_chdb_3.14.dylib | |
| retention-days: 1 | |
| test_on_macos: | |
| name: Test on macOS arm64 | |
| runs-on: macos-14-xlarge | |
| needs: build_universal_wheel_on_linux | |
| # if: ${{ !github.event.pull_request.draft }} | |
| timeout-minutes: 600 | |
| steps: | |
| - name: Check machine architecture | |
| run: | | |
| echo "=== Machine Architecture Information ===" | |
| echo "Machine type: $(uname -m)" | |
| echo "Architecture: $(arch)" | |
| echo "System info: $(uname -a)" | |
| echo "Hardware info:" | |
| system_profiler SPHardwareDataType | grep "Chip\|Processor" | |
| if sysctl -n hw.optional.arm64 2>/dev/null | grep -q "1"; then | |
| echo "This is an ARM64 (Apple Silicon) machine" | |
| else | |
| echo "This is an x86_64 (Intel) machine" | |
| fi | |
| - name: Setup pyenv | |
| run: | | |
| curl https://pyenv.run | bash | |
| export PATH="$HOME/.pyenv/bin:$PATH" | |
| eval "$(pyenv init -)" | |
| pyenv install 3.8:latest | |
| pyenv install 3.9:latest | |
| pyenv install 3.10:latest | |
| pyenv install 3.11:latest | |
| pyenv install 3.12:latest | |
| pyenv install 3.13:latest | |
| pyenv install 3.14:latest | |
| pyenv global 3.8 3.9 3.10 3.11 3.12 3.13 3.14 | |
| echo "Installed versions:" | |
| pyenv versions | |
| - name: Verify pyenv installations | |
| run: | | |
| export PATH="$HOME/.pyenv/bin:$PATH" | |
| eval "$(pyenv init -)" | |
| echo "Installed Python versions:" | |
| pyenv versions | |
| echo "" | |
| echo "Verifying all required Python versions are available:" | |
| for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do | |
| if ! pyenv versions --bare | grep -q "^$version"; then | |
| echo "ERROR: Python $version is not installed!" | |
| exit 1 | |
| fi | |
| echo "✓ Python $version is installed" | |
| done | |
| echo "All Python versions verified successfully!" | |
| - name: Install dependencies for all Python versions | |
| run: | | |
| export PATH="$HOME/.pyenv/bin:$PATH" | |
| eval "$(pyenv init -)" | |
| for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do | |
| echo "Installing dependencies for Python $version" | |
| pyenv shell $version | |
| python -m pip install --upgrade pip | |
| python -m pip install setuptools wheel tox pandas pyarrow twine psutil deltalake wheel>=0.40.0 jupyter nbconvert | |
| pyenv shell --unset | |
| done | |
| - name: Remove /usr/local/bin/python3 | |
| run: | | |
| sudo rm -f /usr/local/bin/python3 | |
| - name: Install go for macOS | |
| run: | | |
| brew update | |
| brew install go | |
| go version | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Update version for release | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| run: | | |
| export PATH="$HOME/.pyenv/bin:$PATH" | |
| eval "$(pyenv init -)" | |
| pyenv shell 3.9 | |
| # Install bump-my-version | |
| python -m pip install bump-my-version | |
| TAG_NAME=${GITHUB_REF#refs/tags/v} | |
| bump-my-version replace --new-version $TAG_NAME | |
| echo "Version files updated to $TAG_NAME" | |
| pyenv shell --unset | |
| - name: Download build artifacts | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: macos-arm64-build-artifacts | |
| path: ./artifacts | |
| - name: Restore artifacts to original paths | |
| run: | | |
| mv ./artifacts/libchdb.so ./ | |
| mv ./artifacts/libchdb.a ./ | |
| mv ./artifacts/chdb/_chdb.abi3.so ./chdb/ | |
| mv ./artifacts/chdb/libpybind11nonlimitedapi_stubs.dylib ./chdb/ | |
| for v in 8 9 10 11 12 13 14; do | |
| mv ./artifacts/chdb/libpybind11nonlimitedapi_chdb_3.${v}.dylib ./chdb/ | |
| done | |
| ls -lh ./libchdb.so ./libchdb.a | |
| ls -lh ./chdb/*.so ./chdb/*.dylib | |
| - name: Run chdb/test_smoke.sh | |
| timeout-minutes: 600 | |
| run: | | |
| export PATH="$HOME/.pyenv/bin:$PATH" | |
| eval "$(pyenv init -)" | |
| pyenv shell 3.8 | |
| bash gen_manifest.sh | |
| bash -x ./chdb/test_smoke.sh cross-compile | |
| - name: Run chdb/build/test_go_example.sh | |
| timeout-minutes: 600 | |
| run: | | |
| bash ./chdb/build/test_go_example.sh ${{ github.workspace }}/libchdb.a | |
| continue-on-error: false | |
| - name: Run libchdb stub in examples dir | |
| run: | | |
| bash -x ./examples/runStub.sh | |
| bash -x ./examples/runArrowTest.sh | |
| - name: Build wheels | |
| run: | | |
| rm -rf chdb/build/ | |
| export PATH="$HOME/.pyenv/bin:$PATH" | |
| eval "$(pyenv init -)" | |
| pyenv shell 3.8 | |
| make wheel | |
| - name: Fix wheel platform tag | |
| run: | | |
| export PATH="$HOME/.pyenv/bin:$PATH" | |
| eval "$(pyenv init -)" | |
| pyenv shell 3.8 | |
| python -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/*.whl | |
| - name: Verify wheel sizes | |
| run: | | |
| echo "=== Wheel sizes ===" | |
| du -sh dist/* | |
| - name: Setup core dump collection | |
| run: | | |
| mkdir -p tmp/core | |
| sudo sysctl kern.corefile=$PWD/tmp/core/core.%P | |
| sudo sysctl kern.coredump=1 | |
| ulimit -c unlimited | |
| - name: Test wheel on all Python versions | |
| run: | | |
| ulimit -c unlimited | |
| export PATH="$HOME/.pyenv/bin:$PATH" | |
| eval "$(pyenv init -)" | |
| for version in 3.8 3.9 3.10 3.11 3.12 3.13 3.14; do | |
| echo "Testing chdb on Python $version" | |
| pyenv shell $version | |
| python -m pip install dist/*.whl --force-reinstall --no-cache-dir | |
| python -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(f'Python $version: {res}')" | |
| make test | |
| python -m pip uninstall -y chdb | |
| pyenv shell --unset | |
| done | |
| continue-on-error: false | |
| - name: Run notebook tests | |
| run: | | |
| export PATH="$HOME/.pyenv/bin:$PATH" | |
| eval "$(pyenv init -)" | |
| pyenv shell 3.8 | |
| python -m pip install dist/*.whl --force-reinstall | |
| jupyter nbconvert --to notebook --execute tests/test_data_insertion.ipynb --output test_data_insertion_output.ipynb | |
| pyenv shell --unset | |
| continue-on-error: false | |
| - name: Check and upload core files if present | |
| if: always() | |
| run: | | |
| if ls tmp/core/core.* >/dev/null 2>&1; then | |
| echo "CORE_FILES_FOUND=true" >> $GITHUB_ENV | |
| tar -czvf core-files-macos-arm64.tar.gz tmp/core/core.* | |
| echo "Core files tar created: core-files-macos-arm64.tar.gz" | |
| ls -lh core-files-macos-arm64.tar.gz | |
| else | |
| echo "CORE_FILES_FOUND=false" >> $GITHUB_ENV | |
| echo "No core files found in tmp/core" | |
| fi | |
| continue-on-error: true | |
| - name: Upload core files artifact | |
| if: always() && env.CORE_FILES_FOUND == 'true' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: core-files-macos-arm64 | |
| path: core-files-macos-arm64.tar.gz | |
| - name: Show files | |
| run: ls -lh dist | |
| shell: bash | |
| - name: Upload wheels to release | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| run: | | |
| gh release upload ${{ github.ref_name }} dist/*.whl --clobber | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} | |
| - name: Packege libchdb.so | |
| run: | | |
| cp programs/local/chdb.h chdb.h | |
| cp programs/local/chdb.hpp chdb.hpp | |
| tar -czvf macos-arm64-libchdb.tar.gz libchdb.so chdb.h chdb.hpp | |
| - name: Package libchdb.a | |
| run: | | |
| cp programs/local/chdb.h chdb.h | |
| cp programs/local/chdb.hpp chdb.hpp | |
| tar -czvf macos-arm64-libchdb-static.tar.gz libchdb.a chdb.h chdb.hpp | |
| - name: Upload libchdb.so to release | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| run: | | |
| gh release upload ${{ github.ref_name }} macos-arm64-libchdb.tar.gz --clobber | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} | |
| - name: Upload libchdb.a to release | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| run: | | |
| gh release upload ${{ github.ref_name }} macos-arm64-libchdb-static.tar.gz --clobber | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} | |
| - uses: actions/upload-artifact@v4 | |
| with: | |
| name: chdb-artifacts-macos-arm64 | |
| path: | | |
| ./dist/*.whl | |
| ./macos-arm64-libchdb.tar.gz | |
| ./macos-arm64-libchdb-static.tar.gz | |
| overwrite: true | |
| - name: Upload pypi | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| run: | | |
| export PATH="$HOME/.pyenv/bin:$PATH" | |
| eval "$(pyenv init -)" | |
| pyenv shell 3.8 | |
| python -m twine upload dist/*.whl | |
| env: | |
| TWINE_USERNAME: __token__ | |
| TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} |