Rollback dependency in pyproject.toml

Author: ktsitsi

.github/workflows/ci-r-python-interop.yml

@@ -104,7 +104,7 @@ jobs:
         with:
           python-version: "3.11"
           cache: pip
-          cache-dependency-path: ./apis/python/setup.py
+          cache-dependency-path: ./apis/python/pyproject.toml
 
       - name: Install tiledbsoma
         # TILEDB_REMOVE_DEPRECATIONS is already set to ON in pyproject.toml

.github/workflows/python-packaging.yml

@@ -25,7 +25,7 @@ jobs:
 
       - name: Install dependencies
         run: |
-          pip install --upgrade 'scikit-build-core>=0.10.0' 'pybind11[global]>=2.10.0' 'cmake>=3.21,<4' 'setuptools>=70.1' 'setuptools-scm>=8.0.0'
+          pip install --upgrade 'build' 'scikit-build-core>=0.10.0' 'pybind11[global]>=2.10.0' 'cmake>=3.21,<4' 'setuptools>=70.1' 'setuptools-scm>=8.0.0'
           pip list
 
       - name: Build sdist
@@ -153,15 +153,13 @@ jobs:
         env:
           CIBW_BUILD: ${{ matrix.python_name }}-${{ matrix.platform }}
           CIBW_BUILD_VERBOSITY: 3
-          CIBW_BEFORE_BUILD_LINUX: ${{ env.CIBW_BEFORE_BUILD_LINUX || '' }}
-          CIBW_ENVIRONMENT_LINUX: VCPKG_FORCE_SYSTEM_BINARIES=${{ env.VCPKG_FORCE_SYSTEM_BINARIES || '' }}
+          # Don't pass CMAKE_TOOLCHAIN_FILE from host - it will be set by setup_build_env.sh
+          # inside the container after vcpkg is bootstrapped
+          CIBW_ENVIRONMENT_LINUX: "VCPKG_FORCE_SYSTEM_BINARIES=1"
+          CIBW_ENVIRONMENT_MACOS: "MACOSX_DEPLOYMENT_TARGET=13.3"
           CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
           CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
           CIBW_ARCHS_MACOS: "x86_64 arm64"
-          CMAKE_TOOLCHAIN_FILE: ${{ env.CMAKE_TOOLCHAIN_FILE }}
-          VCPKG_TARGET_TRIPLET: ${{ env.VCPKG_TARGET_TRIPLET }}
-          CMAKE_OSX_ARCHITECTURES: ${{ env.CMAKE_OSX_ARCHITECTURES || '' }}
-          MACOSX_DEPLOYMENT_TARGET: "13.3"
 
       - name: Check wheel name
         run: ls -l ./wheelhouse/*.whl

.github/workflows/python-remote-storage.yml

@@ -59,7 +59,7 @@ jobs:
         with:
           python-version: ${{ matrix.python_version }}
           cache: pip
-          cache-dependency-path: ./apis/python/setup.py
+          cache-dependency-path: ./apis/python/pyproject.toml
 
       - name: Check out TileDB-SOMA
         uses: actions/checkout@v4

.github/workflows/test-python.yml

@@ -67,7 +67,7 @@ jobs:
         with:
           python-version: ${{ inputs.python_version }}
           cache: pip
-          cache-dependency-path: ./apis/python/setup.py
+          cache-dependency-path: ./apis/python/pyproject.toml
 
       - name: Install tiledbsoma
         # TILEDB_REMOVE_DEPRECATIONS is already set to ON in pyproject.toml

apis/python/CMakeLists.txt

@@ -1,15 +1,19 @@
 cmake_minimum_required(VERSION 3.21)
+
+# set(PYBIND11_FINDPYTHON ON)
+
+
 project(tiledbsoma-python LANGUAGES CXX)
 
 # Set C++ standard
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 # Find pybind11
-find_package(pybind11 REQUIRED)
+# find_package(pybind11 REQUIRED)
 
 # Find Python
-find_package(Python COMPONENTS Interpreter Development REQUIRED)
+# find_package(Python COMPONENTS Interpreter Development REQUIRED)
 
 # Get version from setuptools_scm generated file or fallback
 # setuptools_scm writes version to src/tiledbsoma/_version.py if configured

apis/python/MANIFEST.in

@@ -13,8 +13,8 @@ include src/tiledbsoma/*.h
 # Python package directory (includes all .py files and subdirectories)
 graft src/tiledbsoma
 
-# Requirements files
-include requirements_*.txt
+# Requirements files are now inlined in pyproject.toml
+# No need to include them separately
 
 # Exclude build artifacts and cache files
 prune src/tiledbsoma/__pycache__

apis/python/pyproject.toml

@@ -51,9 +51,48 @@ dependencies = [
 dynamic = ["version"]
 
 [project.optional-dependencies]
-dev = {file = "requirements_dev.txt"}
-spatial-io = {file = "requirements_spatial.txt"}
-all = {file = ["requirements_dev.txt", "requirements_spatial.txt"]}
+# Development dependencies
+dev = [
+    "black",
+    "dask",
+    "pytest",
+    "pytest-cov",
+    "ruff",
+    "sparse",
+    "typeguard==4.4.2",
+    "types-setuptools",
+    "more-itertools",
+    "hypothesis",
+    "deepdiff",
+]
+# Spatial I/O dependencies
+spatial-io = [
+    "geopandas",
+    "tifffile",
+    "pillow",
+    "spatialdata>=0.2.5",
+    "xarray",
+    "dask",
+]
+# All optional dependencies (dev + spatial-io)
+all = [
+    "black",
+    "dask",
+    "pytest",
+    "pytest-cov",
+    "ruff",
+    "sparse",
+    "typeguard==4.4.2",
+    "types-setuptools",
+    "more-itertools",
+    "hypothesis",
+    "deepdiff",
+    "geopandas",
+    "tifffile",
+    "pillow",
+    "spatialdata>=0.2.5",
+    "xarray",
+]
 
 [tool.scikit-build]
 # CMake configuration
@@ -64,6 +103,8 @@ build-dir = "build"
 [tool.scikit-build.cmake]
 # CMake version requirement (use version instead of minimum-version for scikit-build-core >= 0.8)
 version = ">=3.21,<4"  # CMake 4 builds are broken on ARM Linux: https://github.com/single-cell-data/TileDB-SOMA/issues/3890
+# scikit-build-core automatically picks up CMAKE_TOOLCHAIN_FILE from environment
+# We set it in before-build hooks via setup_build_env.sh
 
 [tool.scikit-build.cmake.define]
 # CMake build type - Release for production builds
@@ -99,11 +140,19 @@ manylinux-x86_64-image = "manylinux_2_28"
 manylinux-aarch64-image = "manylinux_2_28"
 
 # Install system dependencies and bootstrap vcpkg
+# Note: vcpkg is checked out inside the container since cibuildwheel only copies apis/python/
+# {project} is the temporary directory where cibuildwheel copies the package
 before-all = [
     "yum install -y git ninja-build perl-IPC-Cmd zip autoconf automake libtool pkg-config",
-    "bash {project}/apis/python/scripts/setup_build_env.sh",
+    "git clone --depth 1 https://github.com/microsoft/vcpkg.git {project}/../vcpkg",
+    "{project}/../vcpkg/bootstrap-vcpkg.sh",
 ]
 
+# Set up vcpkg environment before each build
+# This runs for each Python version and sets environment variables that persist to the build
+# We use eval to execute export commands from the script so they persist to the build step
+before-build = "export VCPKG_ROOT={project}/../vcpkg && {project}/apis/python/scripts/setup_build_env.sh"
+
 # Environment variables for vcpkg build
 environment = {VCPKG_FORCE_SYSTEM_BINARIES = "1"}
 
@@ -112,10 +161,13 @@ environment = {VCPKG_FORCE_SYSTEM_BINARIES = "1"}
 environment = {MACOSX_DEPLOYMENT_TARGET = "13.3"}
 
 # Install system dependencies and bootstrap vcpkg
+# Note: vcpkg is checked out inside the container since cibuildwheel only copies apis/python/
+# {project} is the temporary directory where cibuildwheel copies the package
 before-all = [
     "brew update || true",
     "brew install automake llvm ninja git || true",
-    "bash {project}/apis/python/scripts/setup_build_env.sh",
+    "git clone --depth 1 https://github.com/microsoft/vcpkg.git {project}/../vcpkg",
+    "{project}/../vcpkg/bootstrap-vcpkg.sh",
 ]
 
 [tool.cibuildwheel.windows]
@@ -136,7 +188,8 @@ before-all = [
 # This allows setuptools_scm to find git tags from the repository root
 root = "../.."
 # Write version to a file for CMake to read and for runtime access
-# When root is set, write_to is relative to the root (repository root)
+# When root is set, write_to is resolved relative to the root directory
+# So we need to specify the full path from the root
 write_to = "apis/python/src/tiledbsoma/_version.py"
 # Use git to find version (default behavior)
 # This will work with tags like "1.0.0", "v1.0.0", etc.

apis/python/requirements_dev.txt

@@ -10,31 +10,39 @@ PYTHON_DIR="$(cd "${SCRIPT_DIR}/.." && pwd)"
 REPO_ROOT="$(cd "${PYTHON_DIR}/../.." && pwd)"
 
 # Check if vcpkg is already set up
-if [ -n "${VCPKG_ROOT:-}" ]; then
-    echo "Using existing VCPKG_ROOT: ${VCPKG_ROOT}"
-    export CMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake"
-elif [ -n "${CMAKE_TOOLCHAIN_FILE:-}" ]; then
+if [ -n "${CMAKE_TOOLCHAIN_FILE:-}" ] && [ -f "${CMAKE_TOOLCHAIN_FILE}" ]; then
+    # CMAKE_TOOLCHAIN_FILE is already set and exists - use it
     echo "Using existing CMAKE_TOOLCHAIN_FILE: ${CMAKE_TOOLCHAIN_FILE}"
     export VCPKG_ROOT="$(dirname "$(dirname "${CMAKE_TOOLCHAIN_FILE}")")"
+elif [ -n "${VCPKG_ROOT:-}" ] && [ -d "${VCPKG_ROOT}" ]; then
+    # VCPKG_ROOT is set and exists - use it
+    echo "Using existing VCPKG_ROOT: ${VCPKG_ROOT}"
+    export CMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake"
 else
     # Try to find vcpkg in common locations
     if [ -d "${REPO_ROOT}/vcpkg" ]; then
         export VCPKG_ROOT="${REPO_ROOT}/vcpkg"
+    elif [ -d "${PYTHON_DIR}/../vcpkg" ]; then
+        # For cibuildwheel: vcpkg is cloned one level up from apis/python
+        export VCPKG_ROOT="${PYTHON_DIR}/../vcpkg"
     elif [ -d "${HOME}/vcpkg" ]; then
         export VCPKG_ROOT="${HOME}/vcpkg"
     else
         echo "Error: vcpkg not found. Please set VCPKG_ROOT or CMAKE_TOOLCHAIN_FILE environment variable."
+        echo "Searched locations:"
+        echo "  - ${REPO_ROOT}/vcpkg"
+        echo "  - ${PYTHON_DIR}/../vcpkg"
+        echo "  - ${HOME}/vcpkg"
         exit 1
     fi
 
     # Bootstrap vcpkg if needed
     if [ ! -f "${VCPKG_ROOT}/vcpkg" ] && [ ! -f "${VCPKG_ROOT}/vcpkg.exe" ]; then
         echo "Bootstrapping vcpkg..."
-        if [ "$(uname)" == "Darwin" ]; then
-            "${VCPKG_ROOT}/bootstrap-vcpkg.sh"
-        else
-            "${VCPKG_ROOT}/bootstrap-vcpkg.sh"
-        fi
+        "${VCPKG_ROOT}/bootstrap-vcpkg.sh" || {
+            echo "Error: Failed to bootstrap vcpkg"
+            exit 1
+        }
     fi
 
     export CMAKE_TOOLCHAIN_FILE="${VCPKG_ROOT}/scripts/buildsystems/vcpkg.cmake"
@@ -67,6 +75,18 @@ if [ "$(uname)" == "Darwin" ] && [ -z "${CMAKE_OSX_ARCHITECTURES:-}" ]; then
     export CMAKE_OSX_ARCHITECTURES="$(uname -m)"
 fi
 
+# If -x flag is passed, output export commands instead of executing them
+# This allows the script to be sourced or eval'd to set environment variables
+if [ "${1:-}" = "-x" ]; then
+    echo "export VCPKG_ROOT=\"${VCPKG_ROOT}\""
+    echo "export CMAKE_TOOLCHAIN_FILE=\"${CMAKE_TOOLCHAIN_FILE}\""
+    echo "export VCPKG_TARGET_TRIPLET=\"${VCPKG_TARGET_TRIPLET}\""
+    if [ "$(uname)" == "Darwin" ] && [ -n "${CMAKE_OSX_ARCHITECTURES:-}" ]; then
+        echo "export CMAKE_OSX_ARCHITECTURES=\"${CMAKE_OSX_ARCHITECTURES}\""
+    fi
+    exit 0
+fi
+
 echo "VCPKG_ROOT: ${VCPKG_ROOT}"
 echo "CMAKE_TOOLCHAIN_FILE: ${CMAKE_TOOLCHAIN_FILE}"
 echo "VCPKG_TARGET_TRIPLET: ${VCPKG_TARGET_TRIPLET}"