ci: replace conda testing with pixi (#554)

Author: cpcloud

.github/workflows/conda-python-tests.yaml

@@ -113,6 +113,15 @@ jobs:
       - name: Setup proxy cache
         uses: nv-gha-runners/setup-proxy-cache@main
         continue-on-error: true
+      - name: Compute pixi environment name
+        run: |
+          set -euo pipefail
+          PIXI_ENV="cu-$(cut -d. -f1-2 <<< "${{ matrix.CUDA_VER }}" | tr . -)"
+          echo "PIXI_ENV=${PIXI_ENV}" >> "${GITHUB_ENV}"
+      - name: Setup pixi
+        uses: prefix-dev/setup-pixi@fef5c9568ca6c4ff7707bf840ab0692ba3f08293
+        with:
+          environments: ${{ env.PIXI_ENV }}
       - name: Python tests
         run: ${{ inputs.script }}
         env:

.github/workflows/simulator-test.yaml

@@ -101,6 +101,15 @@ jobs:
           echo "RAPIDS_NIGHTLY_DATE=${{ inputs.date }}" >> "${GITHUB_ENV}"
       - name: Setup proxy cache
         uses: nv-gha-runners/setup-proxy-cache@main
+      - name: Compute pixi environment name
+        run: |
+          set -euo pipefail
+          PIXI_ENV="cu-$(cut -d. -f1-2 <<< "${{ matrix.CUDA_VER }}" | tr . -)"
+          echo "PIXI_ENV=${PIXI_ENV}" >> "${GITHUB_ENV}"
+      - name: Setup pixi
+        uses: prefix-dev/setup-pixi@fef5c9568ca6c4ff7707bf840ab0692ba3f08293
+        with:
+          environments: ${{ env.PIXI_ENV }}
       - name: Python tests
         run: ${{ inputs.script }}
         env:

ci/test_conda.sh

@@ -4,16 +4,6 @@
 
 set -euo pipefail
 
-. /opt/conda/etc/profile.d/conda.sh
-
-CTK_PACKAGE_DEPENDENCIES=(
-    "cuda-cccl"
-    "cuda-nvcc-impl"
-    "cuda-nvrtc"
-    "libcurand-dev"
-    "cuda-cuobjdump"
-)
-
 DISTRO=`cat /etc/os-release | grep "^ID=" | awk 'BEGIN {FS="="} { print $2 }'`
 
 if [ "$DISTRO" = "ubuntu" ]; then
@@ -22,67 +12,30 @@ if [ "$DISTRO" = "ubuntu" ]; then
   apt remove --purge `dpkg --get-selections | grep cuda-nvrtc | awk '{print $1}'` -y
 fi
 
-rapids-logger "Install testing dependencies"
-# TODO: Replace with rapids-dependency-file-generator
-DEPENDENCIES=(
-    "c-compiler"
-    "cxx-compiler"
-    "${CTK_PACKAGE_DEPENDENCIES[@]}"
-    "cuda-python"
-    "cuda-version=${CUDA_VER%.*}"
-    "make"
-    "numba-cuda"
-    "psutil"
-    "pytest"
-    "pytest-xdist"
-    "pytest-benchmark"
-    "cffi"
-    "ml_dtypes"
-    "python=${RAPIDS_PY_VERSION}"
-)
 # Constrain oldest supported dependencies for testing
 if [ "${RAPIDS_DEPENDENCIES:-}" = "oldest" ]; then
-    DEPENDENCIES+=("numba==0.60.0")
+    # add to the default environment's dependencies
+    pixi add "numba=0.60.0"
 fi
 
-rapids-mamba-retry create \
-    -n test \
-    --strict-channel-priority \
-    --channel "`pwd`/conda-repo" \
-    --channel conda-forge \
-    "${DEPENDENCIES[@]}"
-
-# Temporarily allow unbound variables for conda activation.
-set +u
-conda activate test
-set -u
-
-pip install filecheck
-
-rapids-print-env
-
 rapids-logger "Check GPU usage"
 nvidia-smi
 
-rapids-logger "Build test binaries"
-export NUMBA_CUDA_TEST_BIN_DIR=`pwd`/testing
-pushd $NUMBA_CUDA_TEST_BIN_DIR
-make -j $(nproc)
-
-rapids-logger "Show Numba system info"
-python -m numba --sysinfo
-
 EXITCODE=0
 trap "EXITCODE=1" ERR
 set +e
 
+rapids-logger "Show Numba system info"
+pixi run -e "${PIXI_ENV}" python -m numba --sysinfo
+
 rapids-logger "Test importing numba.cuda"
-python -c "from numba import cuda"
+pixi run -e "${PIXI_ENV}" python -c "from numba import cuda"
 
 rapids-logger "Run Tests"
-pytest -v
-
-popd
+pixi run -e "${PIXI_ENV}" test -n auto \
+  --dist loadscope \
+  --loadscope-reorder \
+  -v
 
 rapids-logger "Test script exiting with value: $EXITCODE"
 exit ${EXITCODE}

ci/test_simulator.sh

@@ -4,56 +4,21 @@
 
 set -euo pipefail
 
-. /opt/conda/etc/profile.d/conda.sh
-
-rapids-logger "Install testing dependencies"
-# TODO: Replace with rapids-dependency-file-generator
-DEPENDENCIES=(
-    "psutil"
-    "pytest"
-    "pytest-xdist"
-    "pytest-benchmark"
-    "cffi"
-    "ml_dtypes"
-    "python=${RAPIDS_PY_VERSION}"
-    "numba-cuda"
-)
-rapids-mamba-retry create \
-    -n test \
-    --strict-channel-priority \
-    --channel "`pwd`/conda-repo" \
-    --channel conda-forge \
-    "${DEPENDENCIES[@]}"
-
-# Temporarily allow unbound variables for conda activation.
-set +u
-conda activate test
-set -u
-
-pip install filecheck
-
-rapids-print-env
-
-# The simulator doesn't actually use the test binaries, but we move into the
-# test binaries folder so that we're not in the root of the repo, and therefore
-# numba-cuda code from the installed package will be tested, instead of the
-# code in the source repo.
-rapids-logger "Move to test binaries folder"
-export NUMBA_CUDA_TEST_BIN_DIR=`pwd`/testing
-pushd $NUMBA_CUDA_TEST_BIN_DIR
-
-rapids-logger "Show Numba system info"
-python -m numba --sysinfo
-
 EXITCODE=0
 trap "EXITCODE=1" ERR
 set +e
 
-rapids-logger "Run Tests"
-export NUMBA_ENABLE_CUDASIM=1
-pytest -v
+rapids-logger "Show Numba system info"
+pixi run -e "${PIXI_ENV}" python -m numba --sysinfo
+
+rapids-logger "Test importing numba.cuda"
+pixi run -e "${PIXI_ENV}" python -c "from numba import cuda"
 
-popd
+rapids-logger "Run Tests"
+pixi run -e "${PIXI_ENV}" simtest -n auto \
+  --dist loadscope \
+  --loadscope-reorder \
+  -v
 
 rapids-logger "Test script exiting with value: $EXITCODE"
 exit ${EXITCODE}

numba_cuda/numba/cuda/tests/cudadrv/test_deallocations.py

@@ -17,8 +17,8 @@
 
 
 @skip_on_cudasim("not supported on CUDASIM")
-@skip_if_external_memmgr("Deallocation specific to Numba memory management")
 class TestDeallocation(CUDATestCase):
+    @skip_if_external_memmgr("Deallocation specific to Numba memory management")
     def test_max_pending_count(self):
         # get deallocation manager and flush it
         deallocs = cuda.current_context().memory_manager.deallocations
@@ -32,6 +32,7 @@ def test_max_pending_count(self):
         cuda.to_device(np.arange(1))
         self.assertEqual(len(deallocs), 0)
 
+    @skip_if_external_memmgr("Deallocation specific to Numba memory management")
     def test_max_pending_bytes(self):
         # get deallocation manager and flush it
         ctx = cuda.current_context()
@@ -73,11 +74,8 @@ def test_max_pending_bytes(self):
             # restore old ratio
             config.CUDA_DEALLOCS_RATIO = old_ratio
 
-
-@skip_on_cudasim("defer_cleanup has no effect in CUDASIM")
-@skip_if_external_memmgr("Deallocation specific to Numba memory management")
-class TestDeferCleanup(CUDATestCase):
-    def test_basic(self):
+    @skip_if_external_memmgr("Deallocation specific to Numba memory management")
+    def test_defer_cleanup(self):
         harr = np.arange(5)
         darr1 = cuda.to_device(harr)
         deallocs = cuda.current_context().memory_manager.deallocations
@@ -95,7 +93,8 @@ def test_basic(self):
         deallocs.clear()
         self.assertEqual(len(deallocs), 0)
 
-    def test_nested(self):
+    @skip_if_external_memmgr("Deallocation specific to Numba memory management")
+    def test_nested_defer_cleanup(self):
         harr = np.arange(5)
         darr1 = cuda.to_device(harr)
         deallocs = cuda.current_context().memory_manager.deallocations
@@ -116,6 +115,7 @@ def test_nested(self):
         deallocs.clear()
         self.assertEqual(len(deallocs), 0)
 
+    @skip_if_external_memmgr("Deallocation specific to Numba memory management")
     def test_exception(self):
         harr = np.arange(5)
         darr1 = cuda.to_device(harr)
@@ -141,64 +141,50 @@ class CustomError(Exception):
         deallocs.clear()
         self.assertEqual(len(deallocs), 0)
 
-
-class TestDeferCleanupAvail(CUDATestCase):
-    def test_context_manager(self):
-        # just make sure the API is available
-        with cuda.defer_cleanup():
-            pass
-
-
-@skip_on_cudasim("not supported on CUDASIM")
-class TestDel(CUDATestCase):
-    """
-    Ensure resources are deleted properly without ignored exception.
-    """
-
     @contextmanager
     def check_ignored_exception(self, ctx):
         with captured_stderr() as cap:
             yield
             ctx.deallocations.clear()
         self.assertFalse(cap.getvalue())
 
-    def test_stream(self):
+    def test_del_stream(self):
         ctx = cuda.current_context()
         stream = ctx.create_stream()
         with self.check_ignored_exception(ctx):
             del stream
 
-    def test_event(self):
+    def test_del_event(self):
         ctx = cuda.current_context()
         event = ctx.create_event()
         with self.check_ignored_exception(ctx):
             del event
 
-    def test_pinned_memory(self):
+    def test_del_pinned_memory(self):
         ctx = cuda.current_context()
         mem = ctx.memhostalloc(32)
         with self.check_ignored_exception(ctx):
             del mem
 
-    def test_mapped_memory(self):
+    def test_del_mapped_memory(self):
         ctx = cuda.current_context()
         mem = ctx.memhostalloc(32, mapped=True)
         with self.check_ignored_exception(ctx):
             del mem
 
-    def test_device_memory(self):
+    def test_del_device_memory(self):
         ctx = cuda.current_context()
         mem = ctx.memalloc(32)
         with self.check_ignored_exception(ctx):
             del mem
 
-    def test_managed_memory(self):
+    def test_del_managed_memory(self):
         ctx = cuda.current_context()
         mem = ctx.memallocmanaged(32)
         with self.check_ignored_exception(ctx):
             del mem
 
-    def test_pinned_contextmanager(self):
+    def test_del_pinned_contextmanager(self):
         # Check that temporarily pinned memory is unregistered immediately,
         # such that it can be re-pinned at any time
         class PinnedException(Exception):
@@ -227,7 +213,7 @@ class PinnedException(Exception):
                 with cuda.pinned(arr):
                     pass
 
-    def test_mapped_contextmanager(self):
+    def test_del_mapped_contextmanager(self):
         # Check that temporarily mapped memory is unregistered immediately,
         # such that it can be re-mapped at any time
         class MappedException(Exception):