diff --git a/.github/workflows/array_api.yml b/.github/workflows/array_api.yml index 3287c514ec08..9fb816000973 100644 --- a/.github/workflows/array_api.yml +++ b/.github/workflows/array_api.yml @@ -44,6 +44,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4 with: @@ -63,9 +64,11 @@ jobs: - name: Test run: | export OMP_NUM_THREADS=2 - for task in ${{ join(matrix.tasks, ' ') }}; do + for task in ${TASKS}; do pixi run --skip-deps "$task" -- --durations 3 --timeout=60 done + env: + TASKS: ${{ join(matrix.tasks, ' ') }} - name: Ccache performance shell: bash -l {0} diff --git a/.github/workflows/circle_artifacts.yml b/.github/workflows/circle_artifacts.yml index 7f9149437d9d..e4ee2a7b03e4 100644 --- a/.github/workflows/circle_artifacts.yml +++ b/.github/workflows/circle_artifacts.yml @@ -2,16 +2,22 @@ name: Redirect circleci artifacts on: [status] -permissions: - statuses: write # to report circleci status (scientific-python/circleci-artifacts-redirector-action) +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: {} jobs: circleci_artifacts_redirector_job: runs-on: ubuntu-22.04 + environment: circleci-redirector if: > github.repository == 'scipy/scipy' && github.event.context == 'ci/circleci: build_docs' name: Run CircleCI artifacts redirector + permissions: + statuses: write # needed to communicate status steps: - name: GitHub Action step uses: scientific-python/circleci-artifacts-redirector-action@5d358ff96e96429a5c64a969bb4a574555439f4f # v1.3.1 diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 31e976399068..766075c9f811 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -17,6 +17,10 @@ on: permissions: {} +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: analyze: name: Analyze (${{ matrix.language }}) @@ -27,15 +31,8 @@ jobs: # Consider using larger runners or machines with greater resources for possible analysis time improvements. runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} permissions: - # required for all workflows - security-events: write - - # required to fetch internal or private CodeQL packs - packages: read - - # only required for workflows in private repositories - actions: read - contents: read + security-events: write # required for all workflows + packages: read # required to fetch internal or private CodeQL packs strategy: fail-fast: false @@ -58,6 +55,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false # Add any setup steps before running the `github/codeql-action/init` action. # This includes steps like installing compilers or runtimes (`actions/setup-node` diff --git a/.github/workflows/commit_message.yml b/.github/workflows/commit_message.yml index 9d28ae2f9aab..f34025bacfa4 100644 --- a/.github/workflows/commit_message.yml +++ b/.github/workflows/commit_message.yml @@ -23,6 +23,7 @@ jobs: # Gets the correct commit message for pull request with: ref: ${{ github.event.pull_request.head.sha }} + persist-credentials: false - name: Check for skips id: skip_check diff --git a/.github/workflows/gpu-ci.yml b/.github/workflows/gpu-ci.yml index df9cabf1c73f..99765881c147 100644 --- a/.github/workflows/gpu-ci.yml +++ b/.github/workflows/gpu-ci.yml @@ -68,6 +68,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - name: Cache pixi uses: cirruslabs/cache@1251dca905f872d6420b06b8d7f9e7fc85589448 # v4 diff --git a/.github/workflows/issue-labeler.yml b/.github/workflows/issue-labeler.yml index 7b239b3b9dae..85136f194b0b 100644 --- a/.github/workflows/issue-labeler.yml +++ b/.github/workflows/issue-labeler.yml @@ -5,13 +5,18 @@ on: permissions: {} +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: label_issue: + name: "Label Issue" # Permissions needed for labelling issues automatically # https://docs.github.com/en/actions/security-guides/automatic-token-authentication permissions: contents: read - issues: write + issues: write # needed to apply label runs-on: ubuntu-latest steps: # label based on issue title diff --git a/.github/workflows/label-pr-on-close.yml b/.github/workflows/label-pr-on-close.yml index 0edde13ce0be..7c6bd1b77be3 100644 --- a/.github/workflows/label-pr-on-close.yml +++ b/.github/workflows/label-pr-on-close.yml @@ -5,6 +5,10 @@ on: permissions: {} +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: label_pull_request: name: "Label PR on Close" @@ -12,7 +16,7 @@ jobs: # https://docs.github.com/en/actions/security-guides/automatic-token-authentication permissions: contents: read - pull-requests: write + pull-requests: write # needed to apply label runs-on: ubuntu-latest steps: - uses: github/issue-labeler@c1b0f9f52a63158c4adc09425e858e87b32e9685 # v3.4 diff --git a/.github/workflows/label-pr-on-open.yml b/.github/workflows/label-pr-on-open.yml index 98455199f2e8..7d5f2daeec08 100644 --- a/.github/workflows/label-pr-on-open.yml +++ b/.github/workflows/label-pr-on-open.yml @@ -5,13 +5,18 @@ on: permissions: {} +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: label_pull_request: + name: Label PR # Permissions needed for labelling Pull Requests automatically # https://docs.github.com/en/actions/security-guides/automatic-token-authentication permissions: contents: read - pull-requests: write + pull-requests: write # needed to apply label runs-on: ubuntu-latest steps: # label based on changed files diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 608f50cc79dc..cf0c93012550 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -27,6 +27,7 @@ jobs: with: fetch-depth: 0 # previous commits used in tools/lint.py submodules: recursive + persist-credentials: false - uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4 with: @@ -44,9 +45,10 @@ jobs: run: pixi run check-python-h - name: Check module interdependencies - working-directory: .github/workflows run: pixi run tach-check - name: Check external dependencies - working-directory: .github/workflows run: pixi run tach-check-external + + - name: GitHub Actions static analysis + run: pixi run zizmor diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 1b823442b3fe..0757e0ea477b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -48,6 +48,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -141,6 +142,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - name: Install Ubuntu dependencies run: | @@ -213,6 +215,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - name: Configuring Test Environment run: | @@ -248,6 +251,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -318,6 +322,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -378,6 +383,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - name: build + test in i686 container run: | @@ -420,6 +426,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - name: Setup system dependencies run: | @@ -481,6 +488,7 @@ jobs: with: submodules: recursive fetch-tags: true + persist-credentials: false - name: Install system dependencies run: | @@ -536,6 +544,7 @@ jobs: ################################################################################# free-threaded: + name: free-threaded needs: get_commit_message strategy: matrix: @@ -550,6 +559,7 @@ jobs: with: submodules: recursive fetch-tags: true + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -639,6 +649,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -674,6 +685,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -716,6 +728,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4 with: diff --git a/.github/workflows/linux_blas.yml b/.github/workflows/linux_blas.yml index baf1e3c099de..573205b61142 100644 --- a/.github/workflows/linux_blas.yml +++ b/.github/workflows/linux_blas.yml @@ -47,6 +47,7 @@ jobs: with: submodules: recursive fetch-depth: 0 + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' @@ -81,6 +82,7 @@ jobs: with: submodules: recursive fetch-depth: 0 + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' @@ -102,7 +104,6 @@ jobs: - name: Test run: spin test -j4 - mkl-cyilp64: runs-on: ubuntu-latest name: "MKL ILP64 (cython-blas ILP64)" @@ -115,6 +116,7 @@ jobs: with: submodules: recursive fetch-depth: 0 + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.12' @@ -122,14 +124,30 @@ jobs: - name: Install dependencies run: | pip install -r requirements/build.txt - pip install mkl mkl-devel spin "click<8.3.0" pytest hypothesis pytest-xdist pooch + pip install mkl mkl-devel build pytest hypothesis pytest-xdist pooch - - name: Build with ILP64 + - name: Build SciPy wheel and install run: | - spin build -S-Dblas=mkl-dynamic-ilp64-seq -S-Duse-ilp64=true -S-D_without-fortran=true + python -m build -wnx \ + -Csetup-args=-Dblas=mkl-dynamic-ilp64-seq \ + -Csetup-args=-Duse-ilp64=true \ + -Csetup-args=-D_without-fortran=true + pip install dist/scipy*.whl - - name: Test - run: spin test -j4 + - name: Test SciPy + run: | + cd /tmp + python -m pytest --pyargs scipy -n2 + + - name: Install linalg ILP64 downstream test package + run: | + pip install --no-build-isolation -v scipy/linalg/tests/_cython_examples/ilp64_test_package + + - name: Test linalg ILP64 downstream test package + run: | + cp -r scipy/linalg/tests/_cython_examples/ilp64_test_package/tests /tmp/ilp64_tests + cd /tmp + python -m pytest ilp64_tests scipy-openblas-ilp64: @@ -144,6 +162,7 @@ jobs: with: submodules: recursive fetch-depth: 0 + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.14' @@ -151,15 +170,37 @@ jobs: - name: Install dependencies run: | pip install -r requirements/build.txt - pip install mkl mkl-devel spin "click<8.3.0" pytest hypothesis pytest-xdist pooch + pip install build pytest hypothesis pytest-xdist pooch pip install -r requirements/openblas64.txt - - name: Build with ILP64 + - name: Write out scipy-openblas64.pc + run: | + python -c'import scipy_openblas64 as so64; print(so64.get_pkg_config())' > scipy-openblas.pc + + - name: Build SciPy wheel and install run: | # scipy-openblas64 doesn't have LP64 symbols, so we can only use it # with `_without-fortran=true`. Drop this flag when `scipy.odr` gets # removed, keeping the rest of this job unchanged. - spin build --with-scipy-openblas=64 -S-Duse-ilp64=true -S-D_without-fortran=true -S-Dblas-symbol-suffix=64_ + export PKG_CONFIG_PATH=$PWD + python -m build -wnx \ + -Csetup-args=-Dblas=scipy-openblas \ + -Csetup-args=-Duse-ilp64=true \ + -Csetup-args=-Dblas-symbol-suffix=64_ \ + -Csetup-args=-D_without-fortran=true + pip install dist/scipy*.whl + + - name: Test SciPy + run: | + cd /tmp + python -m pytest --pyargs scipy -n2 - - name: Test - run: spin test -j4 + - name: Install linalg ILP64 downstream test package + run: | + pip install --no-build-isolation -v -Csetup-args=--werror scipy/linalg/tests/_cython_examples/ilp64_test_package + + - name: Test ILP64 downstream test package + run: | + cp -r scipy/linalg/tests/_cython_examples/ilp64_test_package/tests /tmp/ilp64_tests + cd /tmp + python -m pytest ilp64_tests diff --git a/.github/workflows/linux_intel_oneAPI.yml b/.github/workflows/linux_intel_oneAPI.yml index d5023bb20102..75aa428fc837 100644 --- a/.github/workflows/linux_intel_oneAPI.yml +++ b/.github/workflows/linux_intel_oneAPI.yml @@ -55,6 +55,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false # Reference https://github.com/easimon/maximize-build-space/blob/master/action.yml # see https://github.com/scipy/scipy/issues/23813 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index bbc32d8f3c7f..72abcb8409c0 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -37,6 +37,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4 with: @@ -76,6 +77,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4 with: @@ -85,7 +87,9 @@ jobs: - name: Test SciPy # mpmath tests are too slow for CI - run: pixi run test-${{ matrix.environment }} -m full -- -k "not mpmath" + run: pixi run test-${ENVIRONMENT} -m full -- -k "not mpmath" + env: + ENVIRONMENT: ${{ matrix.environment }} clang_ASAN: name: Test under AddressSanitizer diff --git a/.github/workflows/pixi-packages.yml b/.github/workflows/pixi-packages.yml index cef9d14cd84e..88ee267f0454 100644 --- a/.github/workflows/pixi-packages.yml +++ b/.github/workflows/pixi-packages.yml @@ -13,6 +13,10 @@ on: permissions: contents: read # to fetch code (actions/checkout) +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: get_commit_message: name: Get commit message diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index f4e18786efa1..679e89cbedc7 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -68,6 +68,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true + persist-credentials: false - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: @@ -81,17 +82,21 @@ jobs: - name: win_arm64 - set environment variables if: matrix.buildplat[1] == 'win' && matrix.buildplat[2] == 'ARM64' + env: + BUILDPLAT: ${{ matrix.buildplat[2] }} run: | echo "C:\Program Files\LLVM\bin" >> $env:GIHUB_PATH echo "CC=clang-cl" >> $env:GITHUB_ENV echo "CXX=clang-cl" >> $env:GITHUB_ENV echo "FC=flang" >> $env:GITHUB_ENV - echo "TARGET_ARCH=${{ matrix.buildplat[2] }}" >> $env:GITHUB_ENV + echo "TARGET_ARCH=ARM64" >> $env:GITHUB_ENV - name: windows - set PKG_CONFIG_PATH if: ${{ matrix.buildplat[1] == 'win' }} + env: + WORKSPACE_DIR: ${{ github.workspace }} run: | - $CIBW = "${{ github.workspace }}/.openblas" + $CIBW = "$env:WORKSPACE_DIR/.openblas" # pkgconfig needs a complete path, and not just "./openblas since the # build is run in a tmp dir (?) # It seems somewhere in the env passing, `\` is not @@ -127,14 +132,18 @@ jobs: # necessitates using install_name_tool on the scipy_openblas32 dylibs. - name: Setup macOS if: startsWith( matrix.buildplat[0], 'macos-' ) + env: + TARGET_ARCH: ${{ matrix.buildplat[2] }} + BLAS_VARIANT: ${{ matrix.buildplat[3] }} + MACOSX_DEPLOYMENT_TARGET: ${{ matrix.buildplat[4] }} run: | - if [[ ${{ matrix.buildplat[3] }} == "accelerate" ]]; then + if [[ "${BLAS_VARIANT}" == "accelerate" ]]; then echo CIBW_CONFIG_SETTINGS=\"setup-args=-Dblas=accelerate\" >> "$GITHUB_ENV" # Builds with Accelerate only target macOS>=14.0 CIBW_ENV="CIBW_ENVIRONMENT_MACOS=MACOSX_DEPLOYMENT_TARGET=14.0 INSTALL_OPENBLAS=false" - if [[ ${{ matrix.buildplat[2] }} == "arm64" ]]; then + if [[ "${TARGET_ARCH}" == "arm64" ]]; then # use preinstalled gfortran for Accelerate builds ln -s $(which gfortran-13) gfortran export PATH=$PWD:$PATH @@ -148,7 +157,7 @@ jobs: fi else CIBW_ENV="CIBW_ENVIRONMENT_MACOS=PKG_CONFIG_PATH=$PWD/.openblas \ - MACOSX_DEPLOYMENT_TARGET=${{ matrix.buildplat[4] }} \ + MACOSX_DEPLOYMENT_TARGET="${MACOSX_DEPLOYMENT_TARGET}" \ SDKROOT=$(xcrun --sdk macosx --show-sdk-path) \ INSTALL_GFORTRAN=true" fi diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 06d7cae9c0eb..e26ee5904b57 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -35,6 +35,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - uses: prefix-dev/setup-pixi@a0af7a228712d6121d37aba47adf55c1332c9c2e # v0.9.4 with: @@ -66,6 +67,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 @@ -108,6 +110,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/.github/workflows/windows_intel_oneAPI.yml b/.github/workflows/windows_intel_oneAPI.yml index 92a23926fc34..dfd4664bb22f 100644 --- a/.github/workflows/windows_intel_oneAPI.yml +++ b/.github/workflows/windows_intel_oneAPI.yml @@ -54,6 +54,7 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: recursive + persist-credentials: false - name: cache install id: cache-install @@ -66,13 +67,13 @@ jobs: key: install-${{ env.WINDOWS_HPC_URL }}-${{ env.WINDOWS_BASEKIT_URL }}-compiler - name: Install oneAPI Base kit if: steps.cache-install.outputs.cache-hit != 'true' - shell: cmd /C call {0} + shell: cmd /C call {0} # need to use `cmd` for batch scripts run: | echo %WINDOWS_BASEKIT_URL% tools\install_intel_oneAPI_windows.bat "%WINDOWS_BASEKIT_URL%" "" "491dbc593cbcf02570b2ccc1f31656e2aa6fd195bd3191e5d111b1dafb35a45c" &REM 2025.0.1.28 - name: Install oneAPI HPC kit if: steps.cache-install.outputs.cache-hit != 'true' - shell: cmd /C call {0} + shell: cmd /C call {0} # need to use `cmd` for batch scripts run: | echo %WINDOWS_HPC_URL% tools\install_intel_oneAPI_windows.bat "%WINDOWS_HPC_URL%" "" "dfbf8487f6ee5b9c1bb2cad19f2ef8dd842529a533670ce21582d39c303bf979" &REM 2025.0.1.48 @@ -93,25 +94,20 @@ jobs: - run: which python - name: Install packages from conda - shell: cmd /C call {0} - run: | - conda install -c conda-forge pkg-config meson meson-python ninja openblas libblas=*=*openblas numpy==2.0 cython pybind11 pytest pytest-xdist pytest-timeout pooch spin hypothesis "click<8.3.0" + shell: pwsh + run: conda install -c conda-forge pkg-config meson meson-python ninja openblas libblas=*=*openblas numpy==2.0 cython pybind11 pytest pytest-xdist pytest-timeout pooch spin hypothesis "click<8.3.0" # MSVC is unable to compile Pythran code, therefore we need to use # -C-Duse-pythran=false while building SciPy. # Reference - https://github.com/serge-sans-paille/pythran/issues/2215 - name: Initialise Intel oneAPI, MSVC and Build SciPy - shell: cmd /C call {0} + shell: cmd /C call {0} # need to use `cmd` for batch scripts run: | call "C:\Program Files (x86)\Intel\oneAPI\setvars.bat" set FC=ifx call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars64.bat" spin build -S-Duse-pythran=false -S--vsenv - # "import scipy; scipy.test();" fails because - # scipy/sparse/linalg/_eigen/arpack crashes. - # Reference - https://github.com/scipy/scipy/issues/20728 - name: Test scipy.datasets - shell: cmd /C call {0} - run: | - spin test -s datasets + shell: pwsh + run: spin test -s datasets diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 000000000000..b6a1cbb7aaba --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,14 @@ +rules: + dangerous-triggers: + ignore: + # Ideally, we would use a GitHub app for these, + # instead of a `pull_request_target` workflow, + # as that is deemed more secure: https://astral.sh/blog/open-source-security-at-astral#cicd-security. + # However, there is not currently a suitable app maintained by a trusted organisation, + # and self-hosting would be burdensome: https://astral.sh/blog/open-source-security-at-astral#automations. + - label-pr-on-close.yml:2 + - label-pr-on-open.yml:2 + misfeature: + ignore: + # We need to use CMD for intel oneAPI batch scripts + - windows_intel_oneAPI.yml diff --git a/doc/source/building/blas_lapack.rst b/doc/source/building/blas_lapack.rst index ff34fbefa299..ea6889074223 100644 --- a/doc/source/building/blas_lapack.rst +++ b/doc/source/building/blas_lapack.rst @@ -187,6 +187,24 @@ The build configuration can be checked at runtime via ``scipy.show_config()`` — look for the ``'blas cython ilp64'`` entry. +Using Cython BLAS/LAPACK ABI in downstream packages +""""""""""""""""""""""""""""""""""""""""""""""""""" + +Downstream packages which consume ``cython_blas`` or ``cython_lapack`` interfaces +should ideally directly use the ``blas_int`` integer type at all call sites. + +Some packages, however, might prefer to continue using ``int`` types, and manually +map between ``int`` and ``blas_int`` types (It is convenient to localize this mapping +in a single internal wrapper which converts ``int`` inputs to ``blas_int`` before +calling a LAPACK function, and then converts its ``blas_int`` outputs back to ``int``). +We stress that doing this limits the array sizes to ``< INT_MAX`` even if the +LAPACK itself is ILP64 enabled. + +Consult `a worked example`_ which illustrates both approaches. + +.. _a worked example: https://github.com/scipy/scipy/tree/main/scipy/linalg/tests/_cython_examples/ilp64_test_package + + Work-in-progress ---------------- diff --git a/pixi.lock b/pixi.lock index 051272b453a6..99de9bf44979 100644 --- a/pixi.lock +++ b/pixi.lock @@ -4556,6 +4556,7 @@ environments: - conda: https://prefix.dev/conda-forge/linux-64/xorg-xorgproto-2024.1-hb9d3cd8_1.conda - conda: https://prefix.dev/conda-forge/linux-64/yaml-0.2.5-h280c20c_3.conda - conda: https://prefix.dev/conda-forge/noarch/zipp-3.23.0-pyhcf101f3_1.conda + - conda: https://prefix.dev/conda-forge/linux-64/zizmor-1.23.1-hb17b654_0.conda - conda: https://prefix.dev/conda-forge/linux-64/zstd-1.5.7-hb78ec9c_6.conda osx-arm64: - conda: https://prefix.dev/conda-forge/noarch/_python_abi3_support-1.0-hd8ed1ab_2.conda @@ -4644,6 +4645,7 @@ environments: - conda: https://prefix.dev/conda-forge/noarch/wcwidth-0.2.14-pyhd8ed1ab_0.conda - conda: https://prefix.dev/conda-forge/osx-arm64/yaml-0.2.5-h925e9cb_3.conda - conda: https://prefix.dev/conda-forge/noarch/zipp-3.23.0-pyhcf101f3_1.conda + - conda: https://prefix.dev/conda-forge/osx-arm64/zizmor-1.23.1-h6fdd925_0.conda - conda: https://prefix.dev/conda-forge/osx-arm64/zstd-1.5.7-hbf9d68e_6.conda win-64: - conda: https://prefix.dev/conda-forge/win-64/_openmp_mutex-4.5-2_gnu.conda @@ -4740,6 +4742,7 @@ environments: - conda: https://prefix.dev/conda-forge/win-64/xorg-libxt-1.3.1-h0e40799_0.conda - conda: https://prefix.dev/conda-forge/win-64/yaml-0.2.5-h6a83c73_3.conda - conda: https://prefix.dev/conda-forge/noarch/zipp-3.23.0-pyhcf101f3_1.conda + - conda: https://prefix.dev/conda-forge/win-64/zizmor-1.23.1-h18a1a76_0.conda - conda: https://prefix.dev/conda-forge/win-64/zstd-1.5.7-h534d264_6.conda lldb: channels: @@ -24742,6 +24745,40 @@ packages: - pkg:pypi/zipp?source=compressed-mapping size: 24194 timestamp: 1764460141901 +- conda: https://prefix.dev/conda-forge/linux-64/zizmor-1.23.1-hb17b654_0.conda + sha256: 80188255dbd17fba8b2f2d84dd0f5fc67c3e4c62ea40618fb561f409488b8355 + md5: 07a8efd9657796663bb4eb4ac821c012 + depends: + - __glibc >=2.17,<3.0.a0 + - libgcc >=14 + constrains: + - __glibc >=2.17 + license: MIT + license_family: MIT + size: 6958373 + timestamp: 1773569800269 +- conda: https://prefix.dev/conda-forge/osx-arm64/zizmor-1.23.1-h6fdd925_0.conda + sha256: 244ef691a4c3d7fca126f330055591c99ecfbe1972f0d50609ec5366529a9831 + md5: acc150c81d4a508198e921ab3789835b + depends: + - __osx >=11.0 + constrains: + - __osx >=11.0 + license: MIT + license_family: MIT + size: 6443999 + timestamp: 1773569857372 +- conda: https://prefix.dev/conda-forge/win-64/zizmor-1.23.1-h18a1a76_0.conda + sha256: e5e14dc15203bb4f691ee148cf9f422057b3082a7f2c446054f1a14ae0be5715 + md5: b338f9d73052ff4c329b98d7acbea17d + depends: + - vc >=14.3,<15 + - vc14_runtime >=14.44.35208 + - ucrt >=10.0.20348.0 + license: MIT + license_family: MIT + size: 7183577 + timestamp: 1773569843787 - conda: https://prefix.dev/conda-forge/linux-64/zlib-1.3.1-hb9d3cd8_2.conda sha256: 5d7c0e5f0005f74112a34a7425179f4eb6e73c92f5d109e6af4ddeca407c92ab md5: c9f075ab2f33b3bbee9e62d4ad0a6cd8 diff --git a/pixi.toml b/pixi.toml index b75d1d5e00e1..82656b7ca618 100644 --- a/pixi.toml +++ b/pixi.toml @@ -409,6 +409,7 @@ ruff = "*" cython-lint = "*" packaging = "*" tach = "*" +zizmor = "*" [feature.lint.tasks.lint] cmd = "spin lint" @@ -427,6 +428,10 @@ description = "Check inter-module dependencies" cmd = "tach check-external" description = "Check external dependencies" +[feature.lint.tasks.zizmor] +cmd = "zizmor .github --config=.github/zizmor.yml" +description = "Run zizmor (GitHub actions static analysis)" + ### BLAS/LAPACK features ### diff --git a/scipy/integrate/_ivp/rk.py b/scipy/integrate/_ivp/rk.py index 252ba4c92f89..df71a7e63179 100644 --- a/scipy/integrate/_ivp/rk.py +++ b/scipy/integrate/_ivp/rk.py @@ -274,6 +274,85 @@ class RK23(RungeKutta): ---------- .. [1] P. Bogacki, L.F. Shampine, "A 3(2) Pair of Runge-Kutta Formulas", Appl. Math. Lett. Vol. 2, No. 4. pp. 321-325, 1989. + + Examples + -------- + This example demonstrates how to compute one period of a periodic + orbit in a two-dimensional dynamical system. + + >>> import numpy as np + >>> import scipy.integrate as itg + >>> import matplotlib.pylab as plt + + Define a function that returns the right-hand side of a dynamical + system with a stable periodic orbit. + + >>> def dynam_sys(t, x): + ... dx1_dt = 5*(1-x[1]**2)*x[0] - x[1] + ... dx2_dt = x[0] + ... return [dx1_dt, dx2_dt] + + The system has an equilibrium at the origin. Create a solver + object with an initial state slightly away from the + equilibrium. + + >>> solver = itg.RK45(dynam_sys, 0, [0.25,0.25], 1E200, max_step=0.05) + + Create an array in which to store the estimated solution. + To allow for concatenation of future states, initialize the array + by reshaping the initial state. + + >>> soly = solver.y.reshape(1, 2) + + To find a point on the periodic orbit, run the solver + for ``2000`` integration steps. Store the estimated states in + ``soly``. + + >>> for i in range(2000): + ... solver.step() + ... solyn = solver.y.reshape(1, 2) + ... soly = np.concatenate((soly, solyn)) + + Plot ``soly`` with its first element in green and its final + element in magenta. + + >>> plt.plot(soly[:,0], soly[:,1]) + >>> plt.plot(soly[0,0], soly[0,1], "go") + >>> plt.plot(soly[-1,0], soly[-1,1], "mo") + >>> plt.show() + + The plot shows that the last element in ``soly`` is located on a + segment of the periodic orbit where there is relatively little + variation in the estimated states. Create another solver and + initialize it with the last element in ``soly``. + + >>> init = np.array([soly[-1,0], soly[-1,1]]) + >>> POsolver = itg.RK23(dynam_sys, 0, init, 5000, max_step=0.05) + + Create a variable that tracks the distance of the current + estimated state from the initial state. Run ``POsolver`` until + the current estimated state is within ``0.005`` units of the last + estimated state. + + >>> dist_init = 1 + >>> POsoly = POsolver.y.reshape(1, 2) + >>> while dist_init > 0.005: + ... POsolver.step() + ... POsolyn = POsolver.y.reshape(1, 2) + ... POsoly = np.concatenate((POsoly, POsolyn)) + ... dist_init = np.linalg.norm(init-POsolyn) + + Plot ``POsoly`` with its initial element in green and its final + element in magenta. + + >>> plt.plot(POsoly[:,0], POsoly[:,1]) + >>> plt.plot(POsoly[0,0], POsoly[0,1], "go") + >>> plt.plot(POsoly[-1,0], POsoly[-1,1], "mo") + >>> plt.show() + + The plot shows approximately one period of the orbit. + The final and initial states are so close that the final + state marker obscures the initial state marker. """ order = 3 error_estimator_order = 2 @@ -377,6 +456,84 @@ class RK45(RungeKutta): No. 1, pp. 19-26, 1980. .. [2] L. W. Shampine, "Some Practical Runge-Kutta Formulas", Mathematics of Computation,, Vol. 46, No. 173, pp. 135-150, 1986. + + Examples + -------- + This example demonstrates how to compute one period of a periodic + orbit in a two-dimensional dynamical system. + + >>> import numpy as np + >>> import scipy.integrate as itg + >>> import matplotlib.pylab as plt + + Define a function that returns the right-hand side of a dynamical + system with a stable periodic orbit. + + >>> def dynam_sys(t, x): + ... dx1_dt = 5*(1-x[1]**2)*x[0] - x[1] + ... dx2_dt = x[0] + ... return [dx1_dt, dx2_dt] + + The system has an equilibrium at the origin. Create a solver + object with an initial state slightly away from the + equilibrium. + + >>> solver = itg.RK45(dynam_sys, 0, [0.25,0.25], 5000, max_step=0.05) + + Create an array in which to store the estimated solution. + To allow for concatenation of future states, initialize the array + by reshaping the initial state. + + >>> soly = solver.y.reshape(1, 2) + + To find a point on the periodic orbit, run the solver for + ``2000`` integration steps. Store the estimated states in ``soly``. + + >>> for i in range(2000): + ... solver.step() + ... solyn = solver.y.reshape(1, 2) + ... soly = np.concatenate((soly, solyn)) + + Plot ``soly`` with its first element in green and its final + element in magenta. + + >>> plt.plot(soly[:,0], soly[:,1]) + >>> plt.plot(soly[0,0], soly[0,1], "go") + >>> plt.plot(soly[-1,0], soly[-1,1], "mo") + >>> plt.show() + + The plot shows that the last element in ``soly`` is located on a + segment of the periodic orbit where there is relatively little + variation in the estimated states. Create another solver and + initialize it with the last element in ``soly``. + + >>> init = np.array([soly[-1,0], soly[-1,1]]) + >>> POsolver = itg.RK45(dynam_sys, 0, init, 5000, max_step=0.05) + + Create a variable that tracks the distance of the current + estimated state from the initial state. Run ``POsolver`` until + the current estimated state is within ``0.01`` units of the last + estimated state. + + >>> dist_init = 1 + >>> POsoly = POsolver.y.reshape(1, 2) + >>> while dist_init > 0.01: + ... POsolver.step() + ... POsolyn = POsolver.y.reshape(1, 2) + ... POsoly = np.concatenate((POsoly, POsolyn)) + ... dist_init = np.linalg.norm(init-POsolyn) + + Plot ``POsoly`` with its initial element in green and its final + element in magenta. + + >>> plt.plot(POsoly[:,0], POsoly[:,1]) + >>> plt.plot(POsoly[0,0], POsoly[0,1], "go") + >>> plt.plot(POsoly[-1,0], POsoly[-1,1], "mo") + >>> plt.show() + + The plot shows approximately one period of the orbit. + The final and initial states are so close that the final + state marker obscures the initial state marker. """ order = 5 error_estimator_order = 4 diff --git a/scipy/linalg/tests/_cython_examples/ilp64_test_package/README b/scipy/linalg/tests/_cython_examples/ilp64_test_package/README new file mode 100644 index 000000000000..8cbeb0601726 --- /dev/null +++ b/scipy/linalg/tests/_cython_examples/ilp64_test_package/README @@ -0,0 +1,30 @@ +A small test package to demonstrate ILP64 support in downstream Cython packages. + +Currently, SciPy can be built with or without ILP64 support; in the former case +`cython_lapack` ABI can be built to be either LP64 or ILP64, see [1] for a discussion. + +Here we demonstrate two ways of adapting downstream for a Cython-using package, +which relies on `cython_lapack`. + +A difficulty is that `cython_lapack` is using `blas_int` type for integer variables +and arrays, which resolves to either a C `int` or `int64_t`. +A downstream user package has two options: + +1. Use a cython wrapper to cast from `int` to `blas_int` type, so that other modules + can depend on this wrapper and hence don't need changing. + This way, the linear algebra is exposed as LP64 regardless of whether SciPy + is built to internally use LP64 or ILP64 variants. + See `tests/test_wrapper.py` for details. + +2. Directly use SciPy's `blas_int` type without an intermediate wrapper layer. + This way, the downstream functionality is not limited by `INT_MAX` for array sizes. + (see `tests/test_direct.py` for details). + +See the relevant SciPy CI workflow [2] for an example of building and testing +this package. + +------------ + +[1] https://scipy.github.io/devdocs/building/blas_lapack.html#bit-integer-ilp64-blas-lapack +[2] https://github.com/scipy/scipy/blob/main/.github/workflows/linux_blas.yml + diff --git a/scipy/linalg/tests/_cython_examples/ilp64_test_package/meson.build b/scipy/linalg/tests/_cython_examples/ilp64_test_package/meson.build new file mode 100644 index 000000000000..7dab4b7badc9 --- /dev/null +++ b/scipy/linalg/tests/_cython_examples/ilp64_test_package/meson.build @@ -0,0 +1,16 @@ +project('ilp64_test_package', 'c', 'cython', meson_version: '>=1.9.0') + +fs = import('fs') +py = import('python').find_installation(pure: false) +cy = meson.get_compiler('cython') + +if not cy.version().version_compare('>=3.2.0') + error('Requires Cython >= 3.2.0') +endif + +cython_args = [ + '-Xfreethreading_compatible=True', + '-I' + meson.project_build_root() / 'src' / 'ilp64_test_package', +] + +subdir('src/ilp64_test_package') diff --git a/scipy/linalg/tests/_cython_examples/ilp64_test_package/pyproject.toml b/scipy/linalg/tests/_cython_examples/ilp64_test_package/pyproject.toml new file mode 100644 index 000000000000..0fcac85b9f49 --- /dev/null +++ b/scipy/linalg/tests/_cython_examples/ilp64_test_package/pyproject.toml @@ -0,0 +1,10 @@ +[build-system] +requires = ["meson-python>=0.16.0", "cython>=3.2.0", "numpy>=2.0.0", "scipy"] +build-backend = "mesonpy" + +[project] +name = "ilp64_test_package" +version = "0.1.0" +license = "BSD-3-Clause" +requires-python = ">=3.12" +dependencies = ["scipy", "numpy"] diff --git a/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/__init__.py b/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_blas_int.pxi.in b/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_blas_int.pxi.in new file mode 100644 index 000000000000..2a3c550f2d0a --- /dev/null +++ b/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_blas_int.pxi.in @@ -0,0 +1,4 @@ +# This will be a plain `int` if scipy.linalg.cython_blas doesn't have +# `blas_int`, and `blas_int` (cimported from scipy) if it does. +# See the top-level `meson.build` for the compile-time check. +@BLAS_INT_DEF@ diff --git a/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_blas_lapack_wrappers.pxd b/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_blas_lapack_wrappers.pxd new file mode 100644 index 000000000000..d655d9151f62 --- /dev/null +++ b/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_blas_lapack_wrappers.pxd @@ -0,0 +1,22 @@ +# Internal .pxd declaring wrapper functions with `int` parameters. +# This simulates the pattern used by scikit-learn and statsmodels, +# where the existing internal API uses `int` and must be adapted +# to work with ILP64 builds where BLAS/LAPACK expect 64-bit integers. + +cdef double _ddot(int n, const double *x, int incx, + const double *y, int incy) noexcept nogil + +cdef void _daxpy(int n, double alpha, const double *x, int incx, + double *y, int incy) noexcept nogil + +cdef double _dnrm2(int n, const double *x, int incx) noexcept nogil + +cdef void _dgemm(char *transa, char *transb, int m, int n, int k, + double alpha, const double *a, int lda, + const double *b, int ldb, + double beta, double *c, int ldc) noexcept nogil + +cdef int _dgetrf(int m, int n, double *a, int lda, + int *ipiv) noexcept nogil + +cpdef int get_blas_int_size() diff --git a/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_blas_lapack_wrappers.pyx b/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_blas_lapack_wrappers.pyx new file mode 100644 index 000000000000..08d0b4426a49 --- /dev/null +++ b/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_blas_lapack_wrappers.pyx @@ -0,0 +1,82 @@ +#cython: language_level=3 +#cython: boundscheck=False +#cython: wraparound=False +""" +Part 1: sklearn-like BLAS/LAPACK wrappers with int->blas_int conversion. + +This module includes _blas_int.pxi (generated from the .pxi.in template) +which defines blas_int as either `int` (LP64) or `int64_t` (ILP64). +All wrapper functions accept `int` parameters and convert to `blas_int` +before calling the actual BLAS/LAPACK routines. +""" + +include "_blas_int.pxi" + +from scipy.linalg.cython_blas cimport ddot, daxpy, dgemm, dnrm2 +from scipy.linalg.cython_lapack cimport dgetrf + +from libc.stdlib cimport malloc, free + +# Pointer casts from `const double *` to `double *` below are needed because +# SciPy's cython_blas.pxd lacks const qualifiers. This pattern is copied from +# scikit-learn's sklearn/utils/_cython_blas.pyx. +# See: https://github.com/scipy/scipy/issues/14262 + + +cdef double _ddot(int n, const double *x, int incx, + const double *y, int incy) noexcept nogil: + cdef blas_int bn = n, bincx = incx, bincy = incy + return ddot(&bn, x, &bincx, y, &bincy) + + +cdef void _daxpy(int n, double alpha, const double *x, int incx, + double *y, int incy) noexcept nogil: + cdef blas_int bn = n, bincx = incx, bincy = incy + daxpy(&bn, &alpha, x, &bincx, y, &bincy) + + +cdef double _dnrm2(int n, const double *x, int incx) noexcept nogil: + cdef: + blas_int bn = n + blas_int bincx = incx + double nrm2 = dnrm2(&bn, x, &bincx) + return nrm2 + + +cdef void _dgemm(char *transa, char *transb, int m, int n, int k, + double alpha, const double *a, int lda, + const double *b, int ldb, + double beta, double *c, int ldc) noexcept nogil: + cdef blas_int bm = m, bn = n, bk = k + cdef blas_int blda = lda, bldb = ldb, bldc = ldc + dgemm(transa, transb, &bm, &bn, &bk, + &alpha, a, &blda, b, &bldb, &beta, c, &bldc) + + +cdef int _dgetrf(int m, int n, double *a, int lda, + int *ipiv) noexcept nogil: + """LU factorization. The tricky part: ipiv is an output array of + blas_int, but the downstream API uses int. We must allocate a + temporary blas_int array, call LAPACK, then copy back.""" + cdef: + blas_int bm = m, bn = n, blda = lda, info + blas_int *bipiv + int min_mn = min(m, n) + int i + + bipiv = malloc(min_mn * sizeof(blas_int)) + if bipiv == NULL: + return -1000 # allocation failure + + dgetrf(&bm, &bn, a, &blda, bipiv, &info) + + for i in range(min_mn): + ipiv[i] = bipiv[i] + + free(bipiv) + return info + + +cpdef int get_blas_int_size(): + """Return sizeof(blas_int) to verify correct type at compile+runtime.""" + return sizeof(blas_int) diff --git a/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_consumer.pyx b/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_consumer.pyx new file mode 100644 index 000000000000..969b0ae04ecc --- /dev/null +++ b/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_consumer.pyx @@ -0,0 +1,63 @@ +#cython: language_level=3 +#cython: boundscheck=False +#cython: wraparound=False +""" +Part 1 consumer: cimports from the internal _blas_lapack_wrappers module. + +This validates that the internal .pxd works correctly for cross-module +cimport, which is the pattern used by scikit-learn (e.g., other Cython +modules cimporting from sklearn.utils._cython_blas). +""" + +from libc.limits cimport INT_MAX + +from ilp64_test_package._blas_lapack_wrappers cimport ( + _ddot, _daxpy, _dnrm2, _dgemm, _dgetrf, get_blas_int_size +) + + +cpdef double consumer_ddot(double[:] x, double[:] y): + if x.shape[0] > INT_MAX: + raise ValueError("Integer overflow in consumer_ddot.") + cdef int n = x.shape[0] + return _ddot(n, &x[0], 1, &y[0], 1) + + +# Strictly speaking, all functions should check for integer overflow of the input array +# size. We are omitting these checks below, for brevity. + + +cpdef consumer_daxpy(double alpha, double[:] x, double[:] y): + cdef int n = x.shape[0] + _daxpy(n, alpha, &x[0], 1, &y[0], 1) + + +cpdef consumer_dnrm2(double[:] x): + cdef: + int n = x.shape[0] + int incx = x.strides[0] // sizeof(double) + double nrm2 = _dnrm2(n, &x[0], incx) + return nrm2 + + +cpdef consumer_dgemm(double alpha, double[::1,:] a, double[::1,:] b, + double beta, double[::1,:] c): + """Matrix multiply assuming Fortran-contiguous arrays.""" + cdef: + char transa = b'N' + char transb = b'N' + int m = a.shape[0] + int k = a.shape[1] + int n = b.shape[1] + _dgemm(&transa, &transb, m, n, k, + alpha, &a[0, 0], m, &b[0, 0], k, beta, &c[0, 0], m) + + +cpdef int consumer_dgetrf(double[::1,:] a, int[:] ipiv): + """LU factorization via the internal wrappers.""" + cdef int m = a.shape[0], n = a.shape[1] + return _dgetrf(m, n, &a[0, 0], m, &ipiv[0]) + + +cpdef int consumer_blas_int_size(): + return get_blas_int_size() diff --git a/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_direct_blas.pyx b/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_direct_blas.pyx new file mode 100644 index 000000000000..a5f7dbbef551 --- /dev/null +++ b/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/_direct_blas.pyx @@ -0,0 +1,91 @@ +#cython: language_level=3 +#cython: boundscheck=False +#cython: wraparound=False +""" +Part 2: directly uses scipy's blas_int without an intermediate .pxd layer. + +This simulates a downstream package that directly adapts to SciPy's +blas_int type, using it throughout without int->blas_int conversion. +""" + +from scipy.linalg.cython_blas cimport blas_int, ddot, daxpy, dgemm, dnrm2 +from scipy.linalg.cython_lapack cimport dgetrf + +from libc.stdlib cimport malloc, free + + +cpdef double direct_ddot(double[:] x, double[:] y): + cdef: + blas_int n = x.shape[0] + blas_int incx = 1, incy = 1 + return ddot(&n, &x[0], &incx, &y[0], &incy) + + +cpdef direct_daxpy(double alpha, double[:] x, double[:] y): + cdef: + blas_int n = x.shape[0] + blas_int incx = 1, incy = 1 + daxpy(&n, &alpha, &x[0], &incx, &y[0], &incy) + + +cpdef double direct_dnrm2(double[:] x): + cdef: + blas_int n = x.shape[0] + blas_int incx = x.strides[0] // sizeof(double) + double result = dnrm2(&n, &x[0], &incx) + return result + + +cpdef direct_dgemm(double alpha, double[::1,:] a, double[::1,:] b, + double beta, double[::1,:] c): + """Matrix multiply assuming Fortran-contiguous arrays.""" + cdef: + char transa = b'N' + char transb = b'N' + blas_int m = a.shape[0] + blas_int k = a.shape[1] + blas_int n = b.shape[1] + blas_int lda = m, ldb = k, ldc = m + dgemm(&transa, &transb, &m, &n, &k, + &alpha, &a[0, 0], &lda, &b[0, 0], &ldb, &beta, &c[0, 0], &ldc) + + +cpdef int direct_dgetrf(double[::1,:] a, int[:] ipiv_out): + """LU factorization using blas_int directly for ipiv.""" + cdef: + blas_int m = a.shape[0] + blas_int n = a.shape[1] + blas_int lda = m + blas_int info + blas_int *ipiv + int min_mn = min(a.shape[0], a.shape[1]) + int i + + ipiv = malloc(min_mn * sizeof(blas_int)) + if ipiv == NULL: + return -1000 + + dgetrf(&m, &n, &a[0, 0], &lda, ipiv, &info) + + for i in range(min_mn): + ipiv_out[i] = ipiv[i] + + free(ipiv) + return info + + +cpdef int direct_dgetrf_2(double[::1,:] a, blas_int[:] ipiv): + """Similar to `direct_dgetrf` but accepts `ipiv` as a `blas_int` array.""" + cdef: + blas_int m = a.shape[0] + blas_int n = a.shape[1] + blas_int lda = m + blas_int info + + dgetrf(&m, &n, &a[0, 0], &lda, &ipiv[0], &info) + return info + + +cpdef int get_blas_int_size(): + """Return sizeof(blas_int) to verify correct type at compile+runtime.""" + return sizeof(blas_int) diff --git a/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/meson.build b/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/meson.build new file mode 100644 index 000000000000..dd144cf1156d --- /dev/null +++ b/scipy/linalg/tests/_cython_examples/ilp64_test_package/src/ilp64_test_package/meson.build @@ -0,0 +1,58 @@ +# Check if scipy's cython_blas exports blas_int (ILP64 support). +# This is the pattern that downstream packages like scikit-learn and +# statsmodels use to detect and adapt to ILP64 builds. +scipy_has_blas_int = cy.compiles( + 'from scipy.linalg.cython_blas cimport blas_int', + name: 'scipy cython_blas blas_int' +) + +_blas_int_conf = configuration_data() +if scipy_has_blas_int + _blas_int_conf.set('BLAS_INT_DEF', + 'from scipy.linalg.cython_blas cimport blas_int') +else + # Backwards compat for scipy<1.18.0 + _blas_int_conf.set('BLAS_INT_DEF', 'ctypedef int blas_int') +endif + +_blas_int_pxi = configure_file( + input: '_blas_int.pxi.in', + output: '_blas_int.pxi', + configuration: _blas_int_conf, +) + +# Part 1: Internal .pxd + wrappers (sklearn-like pattern) +py.extension_module( + '_blas_lapack_wrappers', + '_blas_lapack_wrappers.pyx', + _blas_int_pxi, + install: true, + subdir: 'ilp64_test_package', + cython_args: cython_args, + c_args: ['-DCYTHON_CCOMPLEX=0'], +) + +# Part 1: Consumer that cimports from the internal .pxd +py.extension_module( + '_consumer', + '_consumer.pyx', + install: true, + subdir: 'ilp64_test_package', + cython_args: cython_args, + c_args: ['-DCYTHON_CCOMPLEX=0'], +) + +# Part 2: Direct use of scipy's blas_int +py.extension_module( + '_direct_blas', + '_direct_blas.pyx', + install: true, + subdir: 'ilp64_test_package', + cython_args: cython_args, + c_args: ['-DCYTHON_CCOMPLEX=0'], +) + +py.install_sources( + '__init__.py', + subdir: 'ilp64_test_package', +) diff --git a/scipy/linalg/tests/_cython_examples/ilp64_test_package/tests/test_direct.py b/scipy/linalg/tests/_cython_examples/ilp64_test_package/tests/test_direct.py new file mode 100644 index 000000000000..63461354beac --- /dev/null +++ b/scipy/linalg/tests/_cython_examples/ilp64_test_package/tests/test_direct.py @@ -0,0 +1,88 @@ +"""Tests for Part 2: direct use of scipy's blas_int.""" + +import numpy as np +from numpy.testing import assert_allclose +import scipy.linalg.cython_blas as cython_blas + +from ilp64_test_package._direct_blas import ( + direct_ddot, + direct_daxpy, + direct_dnrm2, + direct_dgemm, + direct_dgetrf, + direct_dgetrf_2, + get_blas_int_size, +) + + +class TestBlasIntSize: + def test_blas_int_matches_scipy(self): + expected = cython_blas._blas_int_size() + assert get_blas_int_size() == expected + + +class TestBLASLevel1: + def test_ddot(self): + x = np.array([1.0, 2.0, 3.0]) + y = np.array([4.0, 5.0, 6.0]) + result = direct_ddot(x, y) + assert_allclose(result, np.dot(x, y)) + + def test_daxpy(self): + x = np.array([1.0, 2.0, 3.0]) + y = np.array([4.0, 5.0, 6.0]) + direct_daxpy(2.0, x, y) + assert_allclose(y, [6.0, 9.0, 12.0]) + + def test_dnrm2_large_vector(self): + x = np.zeros(2**31, dtype=np.float64) + x[-1] = 1.0 + nrm2 = direct_dnrm2(x) + if get_blas_int_size() == np.int64().itemsize: + # Internal cython_blas has ILP64 support + assert_allclose(nrm2, 1.0, atol=1e-14) + else: + # Internal cython_blas does not have ILP64 support + assert_allclose(nrm2, 0.0, atol=1e-14) + + +class TestBLASLevel3: + def test_dgemm_identity(self): + a = np.eye(3, dtype=np.float64, order='F') + b = np.arange(9, dtype=np.float64).reshape(3, 3, order='F') + c = np.empty((3, 3), dtype=np.float64, order='F') + direct_dgemm(1.0, a, b, 0.0, c) + assert_allclose(c, b) + + def test_dgemm_product(self): + a = np.array([[1, 2], [3, 4]], dtype=np.float64, order='F') + b = np.array([[5, 6], [7, 8]], dtype=np.float64, order='F') + c = np.zeros((2, 2), dtype=np.float64, order='F') + direct_dgemm(1.0, a, b, 0.0, c) + assert_allclose(c, a @ b) + + +class TestLAPACK: + def test_dgetrf(self): + a = np.array([[2, 1], [1, 3]], dtype=np.float64, order='F') + ipiv = np.empty(2, dtype=np.intc) + info = direct_dgetrf(a, ipiv) + assert info == 0 + + def test_dgetrf_rectangular(self): + a = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float64, order='F') + ipiv = np.empty(2, dtype=np.intc) + info = direct_dgetrf(a, ipiv) + assert info == 0 + + def test_dgetrf_2(self): + a = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float64, order='F') + + # direct_dgetrf_2 receives the pivot array as `blas_int` + if get_blas_int_size() == np.intc().itemsize: + int_dtype = np.int32 + else: + int_dtype = np.int64 + ipiv = np.empty(2, dtype=int_dtype) + info = direct_dgetrf_2(a, ipiv) + assert info == 0 diff --git a/scipy/linalg/tests/_cython_examples/ilp64_test_package/tests/test_wrappers.py b/scipy/linalg/tests/_cython_examples/ilp64_test_package/tests/test_wrappers.py new file mode 100644 index 000000000000..bdf2b4692a6b --- /dev/null +++ b/scipy/linalg/tests/_cython_examples/ilp64_test_package/tests/test_wrappers.py @@ -0,0 +1,72 @@ +"""Tests for Part 1: sklearn-like wrappers with internal .pxd and int->blas_int.""" + +import numpy as np +from numpy.testing import assert_allclose +import scipy.linalg.cython_blas as cython_blas + +from ilp64_test_package._consumer import ( + consumer_ddot, + consumer_daxpy, + consumer_dnrm2, + consumer_dgemm, + consumer_dgetrf, + consumer_blas_int_size, +) + + +class TestBlasIntSize: + def test_blas_int_matches_scipy(self): + expected = cython_blas._blas_int_size() + assert consumer_blas_int_size() == expected + + +class TestBLASLevel1: + def test_ddot(self): + x = np.array([1.0, 2.0, 3.0]) + y = np.array([4.0, 5.0, 6.0]) + result = consumer_ddot(x, y) + assert_allclose(result, np.dot(x, y)) + + def test_daxpy(self): + x = np.array([1.0, 2.0, 3.0]) + y = np.array([4.0, 5.0, 6.0]) + consumer_daxpy(2.0, x, y) + assert_allclose(y, [6.0, 9.0, 12.0]) + + def test_dnrm2_large_vector(self): + x = np.zeros(2**31, dtype=np.float64) + x[-1] = 1.0 + nrm2 = consumer_dnrm2(x) + # LP64 answer due to blas_int->int casts in consumer_dnrm2 + # cf a similar test in test_direct.py + assert_allclose(nrm2, 0.0, atol=1e-14) + + +class TestBLASLevel3: + def test_dgemm_identity(self): + a = np.eye(3, dtype=np.float64, order='F') + b = np.arange(9, dtype=np.float64).reshape(3, 3, order='F') + c = np.empty((3, 3), dtype=np.float64, order='F') + consumer_dgemm(1.0, a, b, 0.0, c) + assert_allclose(c, b) + + def test_dgemm_product(self): + a = np.array([[1, 2], [3, 4]], dtype=np.float64, order='F') + b = np.array([[5, 6], [7, 8]], dtype=np.float64, order='F') + c = np.zeros((2, 2), dtype=np.float64, order='F') + consumer_dgemm(1.0, a, b, 0.0, c) + assert_allclose(c, a @ b) + + +class TestLAPACK: + def test_dgetrf(self): + a = np.array([[2, 1], [1, 3]], dtype=np.float64, order='F') + ipiv = np.empty(2, dtype=np.intc) + info = consumer_dgetrf(a, ipiv) + assert info == 0 + + def test_dgetrf_rectangular(self): + a = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float64, order='F') + ipiv = np.empty(2, dtype=np.intc) + info = consumer_dgetrf(a, ipiv) + assert info == 0 diff --git a/tools/check_installation.py b/tools/check_installation.py index 32287a47c722..9ca7739f97d6 100644 --- a/tools/check_installation.py +++ b/tools/check_installation.py @@ -35,9 +35,13 @@ # Files whose installation path will be different from original one changed_installed_path = { 'scipy/_build_utils/tests/test_scipy_version.py': - 'scipy/_lib/tests/test_scipy_version.py' + 'scipy/_lib/tests/test_scipy_version.py', + 'scipy/linalg/tests/_cython_examples/ilp64_test_package/tests/test_wrappers.py': '', } +# Subdirs in the source tree which should not be installed +exclude_paths = ['scipy/linalg/tests/_cython_examples/ilp64_test_package/',] + def main(install_dir, no_tests): INSTALLED_DIR = os.path.join(ROOT_DIR, install_dir) @@ -102,8 +106,13 @@ def get_test_files(dir, ext="py"): test_files = dict() underscore = "_" if ext == "so" else "" for path in glob.glob(f'{dir}/**/{underscore}test_*.{ext}', recursive=True): + if any(submodule_path in path for submodule_path in submodule_paths): continue + + if any(exclude_path in path for exclude_path in exclude_paths): + continue + suffix_path = get_suffix_path(path, 3) suffix_path = changed_installed_path.get(suffix_path, suffix_path) test_files[suffix_path] = path