Merge remote-tracking branch 'upstream/main'

Author: jiridanek

jupyter/datascience/ubi9-python-3.12/Dockerfile.cpu

@@ -40,6 +40,13 @@ EOF
 # cpu-base         #
 ####################
 FROM ${BASE_IMAGE} AS cpu-base
+USER 0
+RUN /bin/bash <<'EOF'
+set -Eeuxo pipefail
+if command -v subscription-manager &> /dev/null; then
+  subscription-manager identity &>/dev/null && subscription-manager refresh || echo "Not registered, skipping refresh."
+fi
+EOF
 
 WORKDIR /opt/app-root/bin
 
@@ -144,7 +151,13 @@ RUN --mount=type=cache,target=/root/.cache/pip \
 set -Eeuxo pipefail
 if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then
     # Install build dependencies (shared for pyarrow and onnx)
-    dnf install -y cmake make gcc-c++ pybind11-devel wget
+    packages=(
+        cmake make gcc-c++ pybind11-devel wget
+        # pyarrow -DARROW_S3=ON pulls in AWS C++ SDK
+        # libcurl-devel is required to build AWS C++ SDK
+        libcurl-devel openssl-devel
+    )
+    dnf install -y "${packages[@]}"
     dnf clean all
     # Build and collect pyarrow wheel
     git clone --depth 1 --branch "apache-arrow-17.0.0" https://github.com/apache/arrow.git
@@ -173,7 +186,8 @@ if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then
     make -j$(nproc) VERBOSE=1
     make install -j$(nproc)
     cd ../../python
-    pip install --no-cache-dir -r requirements-build.txt
+    # aipcc index is missing cython, and maybe more
+    pip install --no-cache-dir --extra-index-url https://pypi.org/simple -r requirements-build.txt
     PYARROW_WITH_PARQUET=1 \
     PYARROW_WITH_DATASET=1 \
     PYARROW_WITH_FILESYSTEM=1 \
@@ -224,10 +238,12 @@ if [ "${TARGETARCH}" = "ppc64le" ]; then
     cd onnx
     git checkout ${ONNX_VERSION}
     git submodule update --init --recursive
-    pip install --no-cache-dir -r requirements.txt
+    # aipcc index does not have numpy>=1.22
+    pip install --no-cache-dir --extra-index-url https://pypi.org/simple -r requirements.txt
     CMAKE_ARGS="-DPython3_EXECUTABLE=$(which python3.12)"
     export CMAKE_ARGS
-    pip wheel . -w /root/onnx_wheel
+    # protobuf>=4.25.1
+    pip wheel --extra-index-url https://pypi.org/simple . -w /root/onnx_wheel
 else
     echo "Skipping ONNX build on non-Power"
     mkdir -p /root/onnx_wheel
@@ -328,7 +344,8 @@ COPY --from=pyarrow-builder /tmp/wheels /tmp/wheels
 RUN /bin/bash <<'EOF'
 set -Eeuxo pipefail
 if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then
-    pip install --no-cache-dir /tmp/wheels/*.whl
+    # aipcc is lacking numpy>=1.16.6
+    pip install --no-cache-dir --extra-index-url https://pypi.org/simple /tmp/wheels/*.whl
 else
     echo "Skipping wheel install for $TARGETARCH"
 fi
@@ -342,7 +359,8 @@ COPY --from=onnx-builder /root/onnx_wheel/ /onnxwheels/
 RUN /bin/bash <<'EOF'
 set -Eeuxo pipefail
 if [ "${TARGETARCH}" = "ppc64le" ]; then
-    pip install --no-cache-dir /onnxwheels/*.whl
+    # aipcc is sure to lack something, so add extra index preemptively
+    pip install --no-cache-dir --extra-index-url https://pypi.org/simple /onnxwheels/*.whl
 else
     echo "Skipping ONNX/OpenBLAS install on non-Power"
 fi
@@ -368,6 +386,26 @@ else
 fi
 EOF
 
+RUN /bin/bash <<'EOF'
+set -Eeuxo pipefail
+if [ "${TARGETARCH}" = "ppc64le" ]; then
+  packages=(
+    # required to compile pillow
+    zlib-devel libjpeg-turbo-devel
+    # optional pillow deps https://pillow.readthedocs.io/en/latest/installation/building-from-source.html#external-libraries
+    #libtiff-devel libwebp-devel openjpeg2-devel lcms2-devel freetype-devel
+    #libimagequant-devel harfbuzz-devel fribidi-devel
+
+    # required to compile maturin
+    openssl-devel
+
+    # required to compile scikit-learn
+    gcc-gfortran
+  )
+  dnf install -y "${packages[@]}"
+fi
+EOF
+
 USER 1001:0
 
 # Install Python packages and Jupyterlab extensions from pylock.toml
@@ -382,10 +420,12 @@ echo "Installing software and packages"
 #  we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common.
 if [ "$TARGETARCH" = "ppc64le" ] || [ "$TARGETARCH" = "s390x" ]; then
     # We need special flags and environment variables when building packages
+    # aipcc does not have some dependencies to build cryptography==43.0.3 on ppc64le, so we need to use pypi.org/simple
     GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 \
     CFLAGS="-O3" CXXFLAGS="-O3" \
     uv pip install --strict --no-deps --no-cache --no-config --no-progress \
         --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match \
+        --extra-index-url https://pypi.org/simple \
         --requirements=./pylock.toml
 else
     # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`,