From f270c220b29acbe1ac92a41131b51e7dfa9dc2f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Fri, 28 Nov 2025 14:24:31 +0100 Subject: [PATCH 1/3] RHAIENG-2174: build(codeserver,trustyai): fix build failure "no space left on device" due to AIPCC base disabling `uv` cache --- codeserver/ubi9-python-3.12/Dockerfile.cpu | 7 +++---- codeserver/ubi9-python-3.12/Dockerfile.konflux.cpu | 7 +++---- jupyter/trustyai/ubi9-python-3.12/Dockerfile.cpu | 6 +++--- jupyter/trustyai/ubi9-python-3.12/Dockerfile.konflux.cpu | 6 +++--- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/codeserver/ubi9-python-3.12/Dockerfile.cpu b/codeserver/ubi9-python-3.12/Dockerfile.cpu index a95c16734..3af1da047 100644 --- a/codeserver/ubi9-python-3.12/Dockerfile.cpu +++ b/codeserver/ubi9-python-3.12/Dockerfile.cpu @@ -59,7 +59,7 @@ pip install --no-cache-dir uv source ./devel_env_setup.sh # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`, # we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common. -UV_LINK_MODE=copy uv pip install --strict --no-deps --refresh --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml +UV_NO_CACHE=false UV_LINK_MODE=copy uv pip install --strict --no-deps --refresh --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml EOF # dummy file to make image build wait for this stage @@ -291,7 +291,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=cache,from=whl-cache,source=/wheelsdir/,target=/wheelsdir/,rw /bin/bash <<'EOF' set -Eeuxo pipefail if [[ $(uname -m) == "ppc64le" ]] || [[ $(uname -m) == "s390x" ]]; then - uv pip install /wheelsdir/*.whl + UV_NO_CACHE=false uv pip install /wheelsdir/*.whl fi EOF @@ -300,8 +300,7 @@ RUN --mount=type=cache,target=/root/.cache/uv /bin/bash <<'EOF' set -Eeuxo pipefail echo "Installing softwares and packages" # we can ensure wheels are consumed from the cache only by restricting internet access for uv install with '--offline' flag -# TODO(jdanek): seen some builds fail on GitHub Actions with --offline and see no need to limit ourselves to the cache, will remove this -UV_LINK_MODE=copy uv pip install --cache-dir /root/.cache/uv --requirements=./pylock.toml +UV_NO_CACHE=false UV_LINK_MODE=copy uv pip install --offline --cache-dir /root/.cache/uv --requirements=./pylock.toml # Note: debugpy wheel availabe on pypi (in uv cache) is none-any but bundles amd64.so files # Build debugpy from source instead UV_LINK_MODE=copy uv pip install --no-cache git+https://github.com/microsoft/debugpy.git@v$(grep -A1 '\"debugpy\"' ./pylock.toml | grep -Eo '\b[0-9\.]+\b') diff --git a/codeserver/ubi9-python-3.12/Dockerfile.konflux.cpu b/codeserver/ubi9-python-3.12/Dockerfile.konflux.cpu index b1112f7f2..66aec6026 100644 --- a/codeserver/ubi9-python-3.12/Dockerfile.konflux.cpu +++ b/codeserver/ubi9-python-3.12/Dockerfile.konflux.cpu @@ -62,7 +62,7 @@ pip install --no-cache-dir uv source ./devel_env_setup.sh # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`, # we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common. -UV_LINK_MODE=copy uv pip install --strict --no-deps --refresh --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml +UV_NO_CACHE=false UV_LINK_MODE=copy uv pip install --strict --no-deps --refresh --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --requirements=./pylock.toml EOF # dummy file to make image build wait for this stage @@ -302,7 +302,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \ --mount=type=cache,from=whl-cache,source=/wheelsdir/,target=/wheelsdir/,rw /bin/bash <<'EOF' set -Eeuxo pipefail if [[ $(uname -m) == "ppc64le" ]] || [[ $(uname -m) == "s390x" ]]; then - uv pip install /wheelsdir/*.whl + UV_NO_CACHE=false uv pip install /wheelsdir/*.whl fi EOF @@ -311,8 +311,7 @@ RUN --mount=type=cache,target=/root/.cache/uv /bin/bash <<'EOF' set -Eeuxo pipefail echo "Installing softwares and packages" # we can ensure wheels are consumed from the cache only by restricting internet access for uv install with '--offline' flag -# TODO(jdanek): seen some builds fail on GitHub Actions with --offline and see no need to limit ourselves to the cache, will remove this -UV_LINK_MODE=copy uv pip install --cache-dir /root/.cache/uv --requirements=./pylock.toml +UV_NO_CACHE=false UV_LINK_MODE=copy uv pip install --offline --cache-dir /root/.cache/uv --requirements=./pylock.toml # Note: debugpy wheel availabe on pypi (in uv cache) is none-any but bundles amd64.so files # Build debugpy from source instead UV_LINK_MODE=copy uv pip install --no-cache git+https://github.com/microsoft/debugpy.git@v$(grep -A1 '\"debugpy\"' ./pylock.toml | grep -Eo '\b[0-9\.]+\b') diff --git a/jupyter/trustyai/ubi9-python-3.12/Dockerfile.cpu b/jupyter/trustyai/ubi9-python-3.12/Dockerfile.cpu index d30ce84f5..2f989acae 100644 --- a/jupyter/trustyai/ubi9-python-3.12/Dockerfile.cpu +++ b/jupyter/trustyai/ubi9-python-3.12/Dockerfile.cpu @@ -51,7 +51,7 @@ source ./devel_env_setup.sh # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`, # we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common. # RHAIENG-2111: building pandas requires cython from PyPI -UV_LINK_MODE=copy uv pip install --strict --no-deps --refresh --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --extra-index-url https://pypi.org/simple --requirements=./pylock.toml +UV_NO_CACHE=false UV_LINK_MODE=copy uv pip install --strict --no-deps --refresh --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --extra-index-url https://pypi.org/simple --requirements=./pylock.toml EOF #################### @@ -227,7 +227,7 @@ RUN --mount=type=cache,from=whl-cache,source=/wheelsdir/,target=/wheelsdir/,rw \ set -Eeuxo pipefail ARCH=$(uname -m) if [ "$ARCH" = "ppc64le" ] || [ "$ARCH" = "s390x" ]; then - UV_LINK_MODE=copy uv pip install /wheelsdir/*.whl accelerate --cache-dir /root/.cache/uv + UV_NO_CACHE=false UV_LINK_MODE=copy uv pip install /wheelsdir/*.whl accelerate --cache-dir /root/.cache/uv fi EOF @@ -236,7 +236,7 @@ set -Eeuxo pipefail echo "Installing softwares and packages" # we can ensure wheels are consumed from the cache only by restricting internet access for uv install with '--offline' flag # RHAIENG-2111: building pandas requires cython from PyPI -UV_LINK_MODE=copy uv pip install --cache-dir /root/.cache/uv --extra-index-url https://pypi.org/simple --requirements=./pylock.toml +UV_NO_CACHE=false UV_LINK_MODE=copy uv pip install --offline --cache-dir /root/.cache/uv --extra-index-url https://pypi.org/simple --requirements=./pylock.toml # Note: debugpy wheel availabe on pypi (in uv cache) is none-any but bundles amd64.so files # Build debugpy from source instead UV_LINK_MODE=copy uv pip install --no-cache git+https://github.com/microsoft/debugpy.git@v$(grep -A1 '\"debugpy\"' ./pylock.toml | grep -Eo '\b[0-9\.]+\b') diff --git a/jupyter/trustyai/ubi9-python-3.12/Dockerfile.konflux.cpu b/jupyter/trustyai/ubi9-python-3.12/Dockerfile.konflux.cpu index ef77a1b3a..f7074d68f 100644 --- a/jupyter/trustyai/ubi9-python-3.12/Dockerfile.konflux.cpu +++ b/jupyter/trustyai/ubi9-python-3.12/Dockerfile.konflux.cpu @@ -64,7 +64,7 @@ source ./devel_env_setup.sh # This may have to download and compile some dependencies, and as we don't lock requirements from `build-system.requires`, # we often don't know the correct hashes and `--require-hashes` would therefore fail on non amd64, where building is common. # RHAIENG-2111: building pandas requires cython from PyPI -UV_LINK_MODE=copy uv pip install --strict --no-deps --refresh --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --extra-index-url https://pypi.org/simple --requirements=./pylock.toml +UV_NO_CACHE=false UV_LINK_MODE=copy uv pip install --strict --no-deps --refresh --no-config --no-progress --verify-hashes --compile-bytecode --index-strategy=unsafe-best-match --extra-index-url https://pypi.org/simple --requirements=./pylock.toml EOF #################### @@ -243,7 +243,7 @@ RUN --mount=type=cache,from=whl-cache,source=/wheelsdir/,target=/wheelsdir/,rw \ set -Eeuxo pipefail ARCH=$(uname -m) if [ "$ARCH" = "ppc64le" ] || [ "$ARCH" = "s390x" ]; then - UV_LINK_MODE=copy uv pip install /wheelsdir/*.whl accelerate --cache-dir /root/.cache/uv + UV_NO_CACHE=false UV_LINK_MODE=copy uv pip install /wheelsdir/*.whl accelerate --cache-dir /root/.cache/uv fi EOF @@ -252,7 +252,7 @@ set -Eeuxo pipefail echo "Installing softwares and packages" # we can ensure wheels are consumed from the cache only by restricting internet access for uv install with '--offline' flag # RHAIENG-2111: building pandas requires cython from PyPI -UV_LINK_MODE=copy uv pip install --cache-dir /root/.cache/uv --extra-index-url https://pypi.org/simple --requirements=./pylock.toml +UV_NO_CACHE=false UV_LINK_MODE=copy uv pip install --offline --cache-dir /root/.cache/uv --extra-index-url https://pypi.org/simple --requirements=./pylock.toml # Note: debugpy wheel availabe on pypi (in uv cache) is none-any but bundles amd64.so files # Build debugpy from source instead UV_LINK_MODE=copy uv pip install --no-cache git+https://github.com/microsoft/debugpy.git@v$(grep -A1 '\"debugpy\"' ./pylock.toml | grep -Eo '\b[0-9\.]+\b') From 39a586922a6175424b532f7c91f25b7d72a4aa26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Fri, 28 Nov 2025 14:26:19 +0100 Subject: [PATCH 2/3] RHOAIENG-28577: tekton(trustyai): enable on-pull-request ppc64le build --- .../odh-workbench-jupyter-trustyai-cpu-py312-pull-request.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.tekton/odh-workbench-jupyter-trustyai-cpu-py312-pull-request.yaml b/.tekton/odh-workbench-jupyter-trustyai-cpu-py312-pull-request.yaml index 4667a5e32..9355b64b4 100644 --- a/.tekton/odh-workbench-jupyter-trustyai-cpu-py312-pull-request.yaml +++ b/.tekton/odh-workbench-jupyter-trustyai-cpu-py312-pull-request.yaml @@ -45,6 +45,7 @@ spec: value: - linux-extra-fast/amd64 - linux-m2xlarge/arm64 + - linux/ppc64le - linux/s390x - name: image-expires-after value: 5d From 74dfec9d330402390beabee342bab9ada4cd0b19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jiri=20Dan=C4=9Bk?= Date: Fri, 28 Nov 2025 18:55:40 +0100 Subject: [PATCH 3/3] enable codeready-builder/crb repository --- jupyter/trustyai/ubi9-python-3.12/devel_env_setup.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/jupyter/trustyai/ubi9-python-3.12/devel_env_setup.sh b/jupyter/trustyai/ubi9-python-3.12/devel_env_setup.sh index 87ef7d530..a25007547 100755 --- a/jupyter/trustyai/ubi9-python-3.12/devel_env_setup.sh +++ b/jupyter/trustyai/ubi9-python-3.12/devel_env_setup.sh @@ -29,6 +29,14 @@ if [[ $(uname -m) == "ppc64le" ]] || [[ $(uname -m) == "s390x" ]]; then # install development packages dnf install -y --setopt=keepcache=1 https://dl.fedoraproject.org/pub/epel/epel-release-latest-9.noarch.rpm + dnf clean all + # enable codeready-builder/crb repository + dnf install -y dnf-plugins-core + if command -v subscription-manager &> /dev/null; then + subscription-manager repos --enable "codeready-builder-for-rhel-9-$(uname -m)-rpms" + else + dnf config-manager --set-enabled crb + fi # patchelf: needed by `auditwheel repair` dnf install -y --setopt=keepcache=1 fribidi-devel lcms2-devel libimagequant-devel patchelf \ libraqm-devel openjpeg2-devel tcl-devel tk-devel unixODBC-devel