RHAIENG-2042, RHAIENG-2111: chore(trustyai): make the dnf cache changes to avoid running out of disk space (#1732)

* opendatahub-io#2697 (comment)

```
[7/7] STEP 15/15: LABEL "architecture"="aarch64" "vcs-type"="git" "vcs-ref"="2810b3b8179aa064eb92cced6b4c54d3879a3d9a" "org.opencontainers.image.revision"="2810b3b8179aa064eb92cced6b4c54d3879a3d9a" "quay.expires-after"="5d" "build-date"="2025-11-25T18:44:58Z" "version"="on-pr-2810b3b8179aa064eb92cced6b4c54d3879a3d9a" "io.openshift.tags"="odh-workbench-jupyter-trustyai-cpu-py312" "url"="https://github.com/jiridanek/notebooks" "release"="1764096054" "git.url"="https://github.com/jiridanek/notebooks" "git.commit"="2810b3b8179aa064eb92cced6b4c54d3879a3d9a"
  [7/7] COMMIT quay.io/rhoai/pull-request-pipelines:notebooks-2810b3b8179aa064eb92cced6b4c54d3879a3d9a-linux-m2xlarge-arm64
  time="2025-11-25T19:49:33Z" level=error msg="While recovering from a failure (applying layer diff), error deleting layer \"f79b5b22e93648e277de7a1a6b8a06c30ef828cfc902b0f40f066f40a2b81a85\": creating root temp directory /var/lib/containers/storage/overlay-layers/tmp failed: mkdir /var/lib/containers/storage/overlay-layers/tmp: no space left on device"
  Error: committing container for step {Env:[MINIMAL_SOURCE_CODE=jupyter/minimal/ubi9-python-3.12 DATASCIENCE_SOURCE_CODE=jupyter/datascience/ubi9-python-3.12 TRUSTYAI_SOURCE_CODE=jupyter/trustyai/ubi9-python-3.12 JUPYTER_REUSABLE_UTILS=jupyter/utils container=oci PYTHON=python3.12 MPI_HOME=/usr/lib64/openmpi APP_ROOT=/opt/app-root HOME=/opt/app-root/src VIRTUAL_ENV=/opt/app-root PIP_NO_CACHE_DIR=off UV_NO_CACHE=true PIP_DISABLE_PIP_VERSION_CHECK=1 PYTHONUNBUFFERED=1 PYTHONIOENCODING=utf-8 LANG=en_US.UTF-8 LC_ALL=en_US.UTF-8 PS1=(app-root) \w$  CNB_USER_ID=1001 CNB_GROUP_ID=0 PIP_CONFIG_FILE=/opt/app-root/pip.conf UV_CONFIG_FILE=/opt/app-root/uv.toml LD_PRELOAD=/usr/lib64/libjemalloc.so.2 PATH=/usr/local/texlive/bin/linux:/usr/local/pandoc/bin:/opt/app-root/src/bin:/opt/app-root/src/.local/bin:/opt/app-root/bin:/usr/local/sbin:/usr/local/bin:/usr/lib64/openmpi/bin:/usr/sbin:/usr/bin:/sbin:/bin LD_LIBRARY_PATH=/usr/lib:/usr/lib64] Command:label Args:[architecture aarch64 vcs-type git vcs-ref 2810b3b org.opencontainers.image.revision 2810b3b quay.expires-after 5d build-date 2025-11-25T18:44:58Z version on-pr-2810b3b8179aa064eb92cced6b4c54d3879a3d9a io.openshift.tags odh-workbench-jupyter-trustyai-cpu-py312 url https://github.com/jiridanek/notebooks release 1764096054 git.url https://github.com/jiridanek/notebooks git.commit 2810b3b] Flags:[] Attrs:map[] Message:LABEL "architecture" "aarch64" "vcs-type" "git" "vcs-ref" "2810b3b8179aa064eb92cced6b4c54d3879a3d9a" "org.opencontainers.image.revision" "2810b3b8179aa064eb92cced6b4c54d3879a3d9a" "quay.expires-after" "5d" "build-date" "2025-11-25T18:44:58Z" "version" "on-pr-2810b3b8179aa064eb92cced6b4c54d3879a3d9a" "io.openshift.tags" "odh-workbench-jupyter-trustyai-cpu-py312" "url" "https://github.com/jiridanek/notebooks" "release" "1764096054" "git.url" "https://github.com/jiridanek/notebooks" "git.commit" "2810b3b8179aa064eb92cced6b4c54d3879a3d9a" Heredocs:[] Original:LABEL "architecture"="aarch64" "vcs-type"="git" "vcs-ref"="2810b3b8179aa064eb92cced6b4c54d3879a3d9a" "org.opencontainers.image.revision"="2810b3b8179aa064eb92cced6b4c54d3879a3d9a" "quay.expires-after"="5d" "build-date"="2025-11-25T18:44:58Z" "version"="on-pr-2810b3b8179aa064eb92cced6b4c54d3879a3d9a" "io.openshift.tags"="odh-workbench-jupyter-trustyai-cpu-py312" "url"="https://github.com/jiridanek/notebooks" "release"="1764096054" "git.url"="https://github.com/jiridanek/notebooks" "git.commit"="2810b3b8179aa064eb92cced6b4c54d3879a3d9a"}: copying layers and metadata for container "0713091dd70494d22b29c5698ccf85a73460dc31df0ce4625c59a169707f81a6": writing blob: adding layer with blob "sha256:9ac8368926bfb58b2dd0ccc99dc6c850f4581f605a124f7629d2e437cd67b734"/""/"sha256:9ac8368926bfb58b2dd0ccc99dc6c850f4581f605a124f7629d2e437cd67b734": unpacking failed (error: exit status 1; output: write /opt/app-root/lib/python3.12/site-packages/mysql/vendor/plugin/authentication_ldap_sasl_client.so: no space left on device)
```

Author: jiridanek

jupyter/trustyai/ubi9-python-3.12/Dockerfile.cpu

@@ -192,7 +192,7 @@ USER 0
 
 # Install jre that is needed to run the trustyai library
 # Also install runtime libraries for s390x/ppc64le
-RUN /bin/bash <<'EOF'
+RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked,id=notebooks-dnf /bin/bash <<'EOF'
 set -Eeuxo pipefail
 INSTALL_PKGS="java-17-openjdk"
 ARCH=$(uname -m)

jupyter/trustyai/ubi9-python-3.12/Dockerfile.konflux.cpu

@@ -52,7 +52,8 @@ EOF
 
 ### END Subscribe with subscription manager
 
-RUN --mount=type=cache,target=/root/.cache/uv /bin/bash <<'EOF'
+RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked,id=notebooks-dnf \
+    --mount=type=cache,target=/root/.cache/uv /bin/bash <<'EOF'
 set -Eeuxo pipefail
 # RHAIENG-2111: Looking in indexes: https://console.redhat.com/api/pypi/public-rhai/rhoai/3.0/cpu-ubi9/simple/
 #  ERROR: Could not find a version that satisfies the requirement uv (from versions: none)
@@ -105,11 +106,9 @@ EOF
 ### END upgrade first to avoid fixable vulnerabilities
 
 # Install useful OS packages
-RUN /bin/bash <<'EOF'
+RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked,id=notebooks-dnf /bin/bash <<'EOF'
 set -Eeuxo pipefail
-dnf install -y perl mesa-libGL skopeo
-dnf clean all
-rm -rf /var/cache/yum
+dnf install -y --setopt=keepcache=1 perl mesa-libGL skopeo
 EOF
 
 # Other apps and tools installed as default user
@@ -171,11 +170,9 @@ WORKDIR /opt/app-root/bin
 USER root
 
 # Install useful OS packages
-RUN /bin/bash <<'EOF'
+RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked,id=notebooks-dnf /bin/bash <<'EOF'
 set -Eeuxo pipefail
-dnf install -y jq unixODBC unixODBC-devel postgresql git-lfs libsndfile libxcrypt-compat
-dnf clean all
-rm -rf /var/cache/yum
+dnf install -y --setopt=keepcache=1 jq unixODBC unixODBC-devel postgresql git-lfs libsndfile libxcrypt-compat
 EOF
 
 # Copy dynamically-linked mongocli built in earlier build stage
@@ -211,16 +208,15 @@ USER 0
 
 # Install jre that is needed to run the trustyai library
 # Also install runtime libraries for s390x/ppc64le
-RUN /bin/bash <<'EOF'
+RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked,id=notebooks-dnf /bin/bash <<'EOF'
 set -Eeuxo pipefail
 INSTALL_PKGS="java-17-openjdk"
 ARCH=$(uname -m)
 if [ "$ARCH" = "s390x" ] || [ "$ARCH" = "ppc64le" ]; then
     # Add runtime libraries needed for s390x/ppc64le (OpenBLAS for PyTorch/NumPy)
     INSTALL_PKGS="$INSTALL_PKGS openblas openblas-threads"
 fi
-dnf install -y --setopt=tsflags=nodocs $INSTALL_PKGS
-dnf -y clean all --enablerepo='*'
+dnf install -y --setopt=tsflags=nodocs --setopt=keepcache=1 $INSTALL_PKGS
 # Create symlink for compatibility (openblas package provides libopenblasp.so.0 but PyTorch looks for libopenblas.so.0)
 if [ "$ARCH" = "s390x" ] || [ "$ARCH" = "ppc64le" ]; then
     ln -sf /usr/lib64/libopenblasp.so.0 /usr/lib64/libopenblas.so.0