Konflux: switch to hermetic builds

.gitignore

@@ -9,3 +9,6 @@ __pycache__/
 
 # generated by `make all`
 /bin/
+
+# generated by `hermeto` when pulling the dependencies locally with hermeto
+hermeto-output/

.tekton/coreos-assembler-pull-request.yaml

@@ -34,10 +34,20 @@ spec:
     value: Dockerfile
   - name: path-context
     value: .
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}, {"path": "ci/hermetic", "type": "generic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
+  - name: build-args
+    value: ["NO_NETWORK=1"]
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

.tekton/coreos-assembler-push.yaml

@@ -31,10 +31,20 @@ spec:
     value: Dockerfile
   - name: path-context
     value: .
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}, {"path": "ci/hermetic", "type": "generic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
+  - name: build-args
+    value: ["NO_NETWORK=1"]
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

.tekton/kola-nfs-pull-request.yaml

@@ -9,8 +9,11 @@ metadata:
     pipelinesascode.tekton.dev/cancel-in-progress: "true"
     pipelinesascode.tekton.dev/max-keep-runs: "3"
     pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch
-      == "main" && ( "./tests/containers/nfs/***".pathChanged() || ".tekton/kola-nfs-pull-request.yaml".pathChanged()
-      || "Containerfile".pathChanged() )
+      == "main" &&
+      ("tests/containers/nfs/***".pathChanged() ||
+       ".tekton/kola-nfs-pull-request.yaml".pathChanged() ||
+       "ci/hermetic/rpms.lock.yaml".pathChanged()
+      )
   creationTimestamp: null
   labels:
     appstudio.openshift.io/application: coreos-assembler
@@ -35,10 +38,18 @@ spec:
     value: Containerfile
   - name: path-context
     value: tests/containers/nfs
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

.tekton/kola-nfs-push.yaml

@@ -8,7 +8,10 @@ metadata:
     pipelinesascode.tekton.dev/cancel-in-progress: "false"
     pipelinesascode.tekton.dev/max-keep-runs: "3"
     pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch
-      == "main" && "./tests/containers/nfs/***".pathChanged()
+      == "main" &&
+      ("tests/containers/nfs/***".pathChanged() ||
+       "ci/hermetic/rpms.lock.yaml".pathChanged()
+      )
   creationTimestamp: null
   labels:
     appstudio.openshift.io/application: coreos-assembler
@@ -31,10 +34,18 @@ spec:
     value: Containerfile
   - name: path-context
     value: tests/containers/nfs
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

.tekton/kola-tang-pull-request.yaml

@@ -9,7 +9,11 @@ metadata:
     pipelinesascode.tekton.dev/cancel-in-progress: "true"
     pipelinesascode.tekton.dev/max-keep-runs: "3"
     pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch
-      == "main" && ( "./tests/containers/tang/***".pathChanged() || ".tekton/kola-tang-pull-request.yaml".pathChanged())
+      == "main" &&
+      ("tests/containers/tang/***".pathChanged() ||
+       ".tekton/kola-tang-pull-request.yaml".pathChanged() ||
+       "ci/hermetic/rpms.lock.yaml".pathChanged()
+      )
   creationTimestamp: null
   labels:
     appstudio.openshift.io/application: coreos-assembler
@@ -34,10 +38,18 @@ spec:
     value: ./tests/containers/tang/Containerfile
   - name: path-context
     value: .
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

.tekton/kola-tang-push.yaml

@@ -8,7 +8,10 @@ metadata:
     pipelinesascode.tekton.dev/cancel-in-progress: "false"
     pipelinesascode.tekton.dev/max-keep-runs: "3"
     pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch
-      == "main" && "./tests/containers/tang/***".pathChanged()
+      == "main" &&
+      ("tests/containers/tang/***".pathChanged() ||
+       "ci/hermetic/rpms.lock.yaml".pathChanged()
+      )
   creationTimestamp: null
   labels:
     appstudio.openshift.io/application: coreos-assembler
@@ -31,10 +34,18 @@ spec:
     value: ./tests/containers/tang/Containerfile
   - name: path-context
     value: .
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

.tekton/kola-targetcli-pull-request.yaml

@@ -9,7 +9,11 @@ metadata:
     pipelinesascode.tekton.dev/cancel-in-progress: "true"
     pipelinesascode.tekton.dev/max-keep-runs: "3"
     pipelinesascode.tekton.dev/on-cel-expression: event == "pull_request" && target_branch
-      == "main" && ( "./tests/containers/targetcli/***".pathChanged() || ".tekton/kola-targetcli-pull-request.yaml".pathChanged())
+      == "main" &&
+      ("tests/containers/targetcli/***".pathChanged() ||
+       ".tekton/kola-targetcli-pull-request.yaml".pathChanged() ||
+       "ci/hermetic/rpms.lock.yaml".pathChanged()
+      )
   creationTimestamp: null
   labels:
     appstudio.openshift.io/application: coreos-assembler
@@ -34,10 +38,18 @@ spec:
     value: ./tests/containers/targetcli/Containerfile
   - name: path-context
     value: .
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

.tekton/kola-targetcli-push.yaml

@@ -8,7 +8,10 @@ metadata:
     pipelinesascode.tekton.dev/cancel-in-progress: "false"
     pipelinesascode.tekton.dev/max-keep-runs: "3"
     pipelinesascode.tekton.dev/on-cel-expression: event == "push" && target_branch
-      == "main" && "./tests/containers/targetcli/***".pathChanged()
+      == "main" &&
+      ("tests/containers/targetcli/***".pathChanged() ||
+       "ci/hermetic/rpms.lock.yaml".pathChanged()
+      )
   creationTimestamp: null
   labels:
     appstudio.openshift.io/application: coreos-assembler
@@ -31,10 +34,18 @@ spec:
     value: ./tests/containers/targetcli/Containerfile
   - name: path-context
     value: .
+  - name: hermetic
+    value: false
+  - name: prefetch-input
+    value: '[{"type": "rpm", "path": "ci/hermetic"}]'
+  # Note: to be removed once rpm fully supported
+  # https://github.com/hermetoproject/hermeto?tab=readme-ov-file#package-managers
+  - name: dev-package-managers
+    value: true
   pipelineRef:
     params:
     - name: bundle
-      value: quay.io/konflux-ci/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:5bc58ee8213aaa3da4c1d67e380007097fbbbfb4dca3d0711777bd2b0d115da1
+      value: quay.io/jcapitao/tekton-catalog/pipeline-docker-build-multi-platform-oci-ta@sha256:a3993688715cba973af5e7fba95bc91f92673e8491f2524853736161974334fb
     - name: name
       value: docker-build-multi-platform-oci-ta
     - name: kind

Dockerfile

@@ -2,6 +2,8 @@
 # https://github.com/openshift/release/tree/master/ci-operator/config/coreos/coreos-assembler/coreos-coreos-assembler-main.yaml
 FROM quay.io/fedora/fedora:42
 WORKDIR /root/containerbuild
+# This variable is enabled by Konflux to build the container image hermatically.
+ARG NO_NETWORK=0
 
 # Keep this Dockerfile idempotent for local development rebuild use cases.
 USER root

build.sh

@@ -31,6 +31,7 @@ srcdir=$(pwd)
 
 configure_yum_repos() {
     [ "${arch}" == "riscv64" ] && return # No continuous repo for riscv64 yet
+    [ "${NO_NETWORK}" == "1" ] && return
     local version_id
     version_id=$(. /etc/os-release && echo ${VERSION_ID})
     # Add continuous tag for latest build tools and mark as required so we
@@ -46,7 +47,7 @@ install_rpms() {
     # First, a general update; this is best practice.  We also hit an issue recently
     # where qemu implicitly depended on an updated libusbx but didn't have a versioned
     # requires https://bugzilla.redhat.com/show_bug.cgi?id=1625641
-    yum -y distro-sync
+    [ "${NO_NETWORK}" == "0" ] && yum -y distro-sync
 
     # xargs is part of findutils, which may not be installed
     yum -y install /usr/bin/xargs
@@ -104,10 +105,16 @@ install_rpms() {
 # to CoreOS.
 install_ocp_tools() {
     [ "${arch}" == "riscv64" ] && return # No ocp tools for riscv64
-    # If $OCP_VERSION is defined we'll grab that specific version.
-    # Otherwise we'll get the latest.
-    local url="https://mirror.openshift.com/pub/openshift-v4/${arch}/clients/ocp/latest${OCP_VERSION:+-$OCP_VERSION}/openshift-client-linux.tar.gz"
-    curl -L "$url" | tar zxf - oc
+    if [ "${NO_NETWORK}" == "0" ]; then
+        # If $OCP_VERSION is defined we'll grab that specific version.
+        # Otherwise we'll get the latest.
+        local url="https://mirror.openshift.com/pub/openshift-v4/${arch}/clients/ocp/latest${OCP_VERSION:+-$OCP_VERSION}/openshift-client-linux.tar.gz"
+        curl -L "$url" | tar zxf - oc
+    else
+        local oc_archive=""
+        oc_archive=$(find /*/output/deps/generic/ -name "openshift-client-linux-${arch}.tar.gz")
+        tar zxf "$oc_archive" oc
+    fi
     mv oc /usr/bin
 }
 

ci/hermetic/Dockerfile

@@ -0,0 +1 @@
+../../Dockerfile

ci/hermetic/README.md

@@ -0,0 +1,34 @@
+# Hermetic builds for coreos-assembler and Konflux
+
+The `*.lock.yaml` files generated will be consumed by the [prefetch-dependencies-oci-ta](https://github.com/konflux-ci/build-definitions/tree/main/task/prefetch-dependencies-oci-ta) Konflux task.
+This task will download the dependencies and generate an OCI image containing them.
+Then the OCI image will be pull during the build process by the [buildah-remote-oci-ta ](https://github.com/konflux-ci/build-definitions/tree/main/task/buildah-remote-oci-ta) Konflux task.
+
+## To generate the rpms.lock.yaml file
+The script below 1. updates the packages list in 'rpms.in.yaml' and 2. updates the 'rpms.lock.yaml' afterward.
+The packages list is generated based on the content of the *deps*.txt file located in src/.
+```bash
+./update_rpms_lockfile
+```
+To test if everything is fine, you can fetch the dependencies and store them on your disk:
+```bash
+alias hermeto='podman run --rm -ti -v "$PWD:$PWD:z" -w "$PWD" quay.io/konflux-ci/hermeto:latest'
+hermeto fetch-deps --dev-package-managers --source ./ --output ./hermeto-output '{"path": ".", "type": "rpm"}'
+```
+Konflux runs similar command within [prefetch-dependencies-oci-ta](https://github.com/konflux-ci/build-definitions/tree/main/task/prefetch-dependencies-oci-ta) task.
+
+## To generate the artifacts.lock.yaml file
+```bash
+./update_artifacts_lockfile
+```
+To test if everything is fine, you can fetch the dependencies and store them on your disk:
+```bash
+alias hermeto='podman run --rm -ti -v "$PWD:$PWD:z" -w "$PWD" quay.io/konflux-ci/hermeto:latest'
+hermeto fetch-deps --source ./ --output ./hermeto-output '{"path": ".", "type": "generic"}'
+```
+
+## Download everything together
+```bash
+alias hermeto='podman run --rm -ti -v "$PWD:$PWD:z" -w "$PWD" quay.io/konflux-ci/hermeto:latest'
+hermeto fetch-deps --dev-package-managers --source ./ --output ./hermeto-output '[{"path": ".", "type": "rpm"}, {"path": ".", "type": "generic"}]'
+```

ci/hermetic/artifacts.lock.yaml

@@ -0,0 +1,15 @@
+metadata:
+  version: '1.0'
+artifacts:
+  - download_url: 'https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/latest/openshift-client-linux.tar.gz'
+    checksum: 'sha256:ee95462864b988040da09613d5aa691c1d3576813b532d2fcec1e67ccbb4164a'
+    filename: 'openshift-client-linux-x86_64.tar.gz'
+  - download_url: 'https://mirror.openshift.com/pub/openshift-v4/s390x/clients/ocp/latest/openshift-client-linux.tar.gz'
+    checksum: 'sha256:516e7dd49806a0664177c04473ac2991d1ada53d28501c552135e7abe86043e5'
+    filename: 'openshift-client-linux-s390x.tar.gz'
+  - download_url: 'https://mirror.openshift.com/pub/openshift-v4/ppc64le/clients/ocp/latest/openshift-client-linux.tar.gz'
+    checksum: 'sha256:49bd2d47add43270f936246bd8eb57123cde8e16e823180f2ad7589e9f480657'
+    filename: 'openshift-client-linux-ppc64le.tar.gz'
+  - download_url: 'https://mirror.openshift.com/pub/openshift-v4/aarch64/clients/ocp/latest/openshift-client-linux.tar.gz'
+    checksum: 'sha256:546ea80a6670b0338b05d9babaf2791ddc9c219411f67e76ec5c41de98e9fefb'
+    filename: 'openshift-client-linux-aarch64.tar.gz'