From 6b2924726e9f25f41a17eb461bf58dcff302abc0 Mon Sep 17 00:00:00 2001 From: fsgh42 Date: Thu, 17 Oct 2024 23:24:04 +0200 Subject: [PATCH 1/2] feat: add imagefactory docker compose example --- examples/imagefactory/.gitignore | 3 + examples/imagefactory/README.md | 99 +++++++++++++++++++ examples/imagefactory/docker-compose.yaml | 62 ++++++++++++ examples/imagefactory/env.example | 11 +++ .../imagefactory/scripts/sync-schematics.sh | 70 +++++++++++++ 5 files changed, 245 insertions(+) create mode 100644 examples/imagefactory/.gitignore create mode 100644 examples/imagefactory/README.md create mode 100644 examples/imagefactory/docker-compose.yaml create mode 100644 examples/imagefactory/env.example create mode 100644 examples/imagefactory/scripts/sync-schematics.sh diff --git a/examples/imagefactory/.gitignore b/examples/imagefactory/.gitignore new file mode 100644 index 0000000..0539144 --- /dev/null +++ b/examples/imagefactory/.gitignore @@ -0,0 +1,3 @@ +.env +keys/* +schematics/* diff --git a/examples/imagefactory/README.md b/examples/imagefactory/README.md new file mode 100644 index 0000000..4e1b500 --- /dev/null +++ b/examples/imagefactory/README.md @@ -0,0 +1,99 @@ +# Siderolabs self-hosted imagefactory example + +This code runs [sidero imagefactory](https://github.com/siderolabs/image-factory) in [docker compose](https://docs.docker.com/compose/). + +It also deploys a few companion components: +* upstream `ghcr.io` [registry](https://distribution.github.io/distribution/) [mirror](https://distribution.github.io/distribution/recipes/mirror/) to avoid potential upstream rate limitings and speed up builds by caching previously pulled image layers +* a script that applies prepared `talos` [image schematics](https://www.talos.dev/v1.8/learn-more/image-factory/#schematics) +* a [registry](https://distribution.github.io/distribution/) used as storage and cache for the generated images + +# how to use + +tested with `talos` version `v1.8.1`, via [iPXE boot](https://www.talos.dev/v1.8/talos-guides/install/bare-metal-platforms/pxe/) and directly applying generated images to disk via `hcloud` (use it's [packer](https://developer.hashicorp.com/packer/integrations/hetznercloud/hcloud) integration and point [this](https://github.com/siderolabs/contrib/blob/9cd1e1c9d2469b77d2278eb07e7f61c09bb32d40/examples/terraform/hcloud/packer/hcloud_talosimage.pkr.hcl#L18) URL to your `imagefactory` instance). + +## preparation + +some preparation is required. + +### signing keys +see [official docs](https://github.com/siderolabs/image-factory?tab=readme-ov-file#development). + +```shell +mkdir -pv keys +openssl ecparam -name prime256v1 -genkey -noout -out keys/cache-signing-key.key +``` + +### schematics + +Refer to the [official docs](https://www.talos.dev/v1.8/learn-more/image-factory/#schematics) on how to create these. +The [script](./scripts/sync-schematics.sh) will find and apply all files in `schematics/*.yaml`. + +Example: +```yaml +# schematics/example.yaml +customization: + extraKernelArgs: + - gfxmode=1280x1024 + - console=ttyS0,115200 + - net.ifnames=0 + - talos.platform=metal + systemExtensions: + officialExtensions: + - siderolabs/amd-ucode + - siderolabs/fuse3 + - siderolabs/intel-ucode + - siderolabs/iscsi-tools + - siderolabs/qemu-guest-agent + - siderolabs/tailscale + - siderolabs/util-linux-tools + meta: + - key: 12 + value: | + machineLabels: + env: prod + type: controlplane +``` + +### environment variables + +copy the [docker compose env file](https://docs.docker.com/compose/how-tos/environment-variables/variable-interpolation/#env-file) and adjust the example values. + +```shell +cp env.example .env +vim .env +``` + +Adjust all domains and `EXT_IP` to where you want to expose your `imagefactory` instance. +This is relevant for payloads sent to `iPXE` clients and URLs generated in the UI. + +## run + +after preparation is done, run `docker compose up -d`. + +# miscellaneous & troubleshooting + +This is a community contribution so expect no official support. +Some trouble I ran into: + +## TLS + +The configuration used does *not* deploy TLS, so you should put this behind something like a reverse proxy that does. + +## connection timeouts + +Image generation can take some time, so clients might have to increase their connection timeout limits. If images are cached, [TTFB](https://en.wikipedia.org/wiki/Time_to_first_byte) is very short, if not `TTFB` can take up to several minutes. + +## iPXE and https + +If you want to [iPXE](https://ipxe.org)-boot from this via `https`, keep in mind that by default `iPXE` does *not* support `https` and you need to compile your own, enabling [this](https://ipxe.org/buildcfg/download_proto_https) flag. This is a pitfall for reverse proxies that automatically redirect plaintext `http` requests to `https`. + +## URLs not working + +There is a tiny problem in the `imagefactory` frontend: The URLs generated contain the external domain used and it is duplicated for some reason. This is particularly mean because the URL _visible_ in the UI looks correct, but the `HTML` `href` is not. +Make sure to sanitize the URL before use, the resulting URL works as expected. Not sure yet as to _why_ this happens (misconfiguration or might be a bug). +Querying the `API` seems to return the correct URL. + +## registry resource consumption + +* When building a large number of images, make sure to provide sufficient storage to the `registry` container and monitor `docker volumes` as it grows in size quite rapidly. +* the image generation process is compute heavy and can take some time, depending on the compute power available. diff --git a/examples/imagefactory/docker-compose.yaml b/examples/imagefactory/docker-compose.yaml new file mode 100644 index 0000000..a04cae5 --- /dev/null +++ b/examples/imagefactory/docker-compose.yaml @@ -0,0 +1,62 @@ +services: + # generated images are pushed to this registry + registry: + image: registry:2 + ports: + - ${EXTERNAL_IP:-127.0.0.1}:5000:5000 + volumes: + # hint: when generating a large number of different schemas, this volume can grow quite large + - registry:/var/lib/registry:rw + # upstream ghcr mirror, caches previously pulled image layers and prevents rate limiting + registry-ghcr: + image: registry:2 + environment: + REGISTRY_PROXY_REMOTEURL: http://ghcr.io + volumes: + - registry-ghcr:/var/lib/registry:rw + # triggers image builds for all schematics defined in `./schematics/*.yaml` + schematics: + image: alpine:3 + environment: + IMAGE_FACTORY_URL: http://imagefactory:6000 + REGISTRY_URL: registry:5000 + TALOS_VERSION: ${SCHEMATICS_TALOS_VERSION?} + ARCH: ${SCHEMATICS_TALOS_ARCH?} + VALIDATE: ${SCHEMATICS_VALIDATE?} + SLEEP_TIME: ${SCHEMATICS_SLEEP_TIME?} + command: > + /scripts/sync-schematics.sh + volumes: + - ${PWD}/scripts:/scripts:ro + - ${PWD}/schematics:/schematics:ro + # container running the actual imagefactory + imagefactory: + image: ghcr.io/siderolabs/image-factory:${IMGFAC_VERSION?} + # required for losetup + privileged: true + volumes: + - ${PWD}/keys:/keys:ro + # required for losetup + - /dev:/dev + ports: + - ${EXTERNAL_IP:-127.0.0.1}:6000:6000 + command: > + -http-port=:6000 + -external-url=${IMGFAC_EXTERNAL_URL?} + -external-pxe-url=http://${IMGFAC_EXTERNAL_URL?} + -cache-signing-key-path=/keys/cache-signing-key.key + -cache-repository=registry:5000/cache + -insecure-cache-repository=true + -image-registry=registry-ghcr:5000 + -insecure-image-registry=true + -schematic-service-repository=registry:5000/image-factory/schematic + -insecure-schematic-service-repository + -installer-internal-repository=registry:5000/siderolabs + -insecure-installer-internal-repository=true + -installer-external-repository=${IMGFAC_EXT_REPO?}/siderolabs + -secureboot=${IMGFAC_SECUREBOOT?} + +volumes: + registry: + registry-tls: + registry-ghcr: diff --git a/examples/imagefactory/env.example b/examples/imagefactory/env.example new file mode 100644 index 0000000..30d0de7 --- /dev/null +++ b/examples/imagefactory/env.example @@ -0,0 +1,11 @@ +EXTERNAL_IP=127.0.0.1 + +SCHEMATICS_TALOS_VERSION=1.8.1 +SCHEMATICS_TALOS_ARCH=amd64 +SCHEMATICS_VALIDATE=false +SCHEMATICS_SLEEP_TIME=600 + +IMGFAC_VERSION=v0.5.0 +IMGFAC_EXTERNAL_URL=imgfac.example.com +IMGFAC_EXT_REPO=reg.imgfac.example.com +IMGFAC_SECUREBOOT=false \ No newline at end of file diff --git a/examples/imagefactory/scripts/sync-schematics.sh b/examples/imagefactory/scripts/sync-schematics.sh new file mode 100644 index 0000000..0a9d072 --- /dev/null +++ b/examples/imagefactory/scripts/sync-schematics.sh @@ -0,0 +1,70 @@ +#!/bin/ash +set -eo pipefail + +trap "exit 0" SIGINT SIGTERM + +: ${IMAGE_FACTORY_URL:?} +: ${REGISTRY_URL:?} +: ${TALOS_VERSION:?} +: ${ARCH:?} +: ${VALIDATE:?} +: ${SLEEP_TIME:?} + +apk add crane yq + +RESULTS_FILE="${RESULTS_FILE:-/tmp/results}" +while true; do + echo '' > "${RESULTS_FILE}" + for SCHEMATIC in /schematics/*.yaml ; do + # this triggers image generation based on the schema provided + # docs: https://github.com/siderolabs/image-factory?tab=readme-ov-file#post-schematics + echo "apply ${SCHEMATIC}" + RESPONSE_FILE=/tmp/wget-response.json + wget \ + --header 'Content-Type: application/yaml' \ + -O "${RESPONSE_FILE}" \ + --post-file=${SCHEMATIC} \ + ${IMAGE_FACTORY_URL}/schematics \ + + # parse the image ID from the response + SCHEMA_ID=$(yq .id < "${RESPONSE_FILE}") + if test -z "${SCHEMA_ID}" ; then + echo 'SCHEMA_ID was empty' + exit 1 + fi + TMP_FILE="/tmp/${SCHEMA_ID}.tar" + rm "${RESPONSE_FILE}" + + # docs: https://github.com/siderolabs/image-factory?tab=readme-ov-file#get-imageschematicversionpath + echo 'download container' + wget \ + -O ${TMP_FILE} \ + ${IMAGE_FACTORY_URL}/image/${SCHEMA_ID}/${TALOS_VERSION}/installer-${ARCH}.tar + + # optional: this calls `crane validate `, validating the generated image is well formed + # docs: https://github.com/google/go-containerregistry/blob/main/cmd/crane/doc/crane_validate.md + if [ "${VALIDATE}" == 'true' ] ; then + echo 'validate container' + crane validate --tarball ${TMP_FILE} + fi + + echo 'publish container' + crane push \ + --insecure \ + ${TMP_FILE} \ + ${REGISTRY_URL}/installer/${SCHEMA_ID}:${TALOS_VERSION} + + rm -v ${TMP_FILE} + echo "${SCHEMATIC} ${SCHEMA_ID}" >> "${RESULTS_FILE}" + done + + # this prints the image IDs resulting from each schema, + # which can then be handed out to clients. + echo "--- results ---" + cat "${RESULTS_FILE}" + echo "---------------" + + echo "all done, sleep ${SLEEP_TIME} sec." + sleep ${SLEEP_TIME} & + wait $! +done From 33556f42db047f3745f8615128f2f2a37ade0c4f Mon Sep 17 00:00:00 2001 From: fsgh42 Date: Tue, 22 Oct 2024 22:55:43 +0200 Subject: [PATCH 2/2] add protocol to --external-url parameter --- examples/imagefactory/docker-compose.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/imagefactory/docker-compose.yaml b/examples/imagefactory/docker-compose.yaml index a04cae5..2237d45 100644 --- a/examples/imagefactory/docker-compose.yaml +++ b/examples/imagefactory/docker-compose.yaml @@ -42,7 +42,7 @@ services: - ${EXTERNAL_IP:-127.0.0.1}:6000:6000 command: > -http-port=:6000 - -external-url=${IMGFAC_EXTERNAL_URL?} + -external-url=http://${IMGFAC_EXTERNAL_URL?} -external-pxe-url=http://${IMGFAC_EXTERNAL_URL?} -cache-signing-key-path=/keys/cache-signing-key.key -cache-repository=registry:5000/cache