From 03a5bf2cce826f233234f6bdd5db56205e4a0696 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Tue, 26 Mar 2024 16:42:07 +0100 Subject: [PATCH 1/5] Makefile: implement "fully source containers" HMS-3883 also changes compose file to be more robust on startup use docker as default for now podman does not seem to run stable. will need to investigate more in docs: document makefile overview also fix execution on MAC --- .gitignore | 3 + Makefile | 205 ++++++++++++++++ README.md | 5 +- docs/src_compile_onprem.dot | 101 ++++++++ docs/src_compile_onprem.svg | 166 +++++++++++++ docs/src_compile_service.dot | 132 +++++++++++ docs/src_compile_service.svg | 221 ++++++++++++++++++ on-prem/bin/run.py | 1 - on-prem/bin/setup-host.py | 22 +- on-prem/src/ogsc/run/composer/entrypoint.py | 69 ------ service/README.md | 106 +++++---- service/config/Dockerfile-config | 24 +- service/config/Dockerfile-config-onprem | 21 ++ service/config/backend/quotas.json | 2 +- service/config/cli/data/blueprint.toml | 9 + .../composer/osbuild-composer-onprem.toml | 7 + service/docker-compose-onprem.yml | 155 ++++++++++++ service/docker-compose.yml | 169 ++++++++++---- service/tools/gen-certs.sh | 9 + tools/git_stack.sh | 62 +++++ 20 files changed, 1300 insertions(+), 189 deletions(-) create mode 100644 Makefile create mode 100644 docs/src_compile_onprem.dot create mode 100644 docs/src_compile_onprem.svg create mode 100644 docs/src_compile_service.dot create mode 100644 docs/src_compile_service.svg create mode 100644 service/config/Dockerfile-config-onprem create mode 100644 service/config/cli/data/blueprint.toml create mode 100644 service/config/composer/osbuild-composer-onprem.toml create mode 100644 service/docker-compose-onprem.yml create mode 100755 tools/git_stack.sh diff --git a/.gitignore b/.gitignore index 68c759d..f53f66f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ on-prem/build/config/*.pem on-prem/build/config/*.toml on-prem/build/config/ca on-prem/build/config/repositories +service_images_built.info +onprem_images_built.info + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..b9182cd --- /dev/null +++ b/Makefile @@ -0,0 +1,205 @@ +.PHONY: help +help: + @echo "make [TARGETS...]" + @echo + @echo "This is the makefile of osbuild-getting-started. The following" + @echo "targets are available:" + @echo + @echo " service_containers: Build all needed containers from source to be able to run the service" + @echo " run_service: Run all containers needed for the 'service'" + @echo " This is running in foreground. Use CTRL-C to stop the containers." + @echo " run_service_no_frontend: Run all containers except for the frontend" + @echo " stop_service: Usually the containers get stopped by CTRL-C. So 'stop-service'" + @echo " is only needed if something strange happened and not all are stopped." + @echo " prune_service: Remove the containers, including the test-data!" + @echo " If you want empty databases" + @echo " clean: Clean all subprojects to assure a rebuild" + +# source where the other repos are locally +# has to end with a trailing slash +SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= ../ + +# either "docker" or "sudo podman" +# podman needs to build as root as it also needs to run as root afterwards +CONTAINER_EXECUTABLE ?= docker +CONTAINER_COMPOSE_EXECUTABLE ?= $(CONTAINER_EXECUTABLE) compose + +MAKE_SUB_CALL := make CONTAINER_EXECUTABLE="$(CONTAINER_EXECUTABLE)" + +# osbuild is indirectly used by osbuild-composer +# but we'll mention it here too for better error messages and usability +COMMON_SRC_DEPS_NAMES := osbuild osbuild-composer pulp-client community-gateway +COMMON_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(COMMON_SRC_DEPS_NAMES)) + +ONPREM_SRC_DEPS_NAMES := weldr-client +ONPREM_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(ONPREM_SRC_DEPS_NAMES)) + +SERVICE_SRC_DEPS_NAMES := image-builder image-builder-frontend +SERVICE_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(SERVICE_SRC_DEPS_NAMES)) + +# should be set if we are already sudo - otherwise we set to "whoami" +SUDO_USER ?= $(shell whoami) + +$(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN): + @for DIR in $@; do if ! [ -d $$DIR ]; then echo "Please checkout $$DIR so it is available at $$DIR"; exit 1; fi; done + +COMPARE_TO_BRANCH ?= origin/main + +SCRATCH_DIR := $(HOME)/.cache/osbuild-getting-started/scratch +export SCRATCH_DIR +COMMON_DIR := image-builder-config +CLI_DIRS := weldr cloudapi dnf-json +DATA_DIR := data/s3/service +ALL_SCRATCH_DIRS := $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIR) $(CLI_DIRS) $(DATA_DIR)) + +# internal rule for sub-calls +# NOTE: This chowns all directories back - as we expect to run partly as root +# also we "git fetch origin" to get the current state! +.PHONY: common_sub_makes +common_sub_makes: + @echo "We need to build everything as root, as the target also needs to run as root." + @echo "At least for podman the password as already needed now" + + # creating container image from osbuild as a basis for worker + $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)osbuild-composer container_worker.dev container_composer.dev + +.PHONY: service_sub_make_backend +service_sub_make_backend: + $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder container.dev + +.PHONY: service_sub_make_frontend +service_sub_make_frontend: + $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder-frontend container.dev + +.PHONY: service_sub_make_cleanup +service_sub_make_cleanup: + @for DIR in $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN); do echo "Giving directory permissions in '$$DIR' back to '$(SUDO_USER)'"; chown -R $(SUDO_USER): $$DIR || sudo chown -R $(SUDO_USER): $$DIR; done + @echo "Your current versions are (comparing to origin/main):" + bash -c './tools/git_stack.sh' + +.PHONY: service_sub_makes_no_frontend +service_sub_makes_no_frontend: service_sub_make_backend service_sub_make_cleanup + +.PHONY: service_sub_makes +service_sub_makes: service_sub_make_backend service_sub_make_frontend service_sub_make_cleanup + +.PHONY: onprem_sub_makes +onprem_sub_makes: + # building the cli + $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)weldr-client container.dev + @for DIR in $(COMMON_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN); do echo "Giving directory permissions in '$$DIR' back to '$(SUDO_USER)'"; chown -R $(SUDO_USER): $$DIR || sudo chown -R $(SUDO_USER): $$DIR; done + @echo "Your current versions are (comparing to origin/main):" + bash -c './tools/git_stack.sh' + +.PHONY: service_containers +service_containers: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) common_sub_makes service_sub_makes service_images_built.info + +.PHONY: service_containers_no_frontend +service_containers_no_frontend: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) common_sub_makes service_sub_makes_no_frontend service_images_built.info + +onprem_containers: $(COMMON_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN) common_sub_makes onprem_sub_makes onprem_images_built.info + +service_images_built.info: service/config/Dockerfile-config service/config/composer/osbuild-composer.toml $(ALL_SCRATCH_DIRS) + # building remaining containers (config, fauxauth) + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml build --build-arg CONFIG_BUILD_DATE=$(shell date -r $(SCRATCH_DIR)/$(COMMON_DIR) +%Y%m%d_%H%M%S) + echo "Images last built on" > $@ + date >> $@ + +onprem_images_built.info: service/config/Dockerfile-config-onprem service/config/composer/osbuild-composer-onprem.toml $(ALL_SCRATCH_DIRS) + # building remaining containers (config) + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml build --build-arg CONFIG_BUILD_DATE=$(shell date -r $(SCRATCH_DIR)/$(COMMON_DIR) +%Y%m%d_%H%M%S) + echo "Images last built on" > $@ + date >> $@ + +$(ALL_SCRATCH_DIRS): + @echo "Creating directory: $@" + mkdir -p $@ || ( echo "Trying as root" ; sudo mkdir -p $@ ) + +.PHONY: wipe_config +wipe_config: + sudo rm -rf $(SCRATCH_DIR)/$(COMMON_DIR) + rm -f $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder-frontend/node_modules/.cache/webpack-dev-server/server.pem + +.PHONY: clean +clean: prune_service prune_onprem wipe_config + rm -f service_images_built.info + rm -f onprem_images_built.info + rm -rf $(SCRATCH_DIR) || (echo "Trying as root" ;sudo rm -rf $(SCRATCH_DIR)) + for DIR in $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN); do $(MAKE_SUB_CALL) -C $$DIR clean; done + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml down --volumes + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml rm --volumes + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml down --volumes + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml rm --volumes + +# for compatibility of relative volume mount paths +# between docker and podman we have to change to the directory +.PHONY: run_service +.ONESHELL: +SHELL := /bin/bash +.SHELLFLAGS := -ec -o pipefail + +run_service: $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIRS)) service_containers + export PORTS="$(shell $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml config|grep -Po '(?<=published: ")[0-9]+')" + echo "-- Checking if any of our ports are used: $$PORTS" + sudo netstat -lntp|grep -E "$$(echo "$$PORTS"|tr ' ' ':')" + echo "-- Check done" + cd service + $(CONTAINER_COMPOSE_EXECUTABLE) up + +# if you want to run the frontend yourself - outside the docker environment +.PHONY: run_service_no_frontend +run_service_no_frontend: service_containers_no_frontend + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml up backend fauxauth worker composer minio postgres_backend postgres_composer + +# only for strange crashes - should shut down properly in normal operation +.PHONY: stop_service +.ONESHELL: +stop_service: + cd service + $(CONTAINER_COMPOSE_EXECUTABLE) stop + +.PHONY: prune_service +.ONESHELL: +prune_service: + cd service + $(CONTAINER_COMPOSE_EXECUTABLE) down + +.PHONY: prune_onprem +.ONESHELL: +prune_onprem: + cd service + $(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml down + +# for compatibility of relative volume mount paths +# between docker and podman we have to change to the directory +.PHONY: run_onprem +.ONESHELL: +run_onprem: $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIRS) $(CLI_DIRS)) onprem_containers + cd service + echo "Remove dangling sockets as root" + sudo rm -f $(addsuffix /api.sock*, $(addprefix $(SCRATCH_DIR)/, $(CLI_DIRS))) + $(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml up -d + @echo "------ Welcome to osbuild! You can now use 'composer-cli' to build images" + @echo " … and 'exit' afterwards" + @echo " You might also be interested to open up a second terminal and" + @echo " run 'docker compose -f $(shell readlink -f service/docker-compose-onprem.yml) logs --follow' to see possible problems" + + $(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml run --entrypoint /bin/bash -ti cli + $(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml stop + +%.svg: %.dot + dot -T svg $< > $@ + +.PHONY: docs +docs: docs/src_compile_service.svg docs/src_compile_onprem.svg + +.PHONY: overview +overview: + @echo "Fetching all repos for better overview if rebase will be necessary:" +ifeq (${SUDO_USER},$(shell whoami)) + bash -c './tools/git_stack.sh fetch --all' +else + sudo -u ${SUDO_USER} bash -c './tools/git_stack.sh fetch --all' +endif + @bash -c './tools/git_stack.sh' + diff --git a/README.md b/README.md index d4aeb11..e2ccb3d 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,9 @@ provide more in-depth explainations of these products, their architecture and ho ## Getting started +You can develop and install the individual components on your local computer, or you can run the +whole stack as containers fully from source. + For further instructions on how to run containerized versions of the CLI and service locally: -- the [on-premise README](./on-prem/README.md) offers explainations on how to run the on-premise CLI tool. +- the [on-premise README](./on-prem/README.md) offers explanations on how to run the on-premise CLI tool. - the [service README](./service/README.md) provides instructions on how to get started with a stack of the hosted service. diff --git a/docs/src_compile_onprem.dot b/docs/src_compile_onprem.dot new file mode 100644 index 0000000..e45f843 --- /dev/null +++ b/docs/src_compile_onprem.dot @@ -0,0 +1,101 @@ +digraph src_compile { + newrank=true; + subgraph cluster_osbuild_getting_started { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild-\ngetting-started"; + osbuild_getting_started_makefile [ + label = "make\nonprem_containers" + width = 2; + ]; + osbuild_getting_started_run [ + label = "make\nrun_onprem" + width = 2; + ]; + } + + subgraph cluster_osbuild { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild\n "; + osbuild_makefile [ + label = "make\ncontainer"; + width = 1.5; + ]; + } + + subgraph cluster_osbuild_composer { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild-composer\n "; + osbuild_composer_makefile_composer [ + label = "make\ncontainer_composer" + width = 2; + ]; + osbuild_composer_makefile_worker [ + label = "make\ncontainer_worker" + width = 2; + ]; + + osbuild_composer_go [ label = "go code"; ]; + } + + subgraph cluster_weldr_client { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "weldr-client"; + weldr_client_makefile [ + label = "make\ncontainer" + width = 1.5; + ]; + } + subgraph cluster_pulp_client { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "pulp-client"; + pulp_client_go [ + label = "go code"; + width = 1.5; + ]; + } + subgraph cluster_images { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "images"; + images_go [label = "go code";]; + } + { rank=same; + osbuild_makefile; + osbuild_composer_makefile_worker; + osbuild_composer_makefile_composer; + weldr_client_makefile; + + } + { rank=same; + edge [style=dashed; headport=s; tailport=s]; + pulp_client_go; + images_go; + } + osbuild_composer_go -> pulp_client_go [style=dashed;]; + osbuild_composer_go -> images_go [style=dashed;]; + + osbuild_getting_started_run -> osbuild_getting_started_makefile; + + osbuild_getting_started_makefile -> osbuild_makefile; + osbuild_getting_started_makefile -> osbuild_composer_makefile_worker; + osbuild_getting_started_makefile -> osbuild_composer_makefile_composer; + + + osbuild_getting_started_makefile -> weldr_client_makefile; + osbuild_composer_makefile_worker -> osbuild_composer_go [style=dashed]; + osbuild_composer_makefile_composer -> osbuild_composer_go [style=dashed]; + + osbuild_composer_makefile_worker -> osbuild_makefile [style=dashed; headport=s; tailport=s]; + +} diff --git a/docs/src_compile_onprem.svg b/docs/src_compile_onprem.svg new file mode 100644 index 0000000..7585d67 --- /dev/null +++ b/docs/src_compile_onprem.svg @@ -0,0 +1,166 @@ + + + + + + +src_compile + + +cluster_osbuild_getting_started + +osbuild- +getting-started + + +cluster_osbuild + +osbuild +  + + +cluster_osbuild_composer + +osbuild-composer +  + + +cluster_weldr_client + +weldr-client + + +cluster_pulp_client + +pulp-client + + +cluster_images + +images + + + +osbuild_getting_started_makefile + +make +onprem_containers + + + +osbuild_makefile + +make +container + + + +osbuild_getting_started_makefile->osbuild_makefile + + + + + +osbuild_composer_makefile_composer + +make +container_composer + + + +osbuild_getting_started_makefile->osbuild_composer_makefile_composer + + + + + +osbuild_composer_makefile_worker + +make +container_worker + + + +osbuild_getting_started_makefile->osbuild_composer_makefile_worker + + + + + +weldr_client_makefile + +make +container + + + +osbuild_getting_started_makefile->weldr_client_makefile + + + + + +osbuild_getting_started_run + +make +run_onprem + + + +osbuild_getting_started_run->osbuild_getting_started_makefile + + + + + +osbuild_composer_go + +go code + + + +osbuild_composer_makefile_composer->osbuild_composer_go + + + + + +osbuild_composer_makefile_worker:s->osbuild_makefile:s + + + + + +osbuild_composer_makefile_worker->osbuild_composer_go + + + + + +pulp_client_go + +go code + + + +osbuild_composer_go->pulp_client_go + + + + + +images_go + +go code + + + +osbuild_composer_go->images_go + + + + + diff --git a/docs/src_compile_service.dot b/docs/src_compile_service.dot new file mode 100644 index 0000000..e139f55 --- /dev/null +++ b/docs/src_compile_service.dot @@ -0,0 +1,132 @@ +digraph src_compile { + newrank=true; + subgraph cluster_osbuild_getting_started { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild-\ngetting-started"; + osbuild_getting_started_makefile [ + label = "make\nservice_containers" + width = 2; + ]; + osbuild_getting_started_run [ + label = "make\nrun_service" + width = 2; + ]; + } + + subgraph cluster_osbuild { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild\n "; + osbuild_makefile [ + label = "make\ncontainer"; + width = 1.5; + ]; + } + + subgraph cluster_osbuild_composer { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild-composer\n "; + osbuild_composer_makefile_composer [ + label = "make\ncontainer_composer" + width = 2; + ]; + osbuild_composer_makefile_worker [ + label = "make\ncontainer_worker" + width = 2; + ]; + + osbuild_composer_go [ label = "go code"; ]; + } + + subgraph cluster_image_builder { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "image-builder\n "; + image_builder_makefile [ + label = "make\ncontainer" + width = 1.5; + ]; + image_builder_go [ label = "go code"; ]; + } + + subgraph cluster_image_builder_frontend { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "image-builder-\nfronted"; + image_builder_frontend_makefile [ + label = "make\ncontainer" + width = 1.5; + ]; + } + subgraph cluster_pulp_client { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "pulp-client"; + pulp_client_go [ + label = "go code"; + width = 1.5; + ]; + } + subgraph cluster_images { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "images"; + images_go [label = "go code";]; + } + subgraph cluster_community_gateway { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "community-gateway"; + community_gateway_go [ + label = "go code"; + width = 2; + ]; + } + + { rank=same; + osbuild_makefile; + osbuild_composer_makefile_composer; + osbuild_composer_makefile_worker; + image_builder_makefile; + image_builder_frontend_makefile; + } + { rank=same; + edge [style=dashed]; + image_builder_go -> osbuild_composer_go; + } + { rank=same; + edge [style=dashed; headport=s; tailport=s]; + pulp_client_go; + images_go; + community_gateway_go; + } + osbuild_composer_go -> pulp_client_go [style=dashed;]; + osbuild_composer_go -> images_go [style=dashed;]; + image_builder_go -> community_gateway_go [style=dashed;]; + + osbuild_getting_started_run -> osbuild_getting_started_makefile; + + osbuild_getting_started_makefile -> osbuild_makefile; + osbuild_getting_started_makefile -> osbuild_composer_makefile_worker; + osbuild_getting_started_makefile -> osbuild_composer_makefile_composer; + osbuild_getting_started_makefile -> image_builder_makefile; + osbuild_getting_started_makefile -> image_builder_frontend_makefile; + osbuild_composer_makefile_worker -> osbuild_composer_go [style=dashed]; + osbuild_composer_makefile_composer -> osbuild_composer_go [style=dashed]; + image_builder_makefile -> image_builder_go [style=dashed]; + + + + osbuild_composer_makefile_worker -> osbuild_makefile [style=dashed; headport=s; tailport=s]; + +} diff --git a/docs/src_compile_service.svg b/docs/src_compile_service.svg new file mode 100644 index 0000000..f3dcf5f --- /dev/null +++ b/docs/src_compile_service.svg @@ -0,0 +1,221 @@ + + + + + + +src_compile + + +cluster_osbuild_getting_started + +osbuild- +getting-started + + +cluster_osbuild + +osbuild +  + + +cluster_osbuild_composer + +osbuild-composer +  + + +cluster_image_builder + +image-builder +  + + +cluster_image_builder_frontend + +image-builder- +fronted + + +cluster_pulp_client + +pulp-client + + +cluster_images + +images + + +cluster_community_gateway + +community-gateway + + + +osbuild_getting_started_makefile + +make +service_containers + + + +osbuild_makefile + +make +container + + + +osbuild_getting_started_makefile->osbuild_makefile + + + + + +osbuild_composer_makefile_composer + +make +container_composer + + + +osbuild_getting_started_makefile->osbuild_composer_makefile_composer + + + + + +osbuild_composer_makefile_worker + +make +container_worker + + + +osbuild_getting_started_makefile->osbuild_composer_makefile_worker + + + + + +image_builder_makefile + +make +container + + + +osbuild_getting_started_makefile->image_builder_makefile + + + + + +image_builder_frontend_makefile + +make +container + + + +osbuild_getting_started_makefile->image_builder_frontend_makefile + + + + + +osbuild_getting_started_run + +make +run_service + + + +osbuild_getting_started_run->osbuild_getting_started_makefile + + + + + +osbuild_composer_go + +go code + + + +osbuild_composer_makefile_composer->osbuild_composer_go + + + + + +osbuild_composer_makefile_worker:s->osbuild_makefile:s + + + + + +osbuild_composer_makefile_worker->osbuild_composer_go + + + + + +pulp_client_go + +go code + + + +osbuild_composer_go->pulp_client_go + + + + + +images_go + +go code + + + +osbuild_composer_go->images_go + + + + + +image_builder_go + +go code + + + +image_builder_makefile->image_builder_go + + + + + +image_builder_go->osbuild_composer_go + + + + + +community_gateway_go + +go code + + + +image_builder_go->community_gateway_go + + + + + diff --git a/on-prem/bin/run.py b/on-prem/bin/run.py index 948e234..289f295 100755 --- a/on-prem/bin/run.py +++ b/on-prem/bin/run.py @@ -93,7 +93,6 @@ async def env( "--network", "podman", "--name", f"{prefix}-composer", f"ogsc/run/composer:{osbuild_composer_version}", - "--dnf-json", "--weldr-api", "--remote-worker-api", "--composer-api", diff --git a/on-prem/bin/setup-host.py b/on-prem/bin/setup-host.py index 97bd577..5ac0c40 100755 --- a/on-prem/bin/setup-host.py +++ b/on-prem/bin/setup-host.py @@ -4,18 +4,15 @@ # of setting up all required dependencies on the host to run osbuild services in # containers. -import sys -import subprocess import logging - +import subprocess +import sys log = logging.getLogger(__name__) # The base set of packages necessary. -package_base = { - "git" -} +package_base = {"git"} package_set = { "container": { @@ -25,11 +22,14 @@ def package_install(packages): - subprocess.run([ - "dnf", - "in", - "-y", - ] + list(packages)) + subprocess.run( + [ + "dnf", + "in", + "-y", + ] + + list(packages) + ) return 0 diff --git a/on-prem/src/ogsc/run/composer/entrypoint.py b/on-prem/src/ogsc/run/composer/entrypoint.py index d545d32..e38baf1 100644 --- a/on-prem/src/ogsc/run/composer/entrypoint.py +++ b/on-prem/src/ogsc/run/composer/entrypoint.py @@ -118,34 +118,12 @@ def _parse_args(self): help="Disable the weldr-API", ) - # --[no-]dnf-json - self._parser.add_argument( - "--dnf-json", - action="store_true", - dest="dnf_json", - help="Enable dnf-json", - ) - self._parser.add_argument( - "--no-dnf-json", - action="store_false", - dest="dnf_json", - help="Disable dnf-json", - ) - self._parser.add_argument( - "--dnf-json-port", - type=int, - default=0, - dest="dnf_json_port", - help="Specify the port dnf-json should listen on", - ) - self._parser.set_defaults( builtin_worker=False, composer_api=False, local_worker_api=False, remote_worker_api=False, weldr_api=False, - dnf_json=False, ) return self._parser.parse_args(self._argv[1:]) @@ -295,51 +273,17 @@ def _spawn_composer(sockets): preexec_fn=preexec_setenv, ) - def _spawn_dnf_json(self): - cmd = [ - "/usr/libexec/osbuild-depsolve-dnf", - ] - - if self.args.dnf_json_port: - sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - self._exitstack.enter_context(contextlib.closing(sock)) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) - sock.bind(("::", self.args.dnf_json_port)) - sock.listen() - else: - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self._exitstack.enter_context(contextlib.closing(sock)) - os.makedirs("/run/osbuild-dnf-json/", exist_ok=True) - sock.bind("/run/osbuild-dnf-json/api.sock") - sock.listen() - - dnfenv = os.environ.copy() - dnfenv["LISTEN_FDS"] = "1" - dnfenv["LISTEN_FD"] = str(sock.fileno()) - - return subprocess.Popen( - cmd, - cwd="/usr/libexec/osbuild-composer", - stdin=subprocess.DEVNULL, - stderr=subprocess.STDOUT, - env=dnfenv, - pass_fds=[sock.fileno()] - ) - def run(self): """Program Runtime""" proc_composer = None proc_worker = None - proc_dnf_json = None res = 0 sockets = self._prepare_sockets() def handler(signum, frame): proc_composer.terminate() proc_worker.terminate() - proc_dnf_json.terminate() signal.signal(signal.SIGTERM, handler) @@ -351,9 +295,6 @@ def handler(signum, frame): if self.args.builtin_worker: proc_worker = self._spawn_worker() - if self.args.dnf_json: - proc_dnf_json = self._spawn_dnf_json() - if any([self.args.weldr_api, self.args.composer_api, self.args.local_worker_api, self.args.remote_worker_api]): proc_composer = self._spawn_composer(sockets) @@ -365,11 +306,6 @@ def handler(signum, frame): proc_worker.terminate() proc_worker.wait() - if proc_dnf_json: - if proc_composer: - proc_dnf_json.terminate() - proc_dnf_json.wait() - except KeyboardInterrupt: if proc_composer: proc_composer.terminate() @@ -377,14 +313,9 @@ def handler(signum, frame): if proc_worker: proc_worker.terminate() proc_worker.wait() - if proc_dnf_json: - proc_dnf_json.terminate() - proc_dnf_json.wait() except: if proc_worker: proc_worker.kill() - if proc_dnf_json: - proc_dnf_json.kill() if proc_composer: proc_composer.kill() raise diff --git a/service/README.md b/service/README.md index 94dc66e..8f495e8 100644 --- a/service/README.md +++ b/service/README.md @@ -1,24 +1,45 @@ # devtools Development Tools for Image Builder +Here are some tools of "Image Builder" to start up a development environment for the +[hosted deployment](https://osbuild.org/docs/hosted/architecture/). +The reason for all the `sudo` below is that the stack has to perform may operations +with loop devices etc. where root privileges are required. ## Setup - -To start local development, first clone the image builder stack: + +To start local development, first clone the image builder stack and all it's dependent source. ```bash +git clone git@github.com:osbuild/images.git git clone git@github.com:osbuild/osbuild-composer.git -git clone git@github.com:osbuild/image-builder.git git clone git@github.com:osbuild/osbuild-getting-started.git +git clone git@github.com:osbuild/osbuild.git +git clone git@github.com:osbuild/pulp-client.git +``` + +For starting the service you need additionally: +```shell +git clone git@github.com:osbuild/community-gateway.git +git clone git@github.com:osbuild/image-builder-frontend.git +git clone git@github.com:osbuild/image-builder.git +``` + +For starting the on-prem version you need additionally: +```shell +git clone git@github.com:osbuild/weldr-client.git ``` -The folder structure should look like: +The folder structure should look like this: ``` . -├── image-builder +├── images +├── osbuild +├── osbuild-composer ├── osbuild-getting-started -└── osbuild-composer +… +└── pulp-client ``` Secondly redirect a stage and prod domains to localhost. If you are outside @@ -29,24 +50,6 @@ echo "127.0.0.1 prod.foo.redhat.com" >> /etc/hosts echo "127.0.0.1 stage.foo.redhat.com" >> /etc/hosts ``` -Lastly, you will need to ensure that the folders for the config volume and the local S3 bucket -have been created: - -> This can be modified in the volumes -> section of the `docker-compose.yaml` -> file - -```bash -sudo mkdir -p /scratch/podman/image-builder-config -``` - -> The name of the bucket used is `service`, -> you can change this in the `.env` file - -```bash -sudo mkdir -p /scratch/data/s3/service -``` - ## Docker compose notes As per the [docker compose cli](https://docs.docker.com/compose/reference/) docs, the new syntax for running docker compose changed from @@ -55,7 +58,7 @@ command. ## Upload Targets -Upload targets need to be configued for the Image Builder backend to upload successfully. +Upload targets need to be configured for the Image Builder backend to upload successfully. This stack comes pre-configured with a generic S3 bucket which can be accessed at: `http://localhost:9000` @@ -95,49 +98,60 @@ The config variables for the worker can be found [here](https://github.com/osbui *NOTE:* If you change the config files, you will either need to modify the worker config in the `/scratch/podman/image-builder-config` file and restart the containers. Alternatively, you will need to remove the named volume and rebuild the config container. The steps for this are as follows: -Run the following from **this directory**. +Run the following from the main **osbuild-getting-started directory** (one level above where this README.md is). + +```bash +make help +``` + - stop the containers and remove volumes ```bash -docker compose down -v +make prune-service ``` -- rebuild the config container +- rebuild containers ```bash -docker compose build config +make service-containers ``` - start the containers again ```bash -docker compose up +make run-service ``` -## Run the backend +## Run the frontend separately -To build the containers run the following command from **this directory**: +The command above (`make run-service`) also starts the frontend by bind-mounting the source +from your computer, so hot-reload of `npm` should work. +If you want to run the frontend yourself (see the [README.md](https://github.com/osbuild/image-builder-frontend/blob/main/README.md) there) +you can use the make target: ```bash -docker compose build +make run-service-no-fronted ``` -To run the containers: +You have to have a "staging account" in order to run this setup (although it's local). +Check if you can log in at https://console.stage.redhat.com/ -```bash -docker compose up -``` +Access the service through the GUI: +[https://stage.foo.redhat.com:1337/beta/insights/image-builder](https://stage.foo.redhat.com:1337/beta/insights/image-builder), or +directly through the API: +[https://stage.foo.redhat.com:1337/docs/api/image-builder](https://stage.foo.redhat.com:1337/docs/api/image-builder). -## Run the frontend +## Debugging +When you want to use your IDE (Integrated Development Environment) to debug the go-code you can check out the +`GODEBUG_PORT` variables in [docker-composer.yml](./docker-composer.yml) and enable debugging for the supported +layers. -The frontend has been removed as a container in favour of running it separately in order to leverage hot-reloading -capabilities. In order to run the frontend with the backend you can run the following command: +## Known Problems -```bash -npm run devel +If you encounter problems with the certificate when you browse to +[https://stage.foo.redhat.com:1337/preview/insights/image-builder](https://stage.foo.redhat.com:1337/preview/insights/image-builder) +you might need to rebuild the certificates with: +``` +make wipe-config ``` -Access the service through the GUI: -[https://stage.foo.redhat.com:1337/beta/insights/image-builder](https://stage.foo.redhat.com:1337/beta/insights/image-builder), or -directly through the API: -[https://stage.foo.redhat.com:1337/docs/api/image-builder](https://stage.foo.redhat.com:1337/docs/api/image-builder). diff --git a/service/config/Dockerfile-config b/service/config/Dockerfile-config index d876df4..fbd2664 100644 --- a/service/config/Dockerfile-config +++ b/service/config/Dockerfile-config @@ -1,15 +1,21 @@ FROM fedora:38 RUN dnf install -y openssl -RUN mkdir -p /config +RUN mkdir -p /config_build COPY ./tools/gen-certs.sh . COPY ./config/x509/openssl.cnf . -COPY ./config/composer/acl.yml /config/. -COPY ./config/composer/osbuild-composer.toml /config/. -COPY ./config/worker/osbuild-worker.toml /config/. -COPY ./config/worker/s3-credentials /config/. -COPY ./config/worker/secret /config/. -COPY ./config/backend/quotas.json /config/. - -RUN ./gen-certs.sh ./openssl.cnf /config /config/ca +COPY ./config/composer/acl.yml /config_build/. +COPY ./config/composer/osbuild-composer.toml /config_build/. +COPY ./config/worker/osbuild-worker.toml /config_build/. +COPY ./config/worker/s3-credentials /config_build/. +COPY ./config/worker/secret /config_build/. +COPY ./config/backend/quotas.json /config_build + +# Helper to re-run gen-certs.sh +ARG CONFIG_BUILD_DATE=NOT_SET +ENV CONFIG_BUILD_DATE=$CONFIG_BUILD_DATE + +RUN ./gen-certs.sh ./openssl.cnf /config_build /config_build/ca + +CMD cp -r /config_build/* /config/ && echo "Done. Provided the config to all other containers." diff --git a/service/config/Dockerfile-config-onprem b/service/config/Dockerfile-config-onprem new file mode 100644 index 0000000..f351d05 --- /dev/null +++ b/service/config/Dockerfile-config-onprem @@ -0,0 +1,21 @@ +FROM fedora:38 + +RUN dnf install -y openssl +RUN mkdir -p /config_build + +COPY ./tools/gen-certs.sh . +COPY ./config/x509/openssl.cnf . +COPY ./config/composer/acl.yml /config_build/. +COPY ./config/composer/osbuild-composer-onprem.toml /config_build/osbuild-composer.toml +# COPY ./config/worker/osbuild-worker-onprem.toml /config_build/osbuild-worker.toml +COPY ./config/worker/s3-credentials /config_build/. +COPY ./config/worker/secret /config_build/. +COPY ./config/backend/quotas.json /config_build + +# Helper to re-run gen-certs.sh +ARG CONFIG_BUILD_DATE=NOT_SET +ENV CONFIG_BUILD_DATE=$CONFIG_BUILD_DATE + +RUN ./gen-certs.sh ./openssl.cnf /config_build /config_build/ca + +CMD cp -r /config_build/* /config/ && cp -r /repositories /config/ && echo "Done. Provided the config to all other containers." diff --git a/service/config/backend/quotas.json b/service/config/backend/quotas.json index c3a2a04..d0b5af9 100644 --- a/service/config/backend/quotas.json +++ b/service/config/backend/quotas.json @@ -3,4 +3,4 @@ "quota":200, "slidingWindow":1209600000000000 } -} + } diff --git a/service/config/cli/data/blueprint.toml b/service/config/cli/data/blueprint.toml new file mode 100644 index 0000000..9971ac3 --- /dev/null +++ b/service/config/cli/data/blueprint.toml @@ -0,0 +1,9 @@ +name = "example-image" +description = "An example image for osbuild-getting-started." +version = "0.0.1" +modules = [] +groups = [] + +[[packages]] +name = "tmux" +version = "*" diff --git a/service/config/composer/osbuild-composer-onprem.toml b/service/config/composer/osbuild-composer-onprem.toml new file mode 100644 index 0000000..6cfff0a --- /dev/null +++ b/service/config/composer/osbuild-composer-onprem.toml @@ -0,0 +1,7 @@ +[koji] +allowed_domains = [ "localhost", "client.osbuild.org" ] +ca = "/etc/osbuild-composer/ca-crt.pem" + +[worker] +allowed_domains = [ "localhost", "worker.osbuild.org", "172.31.0.1" ] +ca = "/etc/osbuild-composer/ca-crt.pem" diff --git a/service/docker-compose-onprem.yml b/service/docker-compose-onprem.yml new file mode 100644 index 0000000..8aadd5e --- /dev/null +++ b/service/docker-compose-onprem.yml @@ -0,0 +1,155 @@ +networks: + onpremnet: + ipam: + driver: default + config: + - subnet: 172.32.0.0/16 + +volumes: + config_volume: + driver_opts: + type: none + # this can be modified as desired + device: /scratch/image-builder-config + o: bind + weldr_volume: + driver_opts: + type: none + # this can be modified as desired + device: /scratch/weldr + o: bind + cloudapi_volume: + driver_opts: + type: none + # this can be modified as desired + device: /scratch/cloudapi + o: bind + dnfjson_volume: + driver_opts: + type: none + # this can be modified as desired + device: /scratch/dnf-json + o: bind + +services: + config: + image: local/config + build: + context: . + dockerfile: ./config/Dockerfile-config-onprem + volumes: + - config_volume:/config:z + - ../../osbuild-composer/repositories:/repositories:z + + composer: + image: osbuild-composer_dev + entrypoint: + [ + "python3", + "/opt/entrypoint.py", + "--weldr-api", + "--remote-worker-api", + "--composer-api", + "--composer-api-port", "8000", + ] + healthcheck: + test: ["CMD-SHELL", "curl -k -s --cert /etc/osbuild-composer/worker-crt.pem --key /etc/osbuild-composer/worker-key.pem -o /dev/null https://localhost:8700/api/worker/v1/status"] + interval: 2s + timeout: 2s + retries: 100 + environment: + - PGHOST=postgres_composer + - PGPORT=5432 + - PGDATABASE=postgres + - PGUSER=postgres + - PGPASSWORD=postgres + - PGSSLMODE=disable + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7730 + volumes: + - config_volume:/etc/osbuild-composer + - weldr_volume:/run/weldr:rw,z + - cloudapi_volume:/run/cloudapi:rw,z + - dnfjson_volume:/run/osbuild-dnf-json:rw,z + ports: + - 8080:8080 + - 8700:8700 + - 7730:7730 + depends_on: + config: + condition: service_completed_successfully + postgres_composer: + condition: service_healthy + + networks: + onpremnet: + ipv4_address: 172.32.0.10 + + worker: + image: osbuild-worker_dev + volumes: + - config_volume:/etc/osbuild-composer:z + - config_volume:/etc/osbuild-worker:z + environment: + - CACHE_DIRECTORY=/var/cache/osbuild-composer + - WORKER_ARGS=composer:8700 + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7740 + # only enable the OSBUILD_PYTHON_DEBUG_PORT if you want live debugging. + # you HAVE to attach a pydev debugger to continue execution (check the logs after starting a build) + # it will connect to your localhost:5679 and expects a python debug server there + # - OSBUILD_PYTHON_DEBUG_PORT=5679 + ports: + - 7740:7740 + privileged: true + cap_add: + - CAP_MKNOD + - SYS_ADMIN + - NET_ADMIN + depends_on: + config: + condition: service_completed_successfully + composer: + condition: service_healthy + restart: on-failure + networks: + onpremnet: + ipv4_address: 172.32.0.20 + extra_hosts: + - "host.docker.internal:host-gateway" + + + postgres_composer: + image: docker.io/postgres:10.5 + healthcheck: + test: [ "CMD", "pg_isready", "-U", "postgres", "-d", "postgres" ] + interval: 2s + timeout: 2s + retries: 10 + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + volumes: + - ../../osbuild-composer/pkg/jobqueue/dbjobqueue/schemas/:/docker-entrypoint-initdb.d/:z + networks: + onpremnet: + ipv4_address: 172.32.0.30 + + cli: + image: osbuild-cli_dev + volumes: + - config_volume:/etc/osbuild-composer:z + - weldr_volume:/run/weldr:rw,z + - dnfjson_volume:/run/osbuild-dnf-json:rw,z + - ./config/cli/data:/data + #environment: + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7750 + networks: + onpremnet: + ipv4_address: 172.32.0.40 + entrypoint: + [ "/usr/bin/true" ] # will be started interactively from Makefile diff --git a/service/docker-compose.yml b/service/docker-compose.yml index 01af030..adb5b53 100644 --- a/service/docker-compose.yml +++ b/service/docker-compose.yml @@ -1,18 +1,16 @@ -version: '3.9' - networks: - net: + servicenet: ipam: driver: default config: - subnet: 172.31.0.0/16 volumes: - config: + config_volume: driver_opts: type: none # this can be modified as desired - device: /scratch/podman/image-builder-config + device: ${SCRATCH_DIR:-/scratch}/image-builder-config o: bind services: @@ -22,14 +20,27 @@ services: context: . dockerfile: ./config/Dockerfile-config volumes: - - config:/config + - config_volume:/config:z - ../../osbuild-composer/repositories:/config/repositories:z + postgres_composer: + image: docker.io/postgres:10.5 + healthcheck: + test: [ "CMD", "pg_isready", "-h", "localhost", "-U", "postgres", "-d", "postgres" ] + interval: 2s + timeout: 2s + retries: 10 + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + volumes: + - ../../osbuild-composer/pkg/jobqueue/dbjobqueue/schemas/:/docker-entrypoint-initdb.d/:z + networks: + servicenet: + ipv4_address: 172.31.0.80 + composer: - image: local/osbuild-composer - build: - context: ../../osbuild-composer - dockerfile: ./distribution/Dockerfile-ubi + image: osbuild-composer_dev entrypoint: [ "python3", @@ -37,46 +48,77 @@ services: "--remote-worker-api", "--composer-api" ] + healthcheck: + test: ["CMD-SHELL", "if curl -k -s -o /dev/null -w '%{http_code}' https://localhost:8700/ | grep -q 401; then exit 0; else exit 1; fi"] + interval: 2s + timeout: 2s + retries: 100 + environment: + - PGHOST=postgres_composer + - PGPORT=5432 + - PGDATABASE=postgres + - PGUSER=postgres + - PGPASSWORD=postgres + - PGSSLMODE=disable + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7730 volumes: - - config:/etc/osbuild-composer + - config_volume:/etc/osbuild-composer ports: - 8080:8080 - 8700:8700 + - 7730:7730 depends_on: - - config + config: + condition: service_completed_successfully + postgres_composer: + condition: service_healthy + networks: - net: + servicenet: ipv4_address: 172.31.0.10 worker: - image: local/osbuild-worker - build: - context: ../../osbuild-composer - dockerfile: ./distribution/Dockerfile-worker - # override the entrypoint to specify composer hostname and port - entrypoint: [ "/usr/libexec/osbuild-composer/osbuild-worker", "composer:8700" ] + image: osbuild-worker_dev volumes: - - config:/etc/osbuild-composer - - config:/etc/osbuild-worker + - config_volume:/etc/osbuild-composer:z + - config_volume:/etc/osbuild-worker:z environment: - CACHE_DIRECTORY=/var/cache/osbuild-composer + - WORKER_ARGS=composer:8700 + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7740 + # only enable the OSBUILD_PYCHARM_DEBUG_PORT if you want live debugging. + # you HAVE to attach a pydev debugger to continue execution (check the logs after starting a build) + # it will connect to your localhost:5679 and expects a python debug server there + # - OSBUILD_PYCHARM_DEBUG_PORT=5679 + # or use OSBUILD_DEBUGPY_DEBUG_PORT for e.g. vscode + # - OSBUILD_DEBUGPY_DEBUG_PORT=5679 + ports: + - 7740:7740 privileged: true cap_add: - CAP_MKNOD - - SYS_ADMIN - NET_ADMIN + - SYS_ADMIN depends_on: - - config - - composer + config: + condition: service_completed_successfully + composer: + condition: service_healthy restart: on-failure networks: - net: + servicenet: ipv4_address: 172.31.0.20 + extra_hosts: + - "host.docker.internal:host-gateway" - postgres: + postgres_backend: image: docker.io/postgres:10.5 healthcheck: - test: [ "CMD", "pg_isready", "-U", "postgres", "-d", "postgres" ] + test: [ "CMD", "pg_isready", "-h", "localhost", "-U", "postgres", "-d", "postgres" ] interval: 2s timeout: 2s retries: 10 @@ -84,30 +126,28 @@ services: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=postgres volumes: - - ../../image-builder/internal/db/migrations-tern/:/docker-entrypoint-initdb.d/:Z + - ../../image-builder/internal/db/migrations-tern/:/docker-entrypoint-initdb.d/:z networks: - net: + servicenet: ipv4_address: 172.31.0.30 backend: - image: local/image-builder - build: - context: ../../image-builder - dockerfile: ./distribution/Dockerfile-ubi + image: image-builder_dev ports: - 8086:8086 + - 7720:7720 healthcheck: test: [ "CMD", "curl", "localhost:8086/status" ] interval: 2s timeout: 2s - retries: 10 + retries: 100 volumes: - - config:/etc/image-builder + - config_volume:/etc/image-builder environment: - LISTEN_ADDRESS=backend:8086 - LOG_LEVEL=DEBUG - ALLOWED_ORG_IDS=* - - PGHOST=postgres + - PGHOST=postgres_backend - PGPORT=5432 - PGDATABASE=postgres - PGUSER=postgres @@ -119,48 +159,75 @@ services: - COMPOSER_CA_PATH=/etc/image-builder/ca-crt.pem - DISTRIBUTIONS_DIR=/app/distributions - QUOTA_FILE=/etc/image-builder/quotas.json + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7720 depends_on: - - config - - composer - - postgres + config: + condition: service_completed_successfully + composer: + condition: service_healthy + postgres_backend: + condition: service_healthy restart: on-failure networks: - net: + servicenet: ipv4_address: 172.31.0.40 fauxauth: - image: local/osbuild-fauxauth + image: local/osbuild-fauxauth_dev build: context: ../../osbuild-composer - dockerfile: ./distribution/Dockerfile-fauxauth + dockerfile: ./distribution/Dockerfile-fauxauth.dev + args: + GOMODARGS: "-modfile=go.local.mod" entrypoint: [ "/opt/fauxauth.py", "-a", "0.0.0.0", "-p", "8888" ] volumes: - - config:/etc/osbuild-composer + - config_volume:/etc/osbuild-composer ports: - 8888:8888 depends_on: - config networks: - net: + servicenet: ipv4_address: 172.31.0.50 minio: - image: minio/minio:latest + image: docker.io/minio/minio:latest volumes: # this can be modified as desired - - /scratch/data:/data:z + - ${SCRATCH_DIR:-/scratch}/data/s3:/data:z command: server /data --console-address ":9090" + working_dir: /data environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password42 + - MINIO_ROOT_USER=admin + - MINIO_ROOT_PASSWORD=password42 ports: - 9000:9000 - - 9090:9090 + - 9091:9090 healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] interval: 30s timeout: 20s retries: 3 networks: - net: + servicenet: ipv4_address: 172.31.0.60 + + frontend: + image: image-builder-frontend_dev + security_opt: + - seccomp=unconfined + environment: + - BACKEND_HOSTNAME=backend + ports: + - 1337:1337 + - 8002:8002 + volumes: + - ../../image-builder-frontend/api:/app/api:z + - ../../image-builder-frontend/config:/app/config:z + - ../../image-builder-frontend/src:/app/src:z + networks: + servicenet: + ipv4_address: 172.31.0.70 + diff --git a/service/tools/gen-certs.sh b/service/tools/gen-certs.sh index c603279..9b7a95a 100755 --- a/service/tools/gen-certs.sh +++ b/service/tools/gen-certs.sh @@ -1,4 +1,13 @@ #!/bin/bash + +if [ -f /config ] ; then + echo "ERROR: /config should not be a file. Your setup seems to be broken!" + echo "Please check your volume mounts of the container runtime (podman/docker)" + exit 1 +fi + +ls -la /config/ + if (( $# != 3 )); then echo "Usage: $0 " echo diff --git a/tools/git_stack.sh b/tools/git_stack.sh new file mode 100755 index 0000000..189f8dc --- /dev/null +++ b/tools/git_stack.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +# repos to respect +BASEDIR=$(dirname $(readlink -f $0))/../.. +REPOS="osbuild-getting-started osbuild osbuild-composer images image-builder image-builder-frontend weldr-client" +REPOS="$REPOS pulp-client community-gateway" + +ARGS=${*:-status --branch --short} + +NUM="$(echo $REPOS | wc -w)" +I=0 +running_jobs=0 +MAX_PARALLEL=4 +tmp_files=() + +RED='\033[0;31m' +NO_COLOR='\033[0m' + +function run_git_command { + local dir=$1 + local args=$2 + local tmp_file=$3 + git -c color.status=always -C "$dir" $args >> "$tmp_file" 2>&1 || echo -e "${RED}ERROR code: $?${NO_COLOR}" >> "$tmp_file" & +} + +for D in $REPOS; do + I=$(( $I + 1 )) + FULL_DIR="$BASEDIR/$D" + if [ "$ARGS" == "walk" ]; then + echo " ------------- ( $I / $NUM ) $D -------------" + pushd $FULL_DIR + git status + bash + popd + else + tmp_file=$(mktemp) + tmp_files+=("$tmp_file") + echo " ------------- ( $I / $NUM ) $D -------------" > "$tmp_file" + echo " -- Started -- ( $I / $NUM ) $D -------------" + run_git_command "$FULL_DIR" "$ARGS" "$tmp_file" + + (( running_jobs++ )) + if (( running_jobs >= MAX_PARALLEL )); then + wait -n + (( running_jobs-- )) + fi + fi +done + +if [ "$ARGS" != "walk" ]; then + wait +fi + +echo " ------------- DONE -------------" + +# Print the buffered output +for tmp_file in "${tmp_files[@]}"; do + echo "" + cat "$tmp_file" + rm "$tmp_file" +done + From 083f156ae8e4d2dee15e75cd795d5ad0927e7437 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Wed, 12 Feb 2025 11:30:25 +0100 Subject: [PATCH 2/5] service/docker-compose-onprem: make SCRATCH_DIR dynamic --- Makefile | 3 +-- service/docker-compose-onprem.yml | 8 ++++---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index b9182cd..041d009 100644 --- a/Makefile +++ b/Makefile @@ -143,8 +143,7 @@ run_service: $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIRS)) service_containers echo "-- Checking if any of our ports are used: $$PORTS" sudo netstat -lntp|grep -E "$$(echo "$$PORTS"|tr ' ' ':')" echo "-- Check done" - cd service - $(CONTAINER_COMPOSE_EXECUTABLE) up + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml up # if you want to run the frontend yourself - outside the docker environment .PHONY: run_service_no_frontend diff --git a/service/docker-compose-onprem.yml b/service/docker-compose-onprem.yml index 8aadd5e..410a859 100644 --- a/service/docker-compose-onprem.yml +++ b/service/docker-compose-onprem.yml @@ -10,25 +10,25 @@ volumes: driver_opts: type: none # this can be modified as desired - device: /scratch/image-builder-config + device: ${SCRATCH_DIR:-/scratch}/image-builder-config o: bind weldr_volume: driver_opts: type: none # this can be modified as desired - device: /scratch/weldr + device: ${SCRATCH_DIR:-/scratch}/weldr o: bind cloudapi_volume: driver_opts: type: none # this can be modified as desired - device: /scratch/cloudapi + device: ${SCRATCH_DIR:-/scratch}/cloudapi o: bind dnfjson_volume: driver_opts: type: none # this can be modified as desired - device: /scratch/dnf-json + device: ${SCRATCH_DIR:-/scratch}/dnf-json o: bind services: From c8df7a47a4e0092beac3c57066fc3084036fdda6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Wed, 14 May 2025 11:19:52 +0200 Subject: [PATCH 3/5] backup: rework to monorepo --- Makefile | 27 +- .../osbuild-composer/Makefile.getting-started | 114 +++++ .../tools/Dockerfile-composer.dev | 46 ++ .../tools/Dockerfile-worker.dev | 38 ++ .../tools/osbuild-worker-entrypoint.sh | 13 + .../tools/osbuild_composer-entrypoint.py | 411 ++++++++++++++++++ repos/osbuild/Makefile.getting-started | 72 +++ repos/osbuild/tools/Containerfile.dev | 36 ++ repos/osbuild/tools/Containerfile.dev.patch | 13 + repos/osbuild/tools/main_cli_dev.py | 28 ++ 10 files changed, 792 insertions(+), 6 deletions(-) create mode 100644 repos/osbuild-composer/Makefile.getting-started create mode 100644 repos/osbuild-composer/tools/Dockerfile-composer.dev create mode 100644 repos/osbuild-composer/tools/Dockerfile-worker.dev create mode 100755 repos/osbuild-composer/tools/osbuild-worker-entrypoint.sh create mode 100644 repos/osbuild-composer/tools/osbuild_composer-entrypoint.py create mode 100644 repos/osbuild/Makefile.getting-started create mode 100644 repos/osbuild/tools/Containerfile.dev create mode 100644 repos/osbuild/tools/Containerfile.dev.patch create mode 100644 repos/osbuild/tools/main_cli_dev.py diff --git a/Makefile b/Makefile index 041d009..9bbd8b4 100644 --- a/Makefile +++ b/Makefile @@ -15,14 +15,17 @@ help: @echo " If you want empty databases" @echo " clean: Clean all subprojects to assure a rebuild" +MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +GETTING_STARTED_DIR := $(patsubst %/,%,$(dir $(MAKEFILE_PATH))) + # source where the other repos are locally # has to end with a trailing slash -SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= ../ +export SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= $(GETTING_STARTED_DIR)/.. # either "docker" or "sudo podman" # podman needs to build as root as it also needs to run as root afterwards -CONTAINER_EXECUTABLE ?= docker -CONTAINER_COMPOSE_EXECUTABLE ?= $(CONTAINER_EXECUTABLE) compose +export CONTAINER_EXECUTABLE ?= docker +export CONTAINER_COMPOSE_EXECUTABLE ?= $(CONTAINER_EXECUTABLE) compose MAKE_SUB_CALL := make CONTAINER_EXECUTABLE="$(CONTAINER_EXECUTABLE)" @@ -52,6 +55,18 @@ CLI_DIRS := weldr cloudapi dnf-json DATA_DIR := data/s3/service ALL_SCRATCH_DIRS := $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIR) $(CLI_DIRS) $(DATA_DIR)) +OSBUILD_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/osbuild +OSBUILD_COMPOSER_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/osbuild-composer + +.PHONY: service_containers +service_containers: + make -C $(OSBUILD_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild/Makefile.getting-started container.dev + make -C $(OSBUILD_COMPOSER_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild-composer/Makefile.getting-started container.dev + +clean: + make -C $(OSBUILD_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild/Makefile.getting-started clean.dev + make -C $(OSBUILD_COMPOSER_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild-composer/Makefile.getting-started clean.dev + # internal rule for sub-calls # NOTE: This chowns all directories back - as we expect to run partly as root # also we "git fetch origin" to get the current state! @@ -91,8 +106,8 @@ onprem_sub_makes: @echo "Your current versions are (comparing to origin/main):" bash -c './tools/git_stack.sh' -.PHONY: service_containers -service_containers: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) common_sub_makes service_sub_makes service_images_built.info +.PHONY: service_containers_old +service_containers_old: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) common_sub_makes service_sub_makes service_images_built.info .PHONY: service_containers_no_frontend service_containers_no_frontend: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) common_sub_makes service_sub_makes_no_frontend service_images_built.info @@ -121,7 +136,7 @@ wipe_config: rm -f $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder-frontend/node_modules/.cache/webpack-dev-server/server.pem .PHONY: clean -clean: prune_service prune_onprem wipe_config +clean_old: prune_service prune_onprem wipe_config rm -f service_images_built.info rm -f onprem_images_built.info rm -rf $(SCRATCH_DIR) || (echo "Trying as root" ;sudo rm -rf $(SCRATCH_DIR)) diff --git a/repos/osbuild-composer/Makefile.getting-started b/repos/osbuild-composer/Makefile.getting-started new file mode 100644 index 0000000..7d56397 --- /dev/null +++ b/repos/osbuild-composer/Makefile.getting-started @@ -0,0 +1,114 @@ +MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +GETTING_STARTED_DIR := $(patsubst %/,%,$(dir $(MAKEFILE_PATH)../../)) +SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= $(GETTING_STARTED_DIR)/../ + +OSBUILD_COMPOSER_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/osbuild-composer + +.ONESHELL: +SHELL := /bin/bash +.SHELLFLAGS := -ec -o pipefail + +include $(OSBUILD_COMPOSER_DIR)/Makefile + +export CONTAINER_EXECUTABLE ?= podman + +CONTAINER_IMAGE_WORKER ?= osbuild-worker_dev +CONTAINERFILE_WORKER := $(GETTING_STARTED_DIR)/repos/osbuild-composer/tools/Dockerfile-worker.dev + +CONTAINER_IMAGE_COMPOSER ?= osbuild-composer_dev +CONTAINERFILE_COMPOSER := $(GETTING_STARTED_DIR)/repos/osbuild-composer/tools/Dockerfile-composer.dev + +GOPROXY ?= https://proxy.golang.org,direct + +# names of folder that have to be git-cloned additionally to be able +# to build all code +SRC_DEPS_EXTERNAL_NAMES := images pulp-client +SRC_DEPS_EXTERNAL_DIRS := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(SRC_DEPS_EXTERNAL_NAMES)) + +$(SRC_DEPS_EXTERNAL_DIRS): + @for DIR in $@; do if ! [ -d $$DIR ]; then echo "Please checkout $$DIR so it is available at $$DIR"; exit 1; fi; done + + +SRC_DEPS_DIRS := internal cmd pkg repositories + +# All files to check for rebuild! +SRC_DEPS := $(shell find $(addprefix $(OSBUILD_COMPOSER_DIR)/,$(SRC_DEPS_DIRS)) -name *.go -or -name *.json) +SRC_DEPS_EXTERNAL := $(shell find $(SRC_DEPS_EXTERNAL_DIRS) -name *.go) + +# dependencies to rebuild worker +WORKER_SRC_DEPS := $(SRC_DEPS) +# dependencies to rebuild composer +COMPOSER_SRC_DEPS := $(SRC_DEPS) + +GOMODARGS ?= -modfile=go.local.mod +# gcflags "-N -l" for golang to allow debugging +GCFLAGS ?= -gcflags=all=-N -gcflags=all=-l + +CONTAINER_DEPS_COMPOSER := $(GETTING_STARTED_DIR)/repos/osbuild-composer/tools/osbuild-composer-entrypoint.py +CONTAINER_DEPS_WORKER := $(GETTING_STARTED_DIR)/repos/osbuild-composer/tools/osbuild-worker-entrypoint.sh + +USE_BTRFS ?= yes + +COMMON_SRC_DEPS_NAMES := osbuild +COMMON_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(COMMON_SRC_DEPS_NAMES)) + +OSBUILD_CONTAINER_INDICATOR := $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/osbuild/container_built.info + +MAKE_SUB_CALL := make CONTAINER_EXECUTABLE="$(CONTAINER_EXECUTABLE)" + +$(COMMON_SRC_DEPS_ORIGIN): + @for DIR in $@; do if ! [ -d $$DIR ]; then echo "Please checkout $$DIR so it is available at $$DIR"; exit 1; fi; done + +# we'll trigger the sub-make for osbuild with "osbuild-container" +# and use OSBUILD_CONTAINER_INDICATOR to check if we need to rebuild our containers here +$(OSBUILD_CONTAINER_INDICATOR): + $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/osbuild container.dev + +go.local.mod go.local.sum: $(SRC_DEPS_EXTERNAL_DIRS) go.mod $(SRC_DEPS_EXTERNAL) $(WORKER_SRC_DEPS) $(COMPOSER_SRC_DEPS) Makefile + cp go.mod go.local.mod + cp go.sum go.local.sum + + go mod edit $(GOMODARGS) -replace github.com/osbuild/images=$(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)images + go mod edit $(GOMODARGS) -replace github.com/osbuild/pulp-client=$(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)pulp-client + go mod edit $(GOMODARGS) -replace github.com/osbuild/osbuild-composer/pkg/splunk_logger=./pkg/splunk_logger + env GOPROXY=$(GOPROXY) go mod tidy $(GOMODARGS) + env GOPROXY=$(GOPROXY) go mod vendor $(GOMODARGS) + +container_worker_built.info: go.local.mod $(WORKER_SRC_DEPS) $(CONTAINER_WORKER) $(CONTAINER_DEPS_WORKER) $(OSBUILD_CONTAINER_INDICATOR) + $(CONTAINER_EXECUTABLE) build \ + -t $(CONTAINER_IMAGE_WORKER) \ + -f $(CONTAINERFILE_WORKER) \ + --build-arg GOMODARGS="$(GOMODARGS)" \ + --build-arg GCFLAGS="$(GCFLAGS)" \ + --build-arg USE_BTRFS=$(USE_BTRFS) \ + $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR) + echo "Worker last built on" > $@ + date >> $@ + +container_composer_built.info: go.local.mod $(COMPOSER_SRC_DEPS) $(CONTAINERFILE_COMPOSER) $(CONTAINER_DEPS_COMPOSER) $(OSBUILD_CONTAINER_INDICATOR) + $(CONTAINER_EXECUTABLE) build \ + -t $(CONTAINER_IMAGE_COMPOSER) \ + -f $(CONTAINERFILE_COMPOSER) \ + --build-arg GOMODARGS="$(GOMODARGS)" \ + --build-arg GCFLAGS="$(GCFLAGS)" \ + $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR) + echo "Composer last built on" > $@ + date >> $@ + +# build a container with a worker from full source +.PHONY: container_worker.dev +container_worker.dev: $(OSBUILD_CONTAINER_INDICATOR) container_worker_built.info + +# build a container with the composer from full source +.PHONY: container_composer.dev +container_composer.dev: $(OSBUILD_CONTAINER_INDICATOR) container_composer_built.info + +container.dev: container_worker.dev container_composer.dev + +.PHONY: clean.dev +clean.dev: clean + rm -rf $(BUILDDIR)/build/ + rm -rf $(GOLANGCI_LINT_CACHE_DIR) + rm -f $(BUILDDIR)/go.local.* + rm -f $(BUILDDIR)/container_worker_built.info + rm -f $(BUILDDIR)/container_composer_built.info diff --git a/repos/osbuild-composer/tools/Dockerfile-composer.dev b/repos/osbuild-composer/tools/Dockerfile-composer.dev new file mode 100644 index 0000000..e7e2c00 --- /dev/null +++ b/repos/osbuild-composer/tools/Dockerfile-composer.dev @@ -0,0 +1,46 @@ +FROM registry.fedoraproject.org/fedora:latest AS builder +ENV GOBIN=/opt/app-root/src/go/bin + +ARG OSBUILD_COMPOSER_DIR=osbuild-composer +ENV OSBUILD_COMPOSER_DIR=${OSBUILD_COMPOSER_DIR} + +ARG GETTING_STARTED_DIR=osbuild-getting-started +ENV GETTING_STARTED_DIR=${GETTING_STARTED_DIR} + +RUN dnf install -y gpgme-devel libassuan-devel device-mapper-devel golang + +WORKDIR /osbuild-composer +COPY ${OSBUILD_COMPOSER_DIR} /osbuild-composer +ENV GOFLAGS="-mod=vendor -tags=exclude_graphdriver_btrfs" + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" +ARG GCFLAGS="" + +RUN go install $GOMODARGS $GCFLAGS ./cmd/osbuild-composer/ + +FROM registry.access.redhat.com/ubi9/go-toolset:latest AS builder2 +RUN go install github.com/jackc/tern@latest + +FROM osbuild_dev + +RUN dnf install -y python3 python3-dnf procps-ng gpgme libassuan device-mapper-libs delve +RUN mkdir -p "/usr/libexec/osbuild-composer" +RUN mkdir -p "/etc/osbuild-composer/" +RUN mkdir -p "/run/osbuild-composer/" +RUN mkdir -p "/var/cache/osbuild-composer/" +RUN mkdir -p "/var/lib/osbuild-composer/" +RUN mkdir -p "/usr/share/osbuild-composer/" +RUN mkdir -p "/opt/migrate/" +COPY --from=builder /opt/app-root/src/go/bin/osbuild-composer /usr/libexec/osbuild-composer/ +COPY ${GETTING_STARTED_DIR}…/containers/osbuild-composer/entrypoint.py /opt/entrypoint.py + +COPY ./pkg/jobqueue/dbjobqueue/schemas /opt/migrate/schemas +COPY --from=builder2 /opt/app-root/src/go/bin/tern /opt/migrate/ + +COPY ./repositories /usr/share/osbuild-composer/repositories + +EXPOSE 8008 8080 8700 +ENTRYPOINT ["python3", "/opt/entrypoint.py", "--remote-worker-api", "--composer-api", "--prometheus", "--shutdown-wait-period", "15"] diff --git a/repos/osbuild-composer/tools/Dockerfile-worker.dev b/repos/osbuild-composer/tools/Dockerfile-worker.dev new file mode 100644 index 0000000..6c53fc5 --- /dev/null +++ b/repos/osbuild-composer/tools/Dockerfile-worker.dev @@ -0,0 +1,38 @@ +FROM registry.fedoraproject.org/fedora:latest AS builder +ENV GOBIN=/opt/app-root/src/go/bin +# extra packages are needed +# to compile osbuild +RUN dnf install -y golang \ + krb5-devel \ + gpgme-devel \ + libassuan-devel + +ARG USE_BTRFS=no +RUN if [[ "$USE_BTRFS" == "yes" ]]; then dnf install -y btrfs-progs-devel device-mapper-devel; fi + +WORKDIR /osbuild-composer +COPY . /osbuild-composer +ENV GOFLAGS=-mod=vendor + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" +ARG GCFLAGS="" + +RUN go install $GOMODARGS $GCFLAGS ./cmd/osbuild-worker + +FROM osbuild_dev + +RUN dnf install -y delve + +RUN mkdir -p "/usr/libexec/osbuild-composer" +RUN mkdir -p "/etc/osbuild-composer/" +RUN mkdir -p "/run/osbuild-composer/" +RUN mkdir -p "/var/cache/osbuild-worker/" +RUN mkdir -p "/var/lib/osbuild-composer/" +RUN mkdir -p "/var/cache/osbuild-composer/output" +COPY --from=builder /opt/app-root/src/go/bin/osbuild-worker /usr/libexec/osbuild-composer/ +COPY distribution/osbuild-worker-entrypoint.sh /usr/libexec/osbuild-composer/ + +ENTRYPOINT ["/usr/libexec/osbuild-composer/osbuild-worker-entrypoint.sh"] diff --git a/repos/osbuild-composer/tools/osbuild-worker-entrypoint.sh b/repos/osbuild-composer/tools/osbuild-worker-entrypoint.sh new file mode 100755 index 0000000..b3f1a15 --- /dev/null +++ b/repos/osbuild-composer/tools/osbuild-worker-entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +APP="/usr/libexec/osbuild-composer/osbuild-worker" +APP_ARGS="${WORKER_ARGS:-}" + +if [[ -n "${GODEBUG_PORT:-}" ]]; then + echo "With golang debugger enabled on port ${GODEBUG_PORT} ..." + echo "NOTE: you HAVE to attach the debugger NOW otherwise the osbuild-worker will not continue running" + /usr/bin/dlv "--listen=:${GODEBUG_PORT}" --headless=true --api-version=2 exec ${APP} -- "${APP_ARGS}" + exit $? +fi + +${APP} "${APP_ARGS}" diff --git a/repos/osbuild-composer/tools/osbuild_composer-entrypoint.py b/repos/osbuild-composer/tools/osbuild_composer-entrypoint.py new file mode 100644 index 0000000..4404b8f --- /dev/null +++ b/repos/osbuild-composer/tools/osbuild_composer-entrypoint.py @@ -0,0 +1,411 @@ +"""entrypoint - Containerized OSBuild Composer + +This provides the entrypoint for a containerized osbuild-composer image. It +spawns `osbuild-composer` on start and manages it until it exits. The main +purpose of this entrypoint is to prepare everything to be usable from within +a container. +""" + +import argparse +import contextlib +import os +import pathlib +import signal +import socket +import subprocess +import sys +import time + + +class Cli(contextlib.AbstractContextManager): + """Command Line Interface""" + + def __init__(self, argv): + self.args = None + self._argv = argv + self._exitstack = None + self._parser = None + + def _parse_args(self): + self._parser = argparse.ArgumentParser( + add_help=True, + allow_abbrev=False, + argument_default=None, + description="Containerized OSBuild Composer", + prog="container/osbuild-composer", + ) + + self._parser.add_argument( + "--shutdown-wait-period", + type=int, + default=0, + dest="shutdown_wait_period", + help="Wait period in seconds before terminating child processes", + ) + + # --[no-]composer-api + self._parser.add_argument( + "--composer-api", + action="store_true", + dest="composer_api", + help="Enable the composer-API", + ) + self._parser.add_argument( + "--no-composer-api", + action="store_false", + dest="composer_api", + help="Disable the composer-API", + ) + self._parser.add_argument( + "--prometheus", + action="store_true", + dest="prometheus", + help="Enable prometheus listener", + ) + self._parser.add_argument( + "--no-prometheus", + action="store_false", + dest="prometheus", + help="Disable prometheus listener", + ) + self._parser.add_argument( + "--composer-api-port", + type=int, + default=8080, + dest="composer_api_port", + help="Port which the composer-API listens on", + ) + self._parser.add_argument( + "--prometheus-port", + type=int, + default=8008, + dest="prometheus_port", + help="Port which prometheus listens on", + ) + self._parser.add_argument( + "--composer-api-bind-address", + type=str, + default="::", + dest="composer_api_bind_address", + help="Bind the composer API to the specified address", + ) + self._parser.add_argument( + "--prometheus-bind-address", + type=str, + default="::", + dest="prometheus_bind_address", + help="Bind the prometheus listener to the specified address", + ) + + # --[no-]local-worker-api + self._parser.add_argument( + "--local-worker-api", + action="store_true", + dest="local_worker_api", + help="Enable the local-worker-API", + ) + self._parser.add_argument( + "--no-local-worker-api", + action="store_false", + dest="local_worker_api", + help="Disable the local-worker-API", + ) + + # --[no-]remote-worker-api + self._parser.add_argument( + "--remote-worker-api", + action="store_true", + dest="remote_worker_api", + help="Enable the remote-worker-API", + ) + self._parser.add_argument( + "--no-remote-worker-api", + action="store_false", + dest="remote_worker_api", + help="Disable the remote-worker-API", + ) + self._parser.add_argument( + "--remote-worker-api-port", + type=int, + default=8700, + dest="remote_worker_api_port", + help="Port which the remote-worker API listens on", + ) + self._parser.add_argument( + "--remote-worker-api-bind-address", + type=str, + default="::", + dest="remote_worker_api_bind_address", + help="Bind the remote worker API to the specified address", + ) + + # --[no-]weldr-api + self._parser.add_argument( + "--weldr-api", + action="store_true", + dest="weldr_api", + help="Enable the weldr-API", + ) + self._parser.add_argument( + "--no-weldr-api", + action="store_false", + dest="weldr_api", + help="Disable the weldr-API", + ) + + self._parser.set_defaults( + builtin_worker=False, + composer_api=False, + prometheus=False, + local_worker_api=False, + remote_worker_api=False, + weldr_api=False, + ) + + return self._parser.parse_args(self._argv[1:]) + + def __enter__(self): + self._exitstack = contextlib.ExitStack() + self.args = self._parse_args() + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + self._exitstack.close() + self._exitstack = None + + def _prepare_sockets(self): + # Prepare all the API sockets that osbuild-composer expectes, and make + # sure to pass them according to the systemd socket-activation API. + # + # Note that we rely on this being called early, so we get the correct + # FD numbers assigned. We need FD-#3 onwards for compatibility with + # socket activation (because python `subprocess.Popen` does not support + # renumbering the sockets we pass down). + + index = 3 + sockets = [] + names = [] + + # osbuild-composer.socket + if self.args.weldr_api: + print("Create weldr-api socket", file=sys.stderr) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.bind("/run/weldr/api.socket") + sock.listen() + sockets.append(sock) + + names.append("osbuild-composer.socket") + + assert(sock.fileno() == index) + index += 1 + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.bind("/run/cloudapi/api.socket") + sock.listen() + sockets.append(sock) + + names.append("osbuild-composer.socket") + + assert(sock.fileno() == index) + index += 1 + + # osbuild-composer-api.socket + if self.args.composer_api: + print("Create composer-api socket on port {}".format(self.args.composer_api_port) , file=sys.stderr) + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind((self.args.composer_api_bind_address, self.args.composer_api_port)) + sock.listen() + sockets.append(sock) + names.append("osbuild-composer-api.socket") + + assert(sock.fileno() == index) + index += 1 + + # osbuild-composer-prometheus.socket + if self.args.prometheus: + print("Create prometheus socket on port {}".format(self.args.prometheus_port), file=sys.stderr) + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind((self.args.prometheus_bind_address, self.args.prometheus_port)) + sock.listen() + sockets.append(sock) + names.append("osbuild-composer-prometheus.socket") + + assert(sock.fileno() == index) + index += 1 + + # osbuild-local-worker.socket + if self.args.local_worker_api: + print("Create local-worker-api socket", file=sys.stderr) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.bind("/run/osbuild-composer/job.socket") + sock.listen() + sockets.append(sock) + names.append("osbuild-local-worker.socket") + + assert(sock.fileno() == index) + index += 1 + + # osbuild-remote-worker.socket + if self.args.remote_worker_api: + print(f"Create remote-worker-api socket on address [{self.args.remote_worker_api_bind_address}]:{self.args.remote_worker_api_port}", file=sys.stderr) + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind((self.args.remote_worker_api_bind_address, self.args.remote_worker_api_port)) + sock.listen(256) + sockets.append(sock) + names.append("osbuild-remote-worker.socket") + + assert(sock.fileno() == index) + index += 1 + + # Prepare FD environment for the child process. + os.environ["LISTEN_FDS"] = str(len(sockets)) + os.environ["LISTEN_FDNAMES"] = ":".join(names) + + return sockets + + @staticmethod + def _spawn_worker(): + cmd = [ + "/usr/libexec/osbuild-composer/osbuild-worker", + "-unix", + "/run/osbuild-composer/job.socket", + ] + + env = os.environ.copy() + env["CACHE_DIRECTORY"] = "/var/cache/osbuild-worker" + env["STATE_DIRECTORY"] = "/var/lib/osbuild-worker" + + return subprocess.Popen( + cmd, + cwd="/", + env=env, + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + ) + + @staticmethod + def _spawn_composer(sockets): + cmd = [ + "/usr/libexec/osbuild-composer/osbuild-composer", + "-verbose", + ] + + # Prepare the environment for osbuild-composer. Note that we cannot use + # the `env` parameter of `subprocess.Popen()`, because it conflicts + # with the `preexec_fn=` parameter. Therefore, we have to modify the + # caller's environment. + os.environ["CACHE_DIRECTORY"] = "/var/cache/osbuild-composer" + os.environ["STATE_DIRECTORY"] = "/var/lib/osbuild-composer" + + # We need to set `LISTEN_PID=` to the target PID. The only way python + # allows us to do this is to hook into `preexec_fn=`, which is executed + # by `subprocess.Popen()` after forking, but before executing the new + # executable. + preexec_setenv = lambda: os.putenv("LISTEN_PID", str(os.getpid())) + + return subprocess.Popen( + cmd, + cwd="/usr/libexec/osbuild-composer", + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + pass_fds=[sock.fileno() for sock in sockets], + preexec_fn=preexec_setenv, + ) + + def run(self): + """Program Runtime""" + + proc_composer = None + proc_worker = None + res = 0 + sockets = self._prepare_sockets() + + def handler(signum, frame): + if self.args.shutdown_wait_period: + time.sleep(self.args.shutdown_wait_period) + proc_composer.terminate() + proc_worker.terminate() + + signal.signal(signal.SIGTERM, handler) + + liveness = pathlib.Path('/tmp/osbuild-composer-live') + + liveness.touch() + + try: + should_launch_composer = any([self.args.weldr_api, self.args.composer_api, self.args.local_worker_api, self.args.remote_worker_api]) + if self.args.builtin_worker or not should_launch_composer: + if not should_launch_composer: + print(f"NOTE: launching worker only - no API for composer enabled") + proc_worker = self._spawn_worker() + + if should_launch_composer: + proc_composer = self._spawn_composer(sockets) + + debug_port = os.environ.get('GODEBUG_PORT') + debugger = None + + if debug_port: + # only debug one - either composer or worker if there is no composer + child_pid = proc_composer.pid if proc_composer else proc_worker.pid + debug_target_name = "image-builder-composer" if proc_composer else "image-builder-worker" + + debugger_cmd = [ + "/usr/bin/dlv", + "attach", + "--headless=true", + "--api-version", "2", + "--listen", f":{debug_port}", + str(child_pid), + "/usr/libexec/osbuild-composer/osbuild-composer" + ] + + print(f"NOTE: you HAVE to attach the debugger NOW otherwise { debug_target_name } " + f"will not continue running", file=sys.stderr) + debugger = subprocess.Popen(debugger_cmd) + + if proc_composer: + res = proc_composer.wait() + + if proc_worker: + if proc_composer: + proc_worker.terminate() + proc_worker.wait() + + if debugger: + debugger.wait() + + except KeyboardInterrupt: + if proc_composer: + proc_composer.terminate() + res = proc_composer.wait() + if proc_worker: + proc_worker.terminate() + proc_worker.wait() + except: + if proc_worker: + proc_worker.kill() + if proc_composer: + proc_composer.kill() + raise + finally: + liveness.unlink() + + return res + + +if __name__ == "__main__": + with Cli(sys.argv) as global_main: + sys.exit(global_main.run()) diff --git a/repos/osbuild/Makefile.getting-started b/repos/osbuild/Makefile.getting-started new file mode 100644 index 0000000..a48e0f3 --- /dev/null +++ b/repos/osbuild/Makefile.getting-started @@ -0,0 +1,72 @@ + +MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +GETTING_STARTED_DIR := $(dir $(MAKEFILE_PATH))../.. +SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= $(GETTING_STARTED_DIR)/.. + +OSBUILD_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/osbuild + +include $(OSBUILD_DIR)/Makefile + +# List of all files which trigger a rebuild of the +# development targets and container + +# Directories to search +SRC_DEPS_DIRS := assemblers devices inputs mounts osbuild runners sources stages tools + +# Function to search for Python files in a directory +define find_python_files +$(shell find $(1) -type f -exec file {} \; | grep "Python script" | grep -Eo "^[^:]+") +endef + +# All files to check for rebuild! +SRC_DEPS := $(shell find $(OSBUILD_DIR) -name "*.py") \ + $(foreach dir,$(SRC_DEPS_DIRS),$(call find_python_files,$(OSBUILD_DIR)/$(dir))) \ + $(shell find $(OSBUILD_DIR)/schemas -name "*.json") + +CONTAINER_EXECUTABLE ?= podman +CONTAINER_IMAGE := osbuild_dev +CONTAINERFILE := $(GETTING_STARTED_DIR)/repos/osbuild/tools/Containerfile.dev + +RPM_SPECFILE_DEV=$(OSBUILD_DIR)/rpmbuild/SPECS/osbuild-SRC.spec + +# building rpm for development with a constant name +# to avoid any semblance to a "release rpm" +$(RPM_SPECFILE_DEV): osbuild.spec + mkdir -p $(OSBUILD_DIR)/rpmbuild/SPECS + echo "%global commit SRC" > $@ + echo "%define _rpmfilename %%{NAME}.rpm" >> $@ + echo "%global source_date_epoch_from_changelog 1" >> $@ + echo "%global clamp_mtime_to_source_date_epoch 1" >> $@ + cat $< >> $@ + +RPM_DEV_FILES := osbuild.rpm \ + osbuild-depsolve-dnf.rpm \ + osbuild-luks2.rpm \ + osbuild-ostree.rpm \ + osbuild-selinux.rpm \ + osbuild-tools.rpm \ + python3-osbuild.rpm + +RPM_DEV := $(addprefix $(OSBUILD_DIR)/rpmbuild/RPMS/,$(RPM_DEV_FILES)) + +.PHONY: rpm.dev +rpm.dev: $(RPM_DEV) + +$(RPM_DEV): $(RPM_SPECFILE_DEV) $(SRC_DEPS) + export SOURCE_DATE_EPOCH=$(shell stat -c %Y $(RPM_SPECFILE_DEV)) + rpmbuild -bb --build-in-place \ + --define "_topdir $(OSBUILD_DIR)/rpmbuild" \ + $(RPM_SPECFILE_DEV) + +clean.dev: clean + rm -rf $(OSBUILD_DIR)/rpmbuild + rm -f $(OSBUILD_DIR)/container_built.info + + +container_built.info: $(CONTAINERFILE) osbuild.spec $(SRC_DEPS) + $(CONTAINER_EXECUTABLE) build -t $(CONTAINER_IMAGE) -f $(CONTAINERFILE) $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR) + echo "Container last built on" > $@ + date >> $@ + +.PHONY: container.dev +container.dev: container_built.info \ No newline at end of file diff --git a/repos/osbuild/tools/Containerfile.dev b/repos/osbuild/tools/Containerfile.dev new file mode 100644 index 0000000..d33f244 --- /dev/null +++ b/repos/osbuild/tools/Containerfile.dev @@ -0,0 +1,36 @@ +FROM registry.fedoraproject.org/fedora:latest + +ARG OSBUILD_DIR=osbuild +ENV OSBUILD_DIR=${OSBUILD_DIR} + +ARG GETTING_STARTED_DIR=osbuild-getting-started +ENV GETTING_STARTED_DIR=${GETTING_STARTED_DIR} + +RUN dnf install -y @rpm-development-tools git + +RUN mkdir /prepare +WORKDIR /prepare + +COPY ${OSBUILD_DIR}/osbuild.spec . + +RUN dnf -y builddep ./osbuild.spec + +RUN pip install pydevd_pycharm debugpy + +RUN mkdir /app +WORKDIR /app + +COPY ${GETTING_STARTED_DIR}/repos/osbuild/Makefile.getting-started Makefile.getting-started + +COPY ${OSBUILD_DIR} /app + +COPY ${GETTING_STARTED_DIR}/repos/osbuild/tools/main_cli_dev.py /app/osbuild/ +COPY ${GETTING_STARTED_DIR}/repos/osbuild/tools/Containerfile.dev.patch /app/tools/ +RUN git apply tools/Containerfile.dev.patch + +# OSBUILD_DIR is different in the container than on the host +RUN OSBUILD_DIR=/app make -f Makefile.getting-started rpm.dev + +RUN dnf install -y ./rpmbuild/RPMS/*.rpm + +ENTRYPOINT [ "/usr/bin/osbuild" ] diff --git a/repos/osbuild/tools/Containerfile.dev.patch b/repos/osbuild/tools/Containerfile.dev.patch new file mode 100644 index 0000000..007d41f --- /dev/null +++ b/repos/osbuild/tools/Containerfile.dev.patch @@ -0,0 +1,13 @@ +diff --git a/setup.py b/setup.py +index cedc5e5d..b5953aa7 100644 +--- a/setup.py ++++ b/setup.py +@@ -18,7 +18,7 @@ setuptools.setup( + ], + entry_points={ + "console_scripts": [ +- "osbuild = osbuild.main_cli:osbuild_cli" ++ "osbuild = osbuild.main_cli_dev:osbuild_dev" + ] + }, + scripts=[ diff --git a/repos/osbuild/tools/main_cli_dev.py b/repos/osbuild/tools/main_cli_dev.py new file mode 100644 index 0000000..49c8256 --- /dev/null +++ b/repos/osbuild/tools/main_cli_dev.py @@ -0,0 +1,28 @@ +"""Development entry point for osbuild + +This is used while building a container with `make container.dev` +This allows to attach the debugger even when more services +of the stack are running in parallel +""" +import os + +import debugpy +import pydevd_pycharm + +from osbuild.main_cli import osbuild_cli + + +def osbuild_dev(): + debug_port = os.getenv("OSBUILD_PYCHARM_DEBUG_PORT") + if debug_port: + debug_host = os.getenv("OSBUILD_PYCHARM_DEBUG_HOST", 'host.docker.internal') + print("Connecting to debugger...") + pydevd_pycharm.settrace(debug_host, port=int(debug_port), stdoutToServer=True, stderrToServer=True) + + debug_port = os.getenv("OSBUILD_DEBUGPY_DEBUG_PORT") + if debug_port: + debug_host = os.getenv("OSBUILD_DEBUGPY_DEBUG_HOST", 'host.docker.internal') + print("Connecting to debugger...") + debugpy.listen(debug_host, port=int(debug_port)) + + osbuild_cli() From 19083afd0e9f709eb911061ae1ec5faad73b68e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Sch=C3=BCller?= Date: Wed, 14 May 2025 12:11:01 +0200 Subject: [PATCH 4/5] backup --- Makefile | 59 +++++++++++------ .../Makefile.getting-started | 63 +++++++++++++++++++ .../tools/Dockerfile-ubi.dev | 50 +++++++++++++++ .../tools/openshift-startup.sh | 26 ++++++++ .../Makefile.getting-started | 42 +++++++++++++ .../tools/Dockerfile.dev | 38 +++++++++++ .../tools/create_storage_conf.sh | 10 +++ .../osbuild-composer/Makefile.getting-started | 19 +++--- .../tools/Dockerfile-composer.dev | 21 +++++-- .../tools/Dockerfile-fauxauth.dev | 36 +++++++++++ .../tools/Dockerfile-worker.dev | 11 +++- ...oint.py => osbuild-composer-entrypoint.py} | 0 service/docker-compose.yml | 6 +- tools/git_stack.sh | 2 +- 14 files changed, 344 insertions(+), 39 deletions(-) create mode 100644 repos/image-builder-crc/Makefile.getting-started create mode 100644 repos/image-builder-crc/tools/Dockerfile-ubi.dev create mode 100644 repos/image-builder-crc/tools/openshift-startup.sh create mode 100644 repos/image-builder-frontend/Makefile.getting-started create mode 100644 repos/image-builder-frontend/tools/Dockerfile.dev create mode 100644 repos/image-builder-frontend/tools/create_storage_conf.sh create mode 100644 repos/osbuild-composer/tools/Dockerfile-fauxauth.dev rename repos/osbuild-composer/tools/{osbuild_composer-entrypoint.py => osbuild-composer-entrypoint.py} (100%) diff --git a/Makefile b/Makefile index 9bbd8b4..39afc11 100644 --- a/Makefile +++ b/Makefile @@ -32,19 +32,37 @@ MAKE_SUB_CALL := make CONTAINER_EXECUTABLE="$(CONTAINER_EXECUTABLE)" # osbuild is indirectly used by osbuild-composer # but we'll mention it here too for better error messages and usability COMMON_SRC_DEPS_NAMES := osbuild osbuild-composer pulp-client community-gateway -COMMON_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(COMMON_SRC_DEPS_NAMES)) +COMMON_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/,$(COMMON_SRC_DEPS_NAMES)) ONPREM_SRC_DEPS_NAMES := weldr-client -ONPREM_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(ONPREM_SRC_DEPS_NAMES)) +ONPREM_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/,$(ONPREM_SRC_DEPS_NAMES)) -SERVICE_SRC_DEPS_NAMES := image-builder image-builder-frontend -SERVICE_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(SERVICE_SRC_DEPS_NAMES)) +SERVICE_SRC_DEPS_NAMES := image-builder-crc image-builder-frontend +SERVICE_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/,$(SERVICE_SRC_DEPS_NAMES)) # should be set if we are already sudo - otherwise we set to "whoami" SUDO_USER ?= $(shell whoami) -$(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN): - @for DIR in $@; do if ! [ -d $$DIR ]; then echo "Please checkout $$DIR so it is available at $$DIR"; exit 1; fi; done +ALL_REQUIRED_DIRS := $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN) + +$(ALL_REQUIRED_DIRS): + @if ! [ -d $@ ]; then \ + echo "Please checkout '$$(basename $@)' so it is available at $$(readlink -f $@)"; \ + echo "I expect a structure like this:"; \ + echo " $$(readlink -f $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR))"; \ + TOTAL=$$(echo $(ALL_REQUIRED_DIRS) | wc -w); \ + COUNT=1; \ + for REPO in $(ALL_REQUIRED_DIRS); do \ + if [ $$COUNT -eq $$TOTAL ]; then \ + PREFIX=" └──"; \ + else \ + PREFIX=" ├──"; \ + fi; \ + echo "$$PREFIX $$(basename $$REPO)"; \ + COUNT=$$((COUNT + 1)); \ + done; \ + exit 1; \ + fi; COMPARE_TO_BRANCH ?= origin/main @@ -57,16 +75,14 @@ ALL_SCRATCH_DIRS := $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIR) $(CLI_DIRS) $(DATA OSBUILD_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/osbuild OSBUILD_COMPOSER_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/osbuild-composer +IMAGE_BUILDER_CRC_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/image-builder-crc +IMAGE_BUILDER_FRONTEND_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/image-builder-frontend .PHONY: service_containers -service_containers: +service_containers: service_sub_make_backend make -C $(OSBUILD_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild/Makefile.getting-started container.dev make -C $(OSBUILD_COMPOSER_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild-composer/Makefile.getting-started container.dev -clean: - make -C $(OSBUILD_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild/Makefile.getting-started clean.dev - make -C $(OSBUILD_COMPOSER_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild-composer/Makefile.getting-started clean.dev - # internal rule for sub-calls # NOTE: This chowns all directories back - as we expect to run partly as root # also we "git fetch origin" to get the current state! @@ -76,15 +92,17 @@ common_sub_makes: @echo "At least for podman the password as already needed now" # creating container image from osbuild as a basis for worker - $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)osbuild-composer container_worker.dev container_composer.dev + make -C $(OSBUILD_COMPOSER_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild-composer/Makefile.getting-started container.dev .PHONY: service_sub_make_backend service_sub_make_backend: - $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder container.dev + make -C $(OSBUILD_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild/Makefile.getting-started container.dev + make -C $(OSBUILD_COMPOSER_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild-composer/Makefile.getting-started container.dev + make -C $(IMAGE_BUILDER_CRC_DIR) -f $(GETTING_STARTED_DIR)/repos/image-builder-crc/Makefile.getting-started container.dev .PHONY: service_sub_make_frontend service_sub_make_frontend: - $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder-frontend container.dev + make -C $(IMAGE_BUILDER_CRC_DIR) -f $(GETTING_STARTED_DIR)/repos/image-builder-frontend/Makefile.getting-started container.dev .PHONY: service_sub_make_cleanup service_sub_make_cleanup: @@ -101,7 +119,7 @@ service_sub_makes: service_sub_make_backend service_sub_make_frontend service_su .PHONY: onprem_sub_makes onprem_sub_makes: # building the cli - $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)weldr-client container.dev + $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/weldr-client container.dev @for DIR in $(COMMON_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN); do echo "Giving directory permissions in '$$DIR' back to '$(SUDO_USER)'"; chown -R $(SUDO_USER): $$DIR || sudo chown -R $(SUDO_USER): $$DIR; done @echo "Your current versions are (comparing to origin/main):" bash -c './tools/git_stack.sh' @@ -133,14 +151,17 @@ $(ALL_SCRATCH_DIRS): .PHONY: wipe_config wipe_config: sudo rm -rf $(SCRATCH_DIR)/$(COMMON_DIR) - rm -f $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder-frontend/node_modules/.cache/webpack-dev-server/server.pem + rm -f $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/image-builder-frontend/node_modules/.cache/webpack-dev-server/server.pem .PHONY: clean -clean_old: prune_service prune_onprem wipe_config +clean: prune_service prune_onprem wipe_config rm -f service_images_built.info rm -f onprem_images_built.info - rm -rf $(SCRATCH_DIR) || (echo "Trying as root" ;sudo rm -rf $(SCRATCH_DIR)) - for DIR in $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN); do $(MAKE_SUB_CALL) -C $$DIR clean; done + rm -rf $(SCRATCH_DIR) 2>/dev/null || (echo "Trying as root" ;sudo rm -rf $(SCRATCH_DIR)) + make -C $(OSBUILD_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild/Makefile.getting-started clean.dev + make -C $(OSBUILD_COMPOSER_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild-composer/Makefile.getting-started clean.dev + make -C $(IMAGE_BUILDER_CRC_DIR) -f $(GETTING_STARTED_DIR)/repos/image-builder-crc/Makefile.getting-started clean.dev + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml down --volumes $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml rm --volumes $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml down --volumes diff --git a/repos/image-builder-crc/Makefile.getting-started b/repos/image-builder-crc/Makefile.getting-started new file mode 100644 index 0000000..bbd1060 --- /dev/null +++ b/repos/image-builder-crc/Makefile.getting-started @@ -0,0 +1,63 @@ +MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +GETTING_STARTED_DIR := $(patsubst %/,%,$(dir $(MAKEFILE_PATH))../../) +SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= $(GETTING_STARTED_DIR)/.. + +IMAGE_BUILDER_CRC_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/image-builder-crc + +.ONESHELL: +SHELL := /bin/bash +.SHELLFLAGS := -ec -o pipefail + +include $(IMAGE_BUILDER_CRC_DIR)/Makefile + +export CONTAINER_EXECUTABLE ?= podman + +DOCKER_IMAGE := image-builder-crc_dev +DOCKERFILE := $(GETTING_STARTED_DIR)/repos/image-builder-crc/tools/Dockerfile-ubi.dev + +SRC_DEPS_EXTERNAL_NAMES := community-gateway osbuild-composer +SRC_DEPS_EXTERNAL_DIRS := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/,$(SRC_DEPS_EXTERNAL_NAMES)) + +SRC_DEPS_NAMES := internal cmd +SRC_DEPS_DIRS := $(addprefix $(IMAGE_BUILDER_CRC_DIR)/,$(SRC_DEPS_NAMES)) + +# All files to check for rebuild! +SRC_DEPS := $(shell find $(SRC_DEPS_DIRS) -name *.go -or -name *.sql) +SRC_DEPS_EXTERNAL := $(shell find $(SRC_DEPS_EXTERNAL_DIRS) -name *.go) + +CONTAINER_DEPS := $(GETTING_STARTED_DIR)/repos/image-builder-crc/tools/openshift-startup.sh + +$(SRC_DEPS_EXTERNAL_DIRS): + @for DIR in $@; do if ! [ -d $$DIR ]; then echo "Please checkout $$DIR so it is available at $$DIR"; exit 1; fi; done + +GOPROXY ?= https://proxy.golang.org,direct + +GOMODARGS ?= -modfile=go.local.mod +# gcflags "-N -l" for golang to allow debugging +GCFLAGS ?= -gcflags=all=-N -gcflags=all=-l +GOPATH ?= $(shell go env GOPATH) + +go.local.mod go.local.sum: $(SRC_DEPS_EXTERNAL_DIRS) go.mod $(SRC_DEPS_EXTERNAL) $(SRC_DEPS) + cp go.mod go.local.mod + cp go.sum go.local.sum + go mod edit $(GOMODARGS) -replace github.com/osbuild/osbuild-composer/pkg/splunk_logger=$(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/osbuild-composer/pkg/splunk_logger + go mod edit $(GOMODARGS) -replace github.com/osbuild/community-gateway=$(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/community-gateway + env GOPROXY=$(GOPROXY) go mod vendor $(GOMODARGS) + +container_built.info: go.local.mod $(DOCKERFILE) $(CONTAINER_DEPS) $(SRC_DEPS) + $(CONTAINER_EXECUTABLE) build \ + -t $(DOCKER_IMAGE) \ + -f $(DOCKERFILE) \ + --build-arg GOMODARGS="$(GOMODARGS)" \ + --build-arg GCFLAGS="$(GCFLAGS)" \ + $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR) + echo "Container last built on" > $@ + date >> $@ + +.PHONY: container.dev +container.dev: container_built.info + +.PHONY: clean.dev +clean.dev: clean + rm -f container_built.info + rm -f go.local.* diff --git a/repos/image-builder-crc/tools/Dockerfile-ubi.dev b/repos/image-builder-crc/tools/Dockerfile-ubi.dev new file mode 100644 index 0000000..c1bf0f9 --- /dev/null +++ b/repos/image-builder-crc/tools/Dockerfile-ubi.dev @@ -0,0 +1,50 @@ +# Use a builder container to build the Go application (which we extract in +# the second container). +FROM registry.access.redhat.com/ubi9/go-toolset:latest AS builder + +ARG IMAGE_BUILDER_CRC_DIR=image-builder-crc +ENV IMAGE_BUILDER_CRC_DIR=$IMAGE_BUILDER_CRC_DIR + +# ubi9/go-toolset defaults to uid 1001. Let's copy the files with this UID as well. +# Otherwise, VCS stamping will fail because git >= 2.35.2 refuses to work in +# a repository owned by a different user. +COPY --chown=1001 ${IMAGE_BUILDER_CRC_DIR} . +ENV GOFLAGS=-mod=vendor + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" +ARG GCFLAGS="" + +RUN go install $GOMODARGS $GCFLAGS ./... + +### builder2 stage +FROM registry.access.redhat.com/ubi9/go-toolset:latest AS builder2 +RUN go install github.com/jackc/tern@latest + +### final stage +# Build an extremely minimal container that only contains our Go application. +FROM registry.access.redhat.com/ubi9/ubi-minimal:latest + +ARG IMAGE_BUILDER_CRC_DIR=image-builder-crc +ENV IMAGE_BUILDER_CRC_DIR=$IMAGE_BUILDER_CRC_DIR + +ARG GETTING_STARTED_DIR=osbuild-getting-started +ENV GETTING_STARTED_DIR=$GETTING_STARTED_DIR + +RUN mkdir /app +RUN mkdir -p "/opt/migrate/" + +COPY --from=builder /opt/app-root/src/go/bin/image-builder /app/ +COPY --from=builder /opt/app-root/src/go/bin/image-builder-migrate-db-tern /app/ +COPY ${IMAGE_BUILDER_CRC_DIR}/distributions /app/distributions +COPY ${IMAGE_BUILDER_CRC_DIR}/internal/db/migrations-tern /app/migrations +COPY ${GETTING_STARTED_DIR}/repos/image-builder-crc/tools/openshift-startup.sh /opt/openshift-startup.sh +RUN chmod a+x /opt/openshift-startup.sh +COPY --from=builder2 /opt/app-root/src/go/bin/tern /opt/migrate/ +COPY --from=builder2 /usr/bin/dlv /usr/bin/dlv +COPY --from=builder2 /usr/share/licenses/delve /usr/share/licenses/delve +ENV TERN_MIGRATIONS_DIR=/app/migrations +EXPOSE 8086 +CMD ["/opt/openshift-startup.sh"] \ No newline at end of file diff --git a/repos/image-builder-crc/tools/openshift-startup.sh b/repos/image-builder-crc/tools/openshift-startup.sh new file mode 100644 index 0000000..425900d --- /dev/null +++ b/repos/image-builder-crc/tools/openshift-startup.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -euo pipefail + +if [[ -z "${KUBERNETES_PORT:-}" ]]; then + echo "Starting image-builder inside container..." + if [[ -n "${GODEBUG_PORT:-}" ]]; then + echo "With golang debugger enabled on port ${GODEBUG_PORT} ..." + echo "NOTE: you HAVE to attach the debugger NOW otherwise the image-builder-backend will not continue running" + /usr/bin/dlv "--listen=:${GODEBUG_PORT}" --headless=true --api-version=2 exec /app/image-builder -- -v + exit $? + fi +# we don't use cloudwatch in ephemeral environment for now +elif [[ "${CLOWDER_ENABLED:=false}" == "true" ]]; then + echo "Starting image-builder inside ephemeral environment..." + echo "Composer URL: ${COMPOSER_URL}" + echo "Composer token URL: ${COMPOSER_TOKEN_URL}" + echo "Distributions dir: ${DISTRIBUTIONS_DIR}" +else + echo "Starting image-builder inside OpenShift..." + echo "Cloudwatch: ${CW_LOG_GROUP} in ${CW_AWS_REGION}" + echo "Composer URL: ${COMPOSER_URL}" + echo "Composer token URL: ${COMPOSER_TOKEN_URL}" + echo "Distributions dir: ${DISTRIBUTIONS_DIR}" +fi + +/app/image-builder -v \ No newline at end of file diff --git a/repos/image-builder-frontend/Makefile.getting-started b/repos/image-builder-frontend/Makefile.getting-started new file mode 100644 index 0000000..1e9e5ab --- /dev/null +++ b/repos/image-builder-frontend/Makefile.getting-started @@ -0,0 +1,42 @@ +MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +GETTING_STARTED_DIR := $(patsubst %/,%,$(dir $(MAKEFILE_PATH))../../) +SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= $(GETTING_STARTED_DIR)/.. + +IMAGE_BUILDER_FRONTEND_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/image-builder-frontend + +.ONESHELL: +SHELL := /bin/bash +.SHELLFLAGS := -ec -o pipefail + +include $(IMAGE_BUILDER_FRONTEND_DIR)/Makefile + +export CONTAINER_EXECUTABLE ?= podman + +DOCKER_IMAGE := image-builder-frontend_dev +DOCKERFILE := $(GETTING_STARTED_DIR)/repos/image-builder-frontend/tools/Dockerfile.dev + +# All files to check for rebuild! +SRC_DEPS := $(shell find $(IMAGE_BUILDER_FRONTEND_DIR) -not -path './node_modules/*' \ + -not -path './dist/*' \ + -not -path './coverage/*' \ + \( -name "*.ts" \ + -or -name "*.tsx" \ + -or -name "*.yml" \ + -or -name "*.yaml" \ + -or -name "*.json" \ + -or -name "*.js" \) ) + +clean.dev: clean + rm -f container_built.info + +container_built.info: $(DOCKERFILE) $(SRC_DEPS) + $(CONTAINER_EXECUTABLE) build \ + -t $(DOCKER_IMAGE) \ + -f $(DOCKERFILE) \ + $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR) \ + --build-arg BUILD_DATE=$(shell date +%Y%m%d_%H%M%S) + echo "Container last built on" > $@ + date >> $@ + +.PHONY: container.dev +container.dev: container_built.info ## build a container with the frontend diff --git a/repos/image-builder-frontend/tools/Dockerfile.dev b/repos/image-builder-frontend/tools/Dockerfile.dev new file mode 100644 index 0000000..3866017 --- /dev/null +++ b/repos/image-builder-frontend/tools/Dockerfile.dev @@ -0,0 +1,38 @@ +FROM node:20 + +USER root +# frontend-components need a container system +# this is the reason why we run in docker -> to start podman then +RUN apt update && apt install -y podman + +WORKDIR /app + +ARG IMAGE_BUILDER_FRONTEND_DIR=image-builder-frontend +ENV IMAGE_BUILDER_FRONTEND_DIR=$IMAGE_BUILDER_FRONTEND_DIR + +ARG GETTING_STARTED_DIR=osbuild-getting-started +ENV GETTING_STARTED_DIR=$GETTING_STARTED_DIR + +ARG BACKEND_HOSTNAME=localhost +ENV BACKEND_HOSTNAME=$BACKEND_HOSTNAME + +ARG BACKEND_PORT=8086 +ENV BACKEND_PORT=$BACKEND_PORT + +COPY ${IMAGE_BUILDER_FRONTEND_DIR} . +COPY ${GETTING_STARTED_DIR}/repos/image-builder-frontend/tools/create_storage_conf.sh /app/distribution/ + + +RUN chmod a+x /app/distribution/*.sh +RUN /app/distribution/create_storage_conf.sh + +# Helper to re-run npm ci +ARG BUILD_DATE=NOT_SET +ENV BUILD_DATE=$BUILD_DATE + +RUN npm ci + +EXPOSE 8002 +EXPOSE 1337 + +CMD [ "npm", "run", "start:local" ] diff --git a/repos/image-builder-frontend/tools/create_storage_conf.sh b/repos/image-builder-frontend/tools/create_storage_conf.sh new file mode 100644 index 0000000..d4a38e8 --- /dev/null +++ b/repos/image-builder-frontend/tools/create_storage_conf.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +mkdir -p /etc/containers + +tee -a /etc/containers/storage.conf < Date: Wed, 14 May 2025 18:04:43 +0200 Subject: [PATCH 5/5] backup --- Makefile | 2 +- service/docker-compose.yml | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 39afc11..d713fef 100644 --- a/Makefile +++ b/Makefile @@ -184,7 +184,7 @@ run_service: $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIRS)) service_containers # if you want to run the frontend yourself - outside the docker environment .PHONY: run_service_no_frontend run_service_no_frontend: service_containers_no_frontend - $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml up backend fauxauth worker composer minio postgres_backend postgres_composer + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml up backend fauxauth worker composer minio postgres_image_builder_crc postgres_composer # only for strange crashes - should shut down properly in normal operation .PHONY: stop_service diff --git a/service/docker-compose.yml b/service/docker-compose.yml index eb823d1..86cb958 100644 --- a/service/docker-compose.yml +++ b/service/docker-compose.yml @@ -24,7 +24,7 @@ services: - ../../osbuild-composer/repositories:/config/repositories:z postgres_composer: - image: docker.io/postgres:10.5 + image: docker.io/postgres:16.8 healthcheck: test: [ "CMD", "pg_isready", "-h", "localhost", "-U", "postgres", "-d", "postgres" ] interval: 2s @@ -115,8 +115,8 @@ services: extra_hosts: - "host.docker.internal:host-gateway" - postgres_backend: - image: docker.io/postgres:10.5 + postgres_image_builder_crc: + image: docker.io/postgres:16.8 healthcheck: test: [ "CMD", "pg_isready", "-h", "localhost", "-U", "postgres", "-d", "postgres" ] interval: 2s @@ -131,7 +131,7 @@ services: servicenet: ipv4_address: 172.31.0.30 - backend: + image_builder_crc: image: image-builder-crc_dev ports: - 8086:8086 @@ -144,10 +144,10 @@ services: volumes: - config_volume:/etc/image-builder environment: - - LISTEN_ADDRESS=backend:8086 + - LISTEN_ADDRESS=image_builder_crc:8086 - LOG_LEVEL=DEBUG - ALLOWED_ORG_IDS=* - - PGHOST=postgres_backend + - PGHOST=postgres_image_builder_crc - PGPORT=5432 - PGDATABASE=postgres - PGUSER=postgres @@ -167,7 +167,7 @@ services: condition: service_completed_successfully composer: condition: service_healthy - postgres_backend: + postgres_image_builder_crc: condition: service_healthy restart: on-failure networks: @@ -219,7 +219,7 @@ services: security_opt: - seccomp=unconfined environment: - - BACKEND_HOSTNAME=backend + - BACKEND_HOSTNAME=image_builder_crc ports: - 1337:1337 - 8002:8002