diff --git a/.gitignore b/.gitignore index 68c759d..f53f66f 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,6 @@ on-prem/build/config/*.pem on-prem/build/config/*.toml on-prem/build/config/ca on-prem/build/config/repositories +service_images_built.info +onprem_images_built.info + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d713fef --- /dev/null +++ b/Makefile @@ -0,0 +1,240 @@ +.PHONY: help +help: + @echo "make [TARGETS...]" + @echo + @echo "This is the makefile of osbuild-getting-started. The following" + @echo "targets are available:" + @echo + @echo " service_containers: Build all needed containers from source to be able to run the service" + @echo " run_service: Run all containers needed for the 'service'" + @echo " This is running in foreground. Use CTRL-C to stop the containers." + @echo " run_service_no_frontend: Run all containers except for the frontend" + @echo " stop_service: Usually the containers get stopped by CTRL-C. So 'stop-service'" + @echo " is only needed if something strange happened and not all are stopped." + @echo " prune_service: Remove the containers, including the test-data!" + @echo " If you want empty databases" + @echo " clean: Clean all subprojects to assure a rebuild" + +MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +GETTING_STARTED_DIR := $(patsubst %/,%,$(dir $(MAKEFILE_PATH))) + +# source where the other repos are locally +# has to end with a trailing slash +export SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= $(GETTING_STARTED_DIR)/.. + +# either "docker" or "sudo podman" +# podman needs to build as root as it also needs to run as root afterwards +export CONTAINER_EXECUTABLE ?= docker +export CONTAINER_COMPOSE_EXECUTABLE ?= $(CONTAINER_EXECUTABLE) compose + +MAKE_SUB_CALL := make CONTAINER_EXECUTABLE="$(CONTAINER_EXECUTABLE)" + +# osbuild is indirectly used by osbuild-composer +# but we'll mention it here too for better error messages and usability +COMMON_SRC_DEPS_NAMES := osbuild osbuild-composer pulp-client community-gateway +COMMON_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/,$(COMMON_SRC_DEPS_NAMES)) + +ONPREM_SRC_DEPS_NAMES := weldr-client +ONPREM_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/,$(ONPREM_SRC_DEPS_NAMES)) + +SERVICE_SRC_DEPS_NAMES := image-builder-crc image-builder-frontend +SERVICE_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/,$(SERVICE_SRC_DEPS_NAMES)) + +# should be set if we are already sudo - otherwise we set to "whoami" +SUDO_USER ?= $(shell whoami) + +ALL_REQUIRED_DIRS := $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN) + +$(ALL_REQUIRED_DIRS): + @if ! [ -d $@ ]; then \ + echo "Please checkout '$$(basename $@)' so it is available at $$(readlink -f $@)"; \ + echo "I expect a structure like this:"; \ + echo " $$(readlink -f $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR))"; \ + TOTAL=$$(echo $(ALL_REQUIRED_DIRS) | wc -w); \ + COUNT=1; \ + for REPO in $(ALL_REQUIRED_DIRS); do \ + if [ $$COUNT -eq $$TOTAL ]; then \ + PREFIX=" └──"; \ + else \ + PREFIX=" ├──"; \ + fi; \ + echo "$$PREFIX $$(basename $$REPO)"; \ + COUNT=$$((COUNT + 1)); \ + done; \ + exit 1; \ + fi; + +COMPARE_TO_BRANCH ?= origin/main + +SCRATCH_DIR := $(HOME)/.cache/osbuild-getting-started/scratch +export SCRATCH_DIR +COMMON_DIR := image-builder-config +CLI_DIRS := weldr cloudapi dnf-json +DATA_DIR := data/s3/service +ALL_SCRATCH_DIRS := $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIR) $(CLI_DIRS) $(DATA_DIR)) + +OSBUILD_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/osbuild +OSBUILD_COMPOSER_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/osbuild-composer +IMAGE_BUILDER_CRC_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/image-builder-crc +IMAGE_BUILDER_FRONTEND_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/image-builder-frontend + +.PHONY: service_containers +service_containers: service_sub_make_backend + make -C $(OSBUILD_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild/Makefile.getting-started container.dev + make -C $(OSBUILD_COMPOSER_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild-composer/Makefile.getting-started container.dev + +# internal rule for sub-calls +# NOTE: This chowns all directories back - as we expect to run partly as root +# also we "git fetch origin" to get the current state! +.PHONY: common_sub_makes +common_sub_makes: + @echo "We need to build everything as root, as the target also needs to run as root." + @echo "At least for podman the password as already needed now" + + # creating container image from osbuild as a basis for worker + make -C $(OSBUILD_COMPOSER_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild-composer/Makefile.getting-started container.dev + +.PHONY: service_sub_make_backend +service_sub_make_backend: + make -C $(OSBUILD_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild/Makefile.getting-started container.dev + make -C $(OSBUILD_COMPOSER_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild-composer/Makefile.getting-started container.dev + make -C $(IMAGE_BUILDER_CRC_DIR) -f $(GETTING_STARTED_DIR)/repos/image-builder-crc/Makefile.getting-started container.dev + +.PHONY: service_sub_make_frontend +service_sub_make_frontend: + make -C $(IMAGE_BUILDER_CRC_DIR) -f $(GETTING_STARTED_DIR)/repos/image-builder-frontend/Makefile.getting-started container.dev + +.PHONY: service_sub_make_cleanup +service_sub_make_cleanup: + @for DIR in $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN); do echo "Giving directory permissions in '$$DIR' back to '$(SUDO_USER)'"; chown -R $(SUDO_USER): $$DIR || sudo chown -R $(SUDO_USER): $$DIR; done + @echo "Your current versions are (comparing to origin/main):" + bash -c './tools/git_stack.sh' + +.PHONY: service_sub_makes_no_frontend +service_sub_makes_no_frontend: service_sub_make_backend service_sub_make_cleanup + +.PHONY: service_sub_makes +service_sub_makes: service_sub_make_backend service_sub_make_frontend service_sub_make_cleanup + +.PHONY: onprem_sub_makes +onprem_sub_makes: + # building the cli + $(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/weldr-client container.dev + @for DIR in $(COMMON_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN); do echo "Giving directory permissions in '$$DIR' back to '$(SUDO_USER)'"; chown -R $(SUDO_USER): $$DIR || sudo chown -R $(SUDO_USER): $$DIR; done + @echo "Your current versions are (comparing to origin/main):" + bash -c './tools/git_stack.sh' + +.PHONY: service_containers_old +service_containers_old: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) common_sub_makes service_sub_makes service_images_built.info + +.PHONY: service_containers_no_frontend +service_containers_no_frontend: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) common_sub_makes service_sub_makes_no_frontend service_images_built.info + +onprem_containers: $(COMMON_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN) common_sub_makes onprem_sub_makes onprem_images_built.info + +service_images_built.info: service/config/Dockerfile-config service/config/composer/osbuild-composer.toml $(ALL_SCRATCH_DIRS) + # building remaining containers (config, fauxauth) + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml build --build-arg CONFIG_BUILD_DATE=$(shell date -r $(SCRATCH_DIR)/$(COMMON_DIR) +%Y%m%d_%H%M%S) + echo "Images last built on" > $@ + date >> $@ + +onprem_images_built.info: service/config/Dockerfile-config-onprem service/config/composer/osbuild-composer-onprem.toml $(ALL_SCRATCH_DIRS) + # building remaining containers (config) + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml build --build-arg CONFIG_BUILD_DATE=$(shell date -r $(SCRATCH_DIR)/$(COMMON_DIR) +%Y%m%d_%H%M%S) + echo "Images last built on" > $@ + date >> $@ + +$(ALL_SCRATCH_DIRS): + @echo "Creating directory: $@" + mkdir -p $@ || ( echo "Trying as root" ; sudo mkdir -p $@ ) + +.PHONY: wipe_config +wipe_config: + sudo rm -rf $(SCRATCH_DIR)/$(COMMON_DIR) + rm -f $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/image-builder-frontend/node_modules/.cache/webpack-dev-server/server.pem + +.PHONY: clean +clean: prune_service prune_onprem wipe_config + rm -f service_images_built.info + rm -f onprem_images_built.info + rm -rf $(SCRATCH_DIR) 2>/dev/null || (echo "Trying as root" ;sudo rm -rf $(SCRATCH_DIR)) + make -C $(OSBUILD_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild/Makefile.getting-started clean.dev + make -C $(OSBUILD_COMPOSER_DIR) -f $(GETTING_STARTED_DIR)/repos/osbuild-composer/Makefile.getting-started clean.dev + make -C $(IMAGE_BUILDER_CRC_DIR) -f $(GETTING_STARTED_DIR)/repos/image-builder-crc/Makefile.getting-started clean.dev + + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml down --volumes + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml rm --volumes + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml down --volumes + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml rm --volumes + +# for compatibility of relative volume mount paths +# between docker and podman we have to change to the directory +.PHONY: run_service +.ONESHELL: +SHELL := /bin/bash +.SHELLFLAGS := -ec -o pipefail + +run_service: $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIRS)) service_containers + export PORTS="$(shell $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml config|grep -Po '(?<=published: ")[0-9]+')" + echo "-- Checking if any of our ports are used: $$PORTS" + sudo netstat -lntp|grep -E "$$(echo "$$PORTS"|tr ' ' ':')" + echo "-- Check done" + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml up + +# if you want to run the frontend yourself - outside the docker environment +.PHONY: run_service_no_frontend +run_service_no_frontend: service_containers_no_frontend + $(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml up backend fauxauth worker composer minio postgres_image_builder_crc postgres_composer + +# only for strange crashes - should shut down properly in normal operation +.PHONY: stop_service +.ONESHELL: +stop_service: + cd service + $(CONTAINER_COMPOSE_EXECUTABLE) stop + +.PHONY: prune_service +.ONESHELL: +prune_service: + cd service + $(CONTAINER_COMPOSE_EXECUTABLE) down + +.PHONY: prune_onprem +.ONESHELL: +prune_onprem: + cd service + $(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml down + +# for compatibility of relative volume mount paths +# between docker and podman we have to change to the directory +.PHONY: run_onprem +.ONESHELL: +run_onprem: $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIRS) $(CLI_DIRS)) onprem_containers + cd service + echo "Remove dangling sockets as root" + sudo rm -f $(addsuffix /api.sock*, $(addprefix $(SCRATCH_DIR)/, $(CLI_DIRS))) + $(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml up -d + @echo "------ Welcome to osbuild! You can now use 'composer-cli' to build images" + @echo " … and 'exit' afterwards" + @echo " You might also be interested to open up a second terminal and" + @echo " run 'docker compose -f $(shell readlink -f service/docker-compose-onprem.yml) logs --follow' to see possible problems" + + $(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml run --entrypoint /bin/bash -ti cli + $(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml stop + +%.svg: %.dot + dot -T svg $< > $@ + +.PHONY: docs +docs: docs/src_compile_service.svg docs/src_compile_onprem.svg + +.PHONY: overview +overview: + @echo "Fetching all repos for better overview if rebase will be necessary:" +ifeq (${SUDO_USER},$(shell whoami)) + bash -c './tools/git_stack.sh fetch --all' +else + sudo -u ${SUDO_USER} bash -c './tools/git_stack.sh fetch --all' +endif + @bash -c './tools/git_stack.sh' + diff --git a/README.md b/README.md index d4aeb11..e2ccb3d 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,9 @@ provide more in-depth explainations of these products, their architecture and ho ## Getting started +You can develop and install the individual components on your local computer, or you can run the +whole stack as containers fully from source. + For further instructions on how to run containerized versions of the CLI and service locally: -- the [on-premise README](./on-prem/README.md) offers explainations on how to run the on-premise CLI tool. +- the [on-premise README](./on-prem/README.md) offers explanations on how to run the on-premise CLI tool. - the [service README](./service/README.md) provides instructions on how to get started with a stack of the hosted service. diff --git a/docs/src_compile_onprem.dot b/docs/src_compile_onprem.dot new file mode 100644 index 0000000..e45f843 --- /dev/null +++ b/docs/src_compile_onprem.dot @@ -0,0 +1,101 @@ +digraph src_compile { + newrank=true; + subgraph cluster_osbuild_getting_started { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild-\ngetting-started"; + osbuild_getting_started_makefile [ + label = "make\nonprem_containers" + width = 2; + ]; + osbuild_getting_started_run [ + label = "make\nrun_onprem" + width = 2; + ]; + } + + subgraph cluster_osbuild { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild\n "; + osbuild_makefile [ + label = "make\ncontainer"; + width = 1.5; + ]; + } + + subgraph cluster_osbuild_composer { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild-composer\n "; + osbuild_composer_makefile_composer [ + label = "make\ncontainer_composer" + width = 2; + ]; + osbuild_composer_makefile_worker [ + label = "make\ncontainer_worker" + width = 2; + ]; + + osbuild_composer_go [ label = "go code"; ]; + } + + subgraph cluster_weldr_client { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "weldr-client"; + weldr_client_makefile [ + label = "make\ncontainer" + width = 1.5; + ]; + } + subgraph cluster_pulp_client { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "pulp-client"; + pulp_client_go [ + label = "go code"; + width = 1.5; + ]; + } + subgraph cluster_images { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "images"; + images_go [label = "go code";]; + } + { rank=same; + osbuild_makefile; + osbuild_composer_makefile_worker; + osbuild_composer_makefile_composer; + weldr_client_makefile; + + } + { rank=same; + edge [style=dashed; headport=s; tailport=s]; + pulp_client_go; + images_go; + } + osbuild_composer_go -> pulp_client_go [style=dashed;]; + osbuild_composer_go -> images_go [style=dashed;]; + + osbuild_getting_started_run -> osbuild_getting_started_makefile; + + osbuild_getting_started_makefile -> osbuild_makefile; + osbuild_getting_started_makefile -> osbuild_composer_makefile_worker; + osbuild_getting_started_makefile -> osbuild_composer_makefile_composer; + + + osbuild_getting_started_makefile -> weldr_client_makefile; + osbuild_composer_makefile_worker -> osbuild_composer_go [style=dashed]; + osbuild_composer_makefile_composer -> osbuild_composer_go [style=dashed]; + + osbuild_composer_makefile_worker -> osbuild_makefile [style=dashed; headport=s; tailport=s]; + +} diff --git a/docs/src_compile_onprem.svg b/docs/src_compile_onprem.svg new file mode 100644 index 0000000..7585d67 --- /dev/null +++ b/docs/src_compile_onprem.svg @@ -0,0 +1,166 @@ + + + + + + +src_compile + + +cluster_osbuild_getting_started + +osbuild- +getting-started + + +cluster_osbuild + +osbuild +  + + +cluster_osbuild_composer + +osbuild-composer +  + + +cluster_weldr_client + +weldr-client + + +cluster_pulp_client + +pulp-client + + +cluster_images + +images + + + +osbuild_getting_started_makefile + +make +onprem_containers + + + +osbuild_makefile + +make +container + + + +osbuild_getting_started_makefile->osbuild_makefile + + + + + +osbuild_composer_makefile_composer + +make +container_composer + + + +osbuild_getting_started_makefile->osbuild_composer_makefile_composer + + + + + +osbuild_composer_makefile_worker + +make +container_worker + + + +osbuild_getting_started_makefile->osbuild_composer_makefile_worker + + + + + +weldr_client_makefile + +make +container + + + +osbuild_getting_started_makefile->weldr_client_makefile + + + + + +osbuild_getting_started_run + +make +run_onprem + + + +osbuild_getting_started_run->osbuild_getting_started_makefile + + + + + +osbuild_composer_go + +go code + + + +osbuild_composer_makefile_composer->osbuild_composer_go + + + + + +osbuild_composer_makefile_worker:s->osbuild_makefile:s + + + + + +osbuild_composer_makefile_worker->osbuild_composer_go + + + + + +pulp_client_go + +go code + + + +osbuild_composer_go->pulp_client_go + + + + + +images_go + +go code + + + +osbuild_composer_go->images_go + + + + + diff --git a/docs/src_compile_service.dot b/docs/src_compile_service.dot new file mode 100644 index 0000000..e139f55 --- /dev/null +++ b/docs/src_compile_service.dot @@ -0,0 +1,132 @@ +digraph src_compile { + newrank=true; + subgraph cluster_osbuild_getting_started { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild-\ngetting-started"; + osbuild_getting_started_makefile [ + label = "make\nservice_containers" + width = 2; + ]; + osbuild_getting_started_run [ + label = "make\nrun_service" + width = 2; + ]; + } + + subgraph cluster_osbuild { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild\n "; + osbuild_makefile [ + label = "make\ncontainer"; + width = 1.5; + ]; + } + + subgraph cluster_osbuild_composer { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "osbuild-composer\n "; + osbuild_composer_makefile_composer [ + label = "make\ncontainer_composer" + width = 2; + ]; + osbuild_composer_makefile_worker [ + label = "make\ncontainer_worker" + width = 2; + ]; + + osbuild_composer_go [ label = "go code"; ]; + } + + subgraph cluster_image_builder { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "image-builder\n "; + image_builder_makefile [ + label = "make\ncontainer" + width = 1.5; + ]; + image_builder_go [ label = "go code"; ]; + } + + subgraph cluster_image_builder_frontend { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "image-builder-\nfronted"; + image_builder_frontend_makefile [ + label = "make\ncontainer" + width = 1.5; + ]; + } + subgraph cluster_pulp_client { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "pulp-client"; + pulp_client_go [ + label = "go code"; + width = 1.5; + ]; + } + subgraph cluster_images { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "images"; + images_go [label = "go code";]; + } + subgraph cluster_community_gateway { + node [ shape=box; fillcolor=white; style=filled; ] + style=filled; fillcolor=lightgrey; + + label = "community-gateway"; + community_gateway_go [ + label = "go code"; + width = 2; + ]; + } + + { rank=same; + osbuild_makefile; + osbuild_composer_makefile_composer; + osbuild_composer_makefile_worker; + image_builder_makefile; + image_builder_frontend_makefile; + } + { rank=same; + edge [style=dashed]; + image_builder_go -> osbuild_composer_go; + } + { rank=same; + edge [style=dashed; headport=s; tailport=s]; + pulp_client_go; + images_go; + community_gateway_go; + } + osbuild_composer_go -> pulp_client_go [style=dashed;]; + osbuild_composer_go -> images_go [style=dashed;]; + image_builder_go -> community_gateway_go [style=dashed;]; + + osbuild_getting_started_run -> osbuild_getting_started_makefile; + + osbuild_getting_started_makefile -> osbuild_makefile; + osbuild_getting_started_makefile -> osbuild_composer_makefile_worker; + osbuild_getting_started_makefile -> osbuild_composer_makefile_composer; + osbuild_getting_started_makefile -> image_builder_makefile; + osbuild_getting_started_makefile -> image_builder_frontend_makefile; + osbuild_composer_makefile_worker -> osbuild_composer_go [style=dashed]; + osbuild_composer_makefile_composer -> osbuild_composer_go [style=dashed]; + image_builder_makefile -> image_builder_go [style=dashed]; + + + + osbuild_composer_makefile_worker -> osbuild_makefile [style=dashed; headport=s; tailport=s]; + +} diff --git a/docs/src_compile_service.svg b/docs/src_compile_service.svg new file mode 100644 index 0000000..f3dcf5f --- /dev/null +++ b/docs/src_compile_service.svg @@ -0,0 +1,221 @@ + + + + + + +src_compile + + +cluster_osbuild_getting_started + +osbuild- +getting-started + + +cluster_osbuild + +osbuild +  + + +cluster_osbuild_composer + +osbuild-composer +  + + +cluster_image_builder + +image-builder +  + + +cluster_image_builder_frontend + +image-builder- +fronted + + +cluster_pulp_client + +pulp-client + + +cluster_images + +images + + +cluster_community_gateway + +community-gateway + + + +osbuild_getting_started_makefile + +make +service_containers + + + +osbuild_makefile + +make +container + + + +osbuild_getting_started_makefile->osbuild_makefile + + + + + +osbuild_composer_makefile_composer + +make +container_composer + + + +osbuild_getting_started_makefile->osbuild_composer_makefile_composer + + + + + +osbuild_composer_makefile_worker + +make +container_worker + + + +osbuild_getting_started_makefile->osbuild_composer_makefile_worker + + + + + +image_builder_makefile + +make +container + + + +osbuild_getting_started_makefile->image_builder_makefile + + + + + +image_builder_frontend_makefile + +make +container + + + +osbuild_getting_started_makefile->image_builder_frontend_makefile + + + + + +osbuild_getting_started_run + +make +run_service + + + +osbuild_getting_started_run->osbuild_getting_started_makefile + + + + + +osbuild_composer_go + +go code + + + +osbuild_composer_makefile_composer->osbuild_composer_go + + + + + +osbuild_composer_makefile_worker:s->osbuild_makefile:s + + + + + +osbuild_composer_makefile_worker->osbuild_composer_go + + + + + +pulp_client_go + +go code + + + +osbuild_composer_go->pulp_client_go + + + + + +images_go + +go code + + + +osbuild_composer_go->images_go + + + + + +image_builder_go + +go code + + + +image_builder_makefile->image_builder_go + + + + + +image_builder_go->osbuild_composer_go + + + + + +community_gateway_go + +go code + + + +image_builder_go->community_gateway_go + + + + + diff --git a/on-prem/bin/run.py b/on-prem/bin/run.py index 948e234..289f295 100755 --- a/on-prem/bin/run.py +++ b/on-prem/bin/run.py @@ -93,7 +93,6 @@ async def env( "--network", "podman", "--name", f"{prefix}-composer", f"ogsc/run/composer:{osbuild_composer_version}", - "--dnf-json", "--weldr-api", "--remote-worker-api", "--composer-api", diff --git a/on-prem/bin/setup-host.py b/on-prem/bin/setup-host.py index 97bd577..5ac0c40 100755 --- a/on-prem/bin/setup-host.py +++ b/on-prem/bin/setup-host.py @@ -4,18 +4,15 @@ # of setting up all required dependencies on the host to run osbuild services in # containers. -import sys -import subprocess import logging - +import subprocess +import sys log = logging.getLogger(__name__) # The base set of packages necessary. -package_base = { - "git" -} +package_base = {"git"} package_set = { "container": { @@ -25,11 +22,14 @@ def package_install(packages): - subprocess.run([ - "dnf", - "in", - "-y", - ] + list(packages)) + subprocess.run( + [ + "dnf", + "in", + "-y", + ] + + list(packages) + ) return 0 diff --git a/on-prem/src/ogsc/run/composer/entrypoint.py b/on-prem/src/ogsc/run/composer/entrypoint.py index d545d32..e38baf1 100644 --- a/on-prem/src/ogsc/run/composer/entrypoint.py +++ b/on-prem/src/ogsc/run/composer/entrypoint.py @@ -118,34 +118,12 @@ def _parse_args(self): help="Disable the weldr-API", ) - # --[no-]dnf-json - self._parser.add_argument( - "--dnf-json", - action="store_true", - dest="dnf_json", - help="Enable dnf-json", - ) - self._parser.add_argument( - "--no-dnf-json", - action="store_false", - dest="dnf_json", - help="Disable dnf-json", - ) - self._parser.add_argument( - "--dnf-json-port", - type=int, - default=0, - dest="dnf_json_port", - help="Specify the port dnf-json should listen on", - ) - self._parser.set_defaults( builtin_worker=False, composer_api=False, local_worker_api=False, remote_worker_api=False, weldr_api=False, - dnf_json=False, ) return self._parser.parse_args(self._argv[1:]) @@ -295,51 +273,17 @@ def _spawn_composer(sockets): preexec_fn=preexec_setenv, ) - def _spawn_dnf_json(self): - cmd = [ - "/usr/libexec/osbuild-depsolve-dnf", - ] - - if self.args.dnf_json_port: - sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - self._exitstack.enter_context(contextlib.closing(sock)) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) - sock.bind(("::", self.args.dnf_json_port)) - sock.listen() - else: - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self._exitstack.enter_context(contextlib.closing(sock)) - os.makedirs("/run/osbuild-dnf-json/", exist_ok=True) - sock.bind("/run/osbuild-dnf-json/api.sock") - sock.listen() - - dnfenv = os.environ.copy() - dnfenv["LISTEN_FDS"] = "1" - dnfenv["LISTEN_FD"] = str(sock.fileno()) - - return subprocess.Popen( - cmd, - cwd="/usr/libexec/osbuild-composer", - stdin=subprocess.DEVNULL, - stderr=subprocess.STDOUT, - env=dnfenv, - pass_fds=[sock.fileno()] - ) - def run(self): """Program Runtime""" proc_composer = None proc_worker = None - proc_dnf_json = None res = 0 sockets = self._prepare_sockets() def handler(signum, frame): proc_composer.terminate() proc_worker.terminate() - proc_dnf_json.terminate() signal.signal(signal.SIGTERM, handler) @@ -351,9 +295,6 @@ def handler(signum, frame): if self.args.builtin_worker: proc_worker = self._spawn_worker() - if self.args.dnf_json: - proc_dnf_json = self._spawn_dnf_json() - if any([self.args.weldr_api, self.args.composer_api, self.args.local_worker_api, self.args.remote_worker_api]): proc_composer = self._spawn_composer(sockets) @@ -365,11 +306,6 @@ def handler(signum, frame): proc_worker.terminate() proc_worker.wait() - if proc_dnf_json: - if proc_composer: - proc_dnf_json.terminate() - proc_dnf_json.wait() - except KeyboardInterrupt: if proc_composer: proc_composer.terminate() @@ -377,14 +313,9 @@ def handler(signum, frame): if proc_worker: proc_worker.terminate() proc_worker.wait() - if proc_dnf_json: - proc_dnf_json.terminate() - proc_dnf_json.wait() except: if proc_worker: proc_worker.kill() - if proc_dnf_json: - proc_dnf_json.kill() if proc_composer: proc_composer.kill() raise diff --git a/repos/image-builder-crc/Makefile.getting-started b/repos/image-builder-crc/Makefile.getting-started new file mode 100644 index 0000000..bbd1060 --- /dev/null +++ b/repos/image-builder-crc/Makefile.getting-started @@ -0,0 +1,63 @@ +MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +GETTING_STARTED_DIR := $(patsubst %/,%,$(dir $(MAKEFILE_PATH))../../) +SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= $(GETTING_STARTED_DIR)/.. + +IMAGE_BUILDER_CRC_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/image-builder-crc + +.ONESHELL: +SHELL := /bin/bash +.SHELLFLAGS := -ec -o pipefail + +include $(IMAGE_BUILDER_CRC_DIR)/Makefile + +export CONTAINER_EXECUTABLE ?= podman + +DOCKER_IMAGE := image-builder-crc_dev +DOCKERFILE := $(GETTING_STARTED_DIR)/repos/image-builder-crc/tools/Dockerfile-ubi.dev + +SRC_DEPS_EXTERNAL_NAMES := community-gateway osbuild-composer +SRC_DEPS_EXTERNAL_DIRS := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/,$(SRC_DEPS_EXTERNAL_NAMES)) + +SRC_DEPS_NAMES := internal cmd +SRC_DEPS_DIRS := $(addprefix $(IMAGE_BUILDER_CRC_DIR)/,$(SRC_DEPS_NAMES)) + +# All files to check for rebuild! +SRC_DEPS := $(shell find $(SRC_DEPS_DIRS) -name *.go -or -name *.sql) +SRC_DEPS_EXTERNAL := $(shell find $(SRC_DEPS_EXTERNAL_DIRS) -name *.go) + +CONTAINER_DEPS := $(GETTING_STARTED_DIR)/repos/image-builder-crc/tools/openshift-startup.sh + +$(SRC_DEPS_EXTERNAL_DIRS): + @for DIR in $@; do if ! [ -d $$DIR ]; then echo "Please checkout $$DIR so it is available at $$DIR"; exit 1; fi; done + +GOPROXY ?= https://proxy.golang.org,direct + +GOMODARGS ?= -modfile=go.local.mod +# gcflags "-N -l" for golang to allow debugging +GCFLAGS ?= -gcflags=all=-N -gcflags=all=-l +GOPATH ?= $(shell go env GOPATH) + +go.local.mod go.local.sum: $(SRC_DEPS_EXTERNAL_DIRS) go.mod $(SRC_DEPS_EXTERNAL) $(SRC_DEPS) + cp go.mod go.local.mod + cp go.sum go.local.sum + go mod edit $(GOMODARGS) -replace github.com/osbuild/osbuild-composer/pkg/splunk_logger=$(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/osbuild-composer/pkg/splunk_logger + go mod edit $(GOMODARGS) -replace github.com/osbuild/community-gateway=$(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/community-gateway + env GOPROXY=$(GOPROXY) go mod vendor $(GOMODARGS) + +container_built.info: go.local.mod $(DOCKERFILE) $(CONTAINER_DEPS) $(SRC_DEPS) + $(CONTAINER_EXECUTABLE) build \ + -t $(DOCKER_IMAGE) \ + -f $(DOCKERFILE) \ + --build-arg GOMODARGS="$(GOMODARGS)" \ + --build-arg GCFLAGS="$(GCFLAGS)" \ + $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR) + echo "Container last built on" > $@ + date >> $@ + +.PHONY: container.dev +container.dev: container_built.info + +.PHONY: clean.dev +clean.dev: clean + rm -f container_built.info + rm -f go.local.* diff --git a/repos/image-builder-crc/tools/Dockerfile-ubi.dev b/repos/image-builder-crc/tools/Dockerfile-ubi.dev new file mode 100644 index 0000000..c1bf0f9 --- /dev/null +++ b/repos/image-builder-crc/tools/Dockerfile-ubi.dev @@ -0,0 +1,50 @@ +# Use a builder container to build the Go application (which we extract in +# the second container). +FROM registry.access.redhat.com/ubi9/go-toolset:latest AS builder + +ARG IMAGE_BUILDER_CRC_DIR=image-builder-crc +ENV IMAGE_BUILDER_CRC_DIR=$IMAGE_BUILDER_CRC_DIR + +# ubi9/go-toolset defaults to uid 1001. Let's copy the files with this UID as well. +# Otherwise, VCS stamping will fail because git >= 2.35.2 refuses to work in +# a repository owned by a different user. +COPY --chown=1001 ${IMAGE_BUILDER_CRC_DIR} . +ENV GOFLAGS=-mod=vendor + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" +ARG GCFLAGS="" + +RUN go install $GOMODARGS $GCFLAGS ./... + +### builder2 stage +FROM registry.access.redhat.com/ubi9/go-toolset:latest AS builder2 +RUN go install github.com/jackc/tern@latest + +### final stage +# Build an extremely minimal container that only contains our Go application. +FROM registry.access.redhat.com/ubi9/ubi-minimal:latest + +ARG IMAGE_BUILDER_CRC_DIR=image-builder-crc +ENV IMAGE_BUILDER_CRC_DIR=$IMAGE_BUILDER_CRC_DIR + +ARG GETTING_STARTED_DIR=osbuild-getting-started +ENV GETTING_STARTED_DIR=$GETTING_STARTED_DIR + +RUN mkdir /app +RUN mkdir -p "/opt/migrate/" + +COPY --from=builder /opt/app-root/src/go/bin/image-builder /app/ +COPY --from=builder /opt/app-root/src/go/bin/image-builder-migrate-db-tern /app/ +COPY ${IMAGE_BUILDER_CRC_DIR}/distributions /app/distributions +COPY ${IMAGE_BUILDER_CRC_DIR}/internal/db/migrations-tern /app/migrations +COPY ${GETTING_STARTED_DIR}/repos/image-builder-crc/tools/openshift-startup.sh /opt/openshift-startup.sh +RUN chmod a+x /opt/openshift-startup.sh +COPY --from=builder2 /opt/app-root/src/go/bin/tern /opt/migrate/ +COPY --from=builder2 /usr/bin/dlv /usr/bin/dlv +COPY --from=builder2 /usr/share/licenses/delve /usr/share/licenses/delve +ENV TERN_MIGRATIONS_DIR=/app/migrations +EXPOSE 8086 +CMD ["/opt/openshift-startup.sh"] \ No newline at end of file diff --git a/repos/image-builder-crc/tools/openshift-startup.sh b/repos/image-builder-crc/tools/openshift-startup.sh new file mode 100644 index 0000000..425900d --- /dev/null +++ b/repos/image-builder-crc/tools/openshift-startup.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -euo pipefail + +if [[ -z "${KUBERNETES_PORT:-}" ]]; then + echo "Starting image-builder inside container..." + if [[ -n "${GODEBUG_PORT:-}" ]]; then + echo "With golang debugger enabled on port ${GODEBUG_PORT} ..." + echo "NOTE: you HAVE to attach the debugger NOW otherwise the image-builder-backend will not continue running" + /usr/bin/dlv "--listen=:${GODEBUG_PORT}" --headless=true --api-version=2 exec /app/image-builder -- -v + exit $? + fi +# we don't use cloudwatch in ephemeral environment for now +elif [[ "${CLOWDER_ENABLED:=false}" == "true" ]]; then + echo "Starting image-builder inside ephemeral environment..." + echo "Composer URL: ${COMPOSER_URL}" + echo "Composer token URL: ${COMPOSER_TOKEN_URL}" + echo "Distributions dir: ${DISTRIBUTIONS_DIR}" +else + echo "Starting image-builder inside OpenShift..." + echo "Cloudwatch: ${CW_LOG_GROUP} in ${CW_AWS_REGION}" + echo "Composer URL: ${COMPOSER_URL}" + echo "Composer token URL: ${COMPOSER_TOKEN_URL}" + echo "Distributions dir: ${DISTRIBUTIONS_DIR}" +fi + +/app/image-builder -v \ No newline at end of file diff --git a/repos/image-builder-frontend/Makefile.getting-started b/repos/image-builder-frontend/Makefile.getting-started new file mode 100644 index 0000000..1e9e5ab --- /dev/null +++ b/repos/image-builder-frontend/Makefile.getting-started @@ -0,0 +1,42 @@ +MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +GETTING_STARTED_DIR := $(patsubst %/,%,$(dir $(MAKEFILE_PATH))../../) +SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= $(GETTING_STARTED_DIR)/.. + +IMAGE_BUILDER_FRONTEND_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/image-builder-frontend + +.ONESHELL: +SHELL := /bin/bash +.SHELLFLAGS := -ec -o pipefail + +include $(IMAGE_BUILDER_FRONTEND_DIR)/Makefile + +export CONTAINER_EXECUTABLE ?= podman + +DOCKER_IMAGE := image-builder-frontend_dev +DOCKERFILE := $(GETTING_STARTED_DIR)/repos/image-builder-frontend/tools/Dockerfile.dev + +# All files to check for rebuild! +SRC_DEPS := $(shell find $(IMAGE_BUILDER_FRONTEND_DIR) -not -path './node_modules/*' \ + -not -path './dist/*' \ + -not -path './coverage/*' \ + \( -name "*.ts" \ + -or -name "*.tsx" \ + -or -name "*.yml" \ + -or -name "*.yaml" \ + -or -name "*.json" \ + -or -name "*.js" \) ) + +clean.dev: clean + rm -f container_built.info + +container_built.info: $(DOCKERFILE) $(SRC_DEPS) + $(CONTAINER_EXECUTABLE) build \ + -t $(DOCKER_IMAGE) \ + -f $(DOCKERFILE) \ + $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR) \ + --build-arg BUILD_DATE=$(shell date +%Y%m%d_%H%M%S) + echo "Container last built on" > $@ + date >> $@ + +.PHONY: container.dev +container.dev: container_built.info ## build a container with the frontend diff --git a/repos/image-builder-frontend/tools/Dockerfile.dev b/repos/image-builder-frontend/tools/Dockerfile.dev new file mode 100644 index 0000000..3866017 --- /dev/null +++ b/repos/image-builder-frontend/tools/Dockerfile.dev @@ -0,0 +1,38 @@ +FROM node:20 + +USER root +# frontend-components need a container system +# this is the reason why we run in docker -> to start podman then +RUN apt update && apt install -y podman + +WORKDIR /app + +ARG IMAGE_BUILDER_FRONTEND_DIR=image-builder-frontend +ENV IMAGE_BUILDER_FRONTEND_DIR=$IMAGE_BUILDER_FRONTEND_DIR + +ARG GETTING_STARTED_DIR=osbuild-getting-started +ENV GETTING_STARTED_DIR=$GETTING_STARTED_DIR + +ARG BACKEND_HOSTNAME=localhost +ENV BACKEND_HOSTNAME=$BACKEND_HOSTNAME + +ARG BACKEND_PORT=8086 +ENV BACKEND_PORT=$BACKEND_PORT + +COPY ${IMAGE_BUILDER_FRONTEND_DIR} . +COPY ${GETTING_STARTED_DIR}/repos/image-builder-frontend/tools/create_storage_conf.sh /app/distribution/ + + +RUN chmod a+x /app/distribution/*.sh +RUN /app/distribution/create_storage_conf.sh + +# Helper to re-run npm ci +ARG BUILD_DATE=NOT_SET +ENV BUILD_DATE=$BUILD_DATE + +RUN npm ci + +EXPOSE 8002 +EXPOSE 1337 + +CMD [ "npm", "run", "start:local" ] diff --git a/repos/image-builder-frontend/tools/create_storage_conf.sh b/repos/image-builder-frontend/tools/create_storage_conf.sh new file mode 100644 index 0000000..d4a38e8 --- /dev/null +++ b/repos/image-builder-frontend/tools/create_storage_conf.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +mkdir -p /etc/containers + +tee -a /etc/containers/storage.conf < $@ + date >> $@ + +container_composer_built.info: go.local.mod $(COMPOSER_SRC_DEPS) $(CONTAINERFILE_COMPOSER) $(CONTAINER_DEPS_COMPOSER) $(OSBUILD_CONTAINER_INDICATOR) + $(CONTAINER_EXECUTABLE) build \ + -t $(CONTAINER_IMAGE_COMPOSER) \ + -f $(CONTAINERFILE_COMPOSER) \ + --build-arg GOMODARGS="$(GOMODARGS)" \ + --build-arg GCFLAGS="$(GCFLAGS)" \ + $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR) + echo "Composer last built on" > $@ + date >> $@ + +# build a container with a worker from full source +.PHONY: container_worker.dev +container_worker.dev: $(OSBUILD_CONTAINER_INDICATOR) container_worker_built.info + +# build a container with the composer from full source +.PHONY: container_composer.dev +container_composer.dev: $(OSBUILD_CONTAINER_INDICATOR) container_composer_built.info + +container.dev: container_worker.dev container_composer.dev + +.PHONY: clean.dev +clean.dev: clean + rm -rf $(BUILDDIR)/build/ + rm -rf $(GOLANGCI_LINT_CACHE_DIR) + rm -f $(BUILDDIR)/go.local.* + rm -f $(BUILDDIR)/container_worker_built.info + rm -f $(BUILDDIR)/container_composer_built.info diff --git a/repos/osbuild-composer/tools/Dockerfile-composer.dev b/repos/osbuild-composer/tools/Dockerfile-composer.dev new file mode 100644 index 0000000..dacb1e3 --- /dev/null +++ b/repos/osbuild-composer/tools/Dockerfile-composer.dev @@ -0,0 +1,57 @@ +### builder stage +FROM registry.fedoraproject.org/fedora:latest AS builder +ENV GOBIN=/opt/app-root/src/go/bin + + +ARG OSBUILD_COMPOSER_DIR=osbuild-composer +ENV OSBUILD_COMPOSER_DIR=$OSBUILD_COMPOSER_DIR + +ARG GETTING_STARTED_DIR=osbuild-getting-started +ENV GETTING_STARTED_DIR=$GETTING_STARTED_DIR + +RUN dnf install -y gpgme-devel libassuan-devel device-mapper-devel golang + +WORKDIR /osbuild-composer +COPY ${OSBUILD_COMPOSER_DIR} /osbuild-composer +ENV GOFLAGS="-mod=vendor -tags=exclude_graphdriver_btrfs" + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" +ARG GCFLAGS="" + +RUN go install $GOMODARGS $GCFLAGS ./cmd/osbuild-composer/ + +### builder2 stage +FROM registry.access.redhat.com/ubi9/go-toolset:latest AS builder2 +RUN go install github.com/jackc/tern@latest + +### final stage +FROM osbuild_dev + +ARG OSBUILD_COMPOSER_DIR=osbuild-composer +ENV OSBUILD_COMPOSER_DIR=$OSBUILD_COMPOSER_DIR + +ARG IMAGES_DIR=images +ENV IMAGES_DIR=$IMAGES_DIR + +RUN dnf install -y python3 python3-dnf procps-ng gpgme libassuan device-mapper-libs delve +RUN mkdir -p "/usr/libexec/osbuild-composer" +RUN mkdir -p "/etc/osbuild-composer/" +RUN mkdir -p "/run/osbuild-composer/" +RUN mkdir -p "/var/cache/osbuild-composer/" +RUN mkdir -p "/var/lib/osbuild-composer/" +RUN mkdir -p "/usr/share/osbuild-composer/" +RUN mkdir -p "/opt/migrate/" +COPY --from=builder /opt/app-root/src/go/bin/osbuild-composer /usr/libexec/osbuild-composer/ +COPY ${OSBUILD_COMPOSER_DIR}/containers/osbuild-composer/entrypoint.py /opt/entrypoint.py + +COPY ${OSBUILD_COMPOSER_DIR}/pkg/jobqueue/dbjobqueue/schemas /opt/migrate/schemas +COPY --from=builder2 /opt/app-root/src/go/bin/tern /opt/migrate/ + +COPY ${OSBUILD_COMPOSER_DIR}/repositories /usr/share/osbuild-composer/repositories +COPY ${IMAGES_DIR}/data/repositories /usr/share/osbuild-composer/repositories + +EXPOSE 8008 8080 8700 +ENTRYPOINT ["python3", "/opt/entrypoint.py", "--remote-worker-api", "--composer-api", "--prometheus", "--shutdown-wait-period", "15"] diff --git a/repos/osbuild-composer/tools/Dockerfile-fauxauth.dev b/repos/osbuild-composer/tools/Dockerfile-fauxauth.dev new file mode 100644 index 0000000..f47c9a9 --- /dev/null +++ b/repos/osbuild-composer/tools/Dockerfile-fauxauth.dev @@ -0,0 +1,36 @@ +### builder stage +FROM registry.fedoraproject.org/fedora:latest AS builder +ENV GOBIN=/opt/app-root/src/go/bin + +RUN dnf install -y gpgme-devel libassuan-devel device-mapper-devel golang + + +ARG OSBUILD_COMPOSER_DIR=osbuild-composer +ENV OSBUILD_COMPOSER_DIR=$OSBUILD_COMPOSER_DIR + +WORKDIR /osbuild-composer +COPY ${OSBUILD_COMPOSER_DIR} /osbuild-composer +ENV GOFLAGS=-mod=vendor + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" + +RUN go install $GOMODARGS ./cmd/osbuild-mock-openid-provider/ + +### final stage +FROM registry.access.redhat.com/ubi9/ubi-minimal:latest + +ARG OSBUILD_COMPOSER_DIR=osbuild-composer +ENV OSBUILD_COMPOSER_DIR=$OSBUILD_COMPOSER_DIR + +RUN microdnf install -y python3 +RUN mkdir -p "/usr/libexec/osbuild-composer" +RUN mkdir -p "/etc/osbuild-composer/" + +COPY --from=builder /opt/app-root/src/go/bin/osbuild-mock-openid-provider /usr/libexec/osbuild-composer/ +COPY ${OSBUILD_COMPOSER_DIR}/containers/fauxauth/fauxauth.py /opt/fauxauth.py + +EXPOSE 8080 8080 +ENTRYPOINT "/opt/fauxauth.py" diff --git a/repos/osbuild-composer/tools/Dockerfile-worker.dev b/repos/osbuild-composer/tools/Dockerfile-worker.dev new file mode 100644 index 0000000..e73c39d --- /dev/null +++ b/repos/osbuild-composer/tools/Dockerfile-worker.dev @@ -0,0 +1,45 @@ +FROM registry.fedoraproject.org/fedora:latest AS builder +ENV GOBIN=/opt/app-root/src/go/bin + +ARG OSBUILD_COMPOSER_DIR=osbuild-composer +ENV OSBUILD_COMPOSER_DIR=${OSBUILD_COMPOSER_DIR} + +ARG GETTING_STARTED_DIR=osbuild-getting-started +ENV GETTING_STARTED_DIR=${GETTING_STARTED_DIR} + +# extra packages are needed +# to compile osbuild +RUN dnf install -y golang \ + krb5-devel \ + gpgme-devel \ + libassuan-devel + +ARG USE_BTRFS=no +RUN if [[ "$USE_BTRFS" == "yes" ]]; then dnf install -y btrfs-progs-devel device-mapper-devel; fi + +WORKDIR /osbuild-composer +COPY ${OSBUILD_COMPOSER_DIR} /osbuild-composer +ENV GOFLAGS=-mod=vendor + +ARG GOPROXY=https://proxy.golang.org,direct +RUN go env -w GOPROXY=$GOPROXY + +ARG GOMODARGS="" +ARG GCFLAGS="" + +RUN go install $GOMODARGS $GCFLAGS ./cmd/osbuild-worker + +FROM osbuild_dev + +RUN dnf install -y delve + +RUN mkdir -p "/usr/libexec/osbuild-composer" +RUN mkdir -p "/etc/osbuild-composer/" +RUN mkdir -p "/run/osbuild-composer/" +RUN mkdir -p "/var/cache/osbuild-worker/" +RUN mkdir -p "/var/lib/osbuild-composer/" +RUN mkdir -p "/var/cache/osbuild-composer/output" +COPY --from=builder /opt/app-root/src/go/bin/osbuild-worker /usr/libexec/osbuild-composer/ +COPY ${GETTING_STARTED_DIR}/repos/osbuild-composer/tools/osbuild-worker-entrypoint.sh /usr/libexec/osbuild-composer/ + +ENTRYPOINT ["/usr/libexec/osbuild-composer/osbuild-worker-entrypoint.sh"] diff --git a/repos/osbuild-composer/tools/osbuild-composer-entrypoint.py b/repos/osbuild-composer/tools/osbuild-composer-entrypoint.py new file mode 100644 index 0000000..4404b8f --- /dev/null +++ b/repos/osbuild-composer/tools/osbuild-composer-entrypoint.py @@ -0,0 +1,411 @@ +"""entrypoint - Containerized OSBuild Composer + +This provides the entrypoint for a containerized osbuild-composer image. It +spawns `osbuild-composer` on start and manages it until it exits. The main +purpose of this entrypoint is to prepare everything to be usable from within +a container. +""" + +import argparse +import contextlib +import os +import pathlib +import signal +import socket +import subprocess +import sys +import time + + +class Cli(contextlib.AbstractContextManager): + """Command Line Interface""" + + def __init__(self, argv): + self.args = None + self._argv = argv + self._exitstack = None + self._parser = None + + def _parse_args(self): + self._parser = argparse.ArgumentParser( + add_help=True, + allow_abbrev=False, + argument_default=None, + description="Containerized OSBuild Composer", + prog="container/osbuild-composer", + ) + + self._parser.add_argument( + "--shutdown-wait-period", + type=int, + default=0, + dest="shutdown_wait_period", + help="Wait period in seconds before terminating child processes", + ) + + # --[no-]composer-api + self._parser.add_argument( + "--composer-api", + action="store_true", + dest="composer_api", + help="Enable the composer-API", + ) + self._parser.add_argument( + "--no-composer-api", + action="store_false", + dest="composer_api", + help="Disable the composer-API", + ) + self._parser.add_argument( + "--prometheus", + action="store_true", + dest="prometheus", + help="Enable prometheus listener", + ) + self._parser.add_argument( + "--no-prometheus", + action="store_false", + dest="prometheus", + help="Disable prometheus listener", + ) + self._parser.add_argument( + "--composer-api-port", + type=int, + default=8080, + dest="composer_api_port", + help="Port which the composer-API listens on", + ) + self._parser.add_argument( + "--prometheus-port", + type=int, + default=8008, + dest="prometheus_port", + help="Port which prometheus listens on", + ) + self._parser.add_argument( + "--composer-api-bind-address", + type=str, + default="::", + dest="composer_api_bind_address", + help="Bind the composer API to the specified address", + ) + self._parser.add_argument( + "--prometheus-bind-address", + type=str, + default="::", + dest="prometheus_bind_address", + help="Bind the prometheus listener to the specified address", + ) + + # --[no-]local-worker-api + self._parser.add_argument( + "--local-worker-api", + action="store_true", + dest="local_worker_api", + help="Enable the local-worker-API", + ) + self._parser.add_argument( + "--no-local-worker-api", + action="store_false", + dest="local_worker_api", + help="Disable the local-worker-API", + ) + + # --[no-]remote-worker-api + self._parser.add_argument( + "--remote-worker-api", + action="store_true", + dest="remote_worker_api", + help="Enable the remote-worker-API", + ) + self._parser.add_argument( + "--no-remote-worker-api", + action="store_false", + dest="remote_worker_api", + help="Disable the remote-worker-API", + ) + self._parser.add_argument( + "--remote-worker-api-port", + type=int, + default=8700, + dest="remote_worker_api_port", + help="Port which the remote-worker API listens on", + ) + self._parser.add_argument( + "--remote-worker-api-bind-address", + type=str, + default="::", + dest="remote_worker_api_bind_address", + help="Bind the remote worker API to the specified address", + ) + + # --[no-]weldr-api + self._parser.add_argument( + "--weldr-api", + action="store_true", + dest="weldr_api", + help="Enable the weldr-API", + ) + self._parser.add_argument( + "--no-weldr-api", + action="store_false", + dest="weldr_api", + help="Disable the weldr-API", + ) + + self._parser.set_defaults( + builtin_worker=False, + composer_api=False, + prometheus=False, + local_worker_api=False, + remote_worker_api=False, + weldr_api=False, + ) + + return self._parser.parse_args(self._argv[1:]) + + def __enter__(self): + self._exitstack = contextlib.ExitStack() + self.args = self._parse_args() + return self + + def __exit__(self, exc_type, exc_value, exc_tb): + self._exitstack.close() + self._exitstack = None + + def _prepare_sockets(self): + # Prepare all the API sockets that osbuild-composer expectes, and make + # sure to pass them according to the systemd socket-activation API. + # + # Note that we rely on this being called early, so we get the correct + # FD numbers assigned. We need FD-#3 onwards for compatibility with + # socket activation (because python `subprocess.Popen` does not support + # renumbering the sockets we pass down). + + index = 3 + sockets = [] + names = [] + + # osbuild-composer.socket + if self.args.weldr_api: + print("Create weldr-api socket", file=sys.stderr) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.bind("/run/weldr/api.socket") + sock.listen() + sockets.append(sock) + + names.append("osbuild-composer.socket") + + assert(sock.fileno() == index) + index += 1 + + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.bind("/run/cloudapi/api.socket") + sock.listen() + sockets.append(sock) + + names.append("osbuild-composer.socket") + + assert(sock.fileno() == index) + index += 1 + + # osbuild-composer-api.socket + if self.args.composer_api: + print("Create composer-api socket on port {}".format(self.args.composer_api_port) , file=sys.stderr) + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind((self.args.composer_api_bind_address, self.args.composer_api_port)) + sock.listen() + sockets.append(sock) + names.append("osbuild-composer-api.socket") + + assert(sock.fileno() == index) + index += 1 + + # osbuild-composer-prometheus.socket + if self.args.prometheus: + print("Create prometheus socket on port {}".format(self.args.prometheus_port), file=sys.stderr) + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind((self.args.prometheus_bind_address, self.args.prometheus_port)) + sock.listen() + sockets.append(sock) + names.append("osbuild-composer-prometheus.socket") + + assert(sock.fileno() == index) + index += 1 + + # osbuild-local-worker.socket + if self.args.local_worker_api: + print("Create local-worker-api socket", file=sys.stderr) + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.bind("/run/osbuild-composer/job.socket") + sock.listen() + sockets.append(sock) + names.append("osbuild-local-worker.socket") + + assert(sock.fileno() == index) + index += 1 + + # osbuild-remote-worker.socket + if self.args.remote_worker_api: + print(f"Create remote-worker-api socket on address [{self.args.remote_worker_api_bind_address}]:{self.args.remote_worker_api_port}", file=sys.stderr) + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._exitstack.enter_context(contextlib.closing(sock)) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + sock.bind((self.args.remote_worker_api_bind_address, self.args.remote_worker_api_port)) + sock.listen(256) + sockets.append(sock) + names.append("osbuild-remote-worker.socket") + + assert(sock.fileno() == index) + index += 1 + + # Prepare FD environment for the child process. + os.environ["LISTEN_FDS"] = str(len(sockets)) + os.environ["LISTEN_FDNAMES"] = ":".join(names) + + return sockets + + @staticmethod + def _spawn_worker(): + cmd = [ + "/usr/libexec/osbuild-composer/osbuild-worker", + "-unix", + "/run/osbuild-composer/job.socket", + ] + + env = os.environ.copy() + env["CACHE_DIRECTORY"] = "/var/cache/osbuild-worker" + env["STATE_DIRECTORY"] = "/var/lib/osbuild-worker" + + return subprocess.Popen( + cmd, + cwd="/", + env=env, + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + ) + + @staticmethod + def _spawn_composer(sockets): + cmd = [ + "/usr/libexec/osbuild-composer/osbuild-composer", + "-verbose", + ] + + # Prepare the environment for osbuild-composer. Note that we cannot use + # the `env` parameter of `subprocess.Popen()`, because it conflicts + # with the `preexec_fn=` parameter. Therefore, we have to modify the + # caller's environment. + os.environ["CACHE_DIRECTORY"] = "/var/cache/osbuild-composer" + os.environ["STATE_DIRECTORY"] = "/var/lib/osbuild-composer" + + # We need to set `LISTEN_PID=` to the target PID. The only way python + # allows us to do this is to hook into `preexec_fn=`, which is executed + # by `subprocess.Popen()` after forking, but before executing the new + # executable. + preexec_setenv = lambda: os.putenv("LISTEN_PID", str(os.getpid())) + + return subprocess.Popen( + cmd, + cwd="/usr/libexec/osbuild-composer", + stdin=subprocess.DEVNULL, + stderr=subprocess.STDOUT, + pass_fds=[sock.fileno() for sock in sockets], + preexec_fn=preexec_setenv, + ) + + def run(self): + """Program Runtime""" + + proc_composer = None + proc_worker = None + res = 0 + sockets = self._prepare_sockets() + + def handler(signum, frame): + if self.args.shutdown_wait_period: + time.sleep(self.args.shutdown_wait_period) + proc_composer.terminate() + proc_worker.terminate() + + signal.signal(signal.SIGTERM, handler) + + liveness = pathlib.Path('/tmp/osbuild-composer-live') + + liveness.touch() + + try: + should_launch_composer = any([self.args.weldr_api, self.args.composer_api, self.args.local_worker_api, self.args.remote_worker_api]) + if self.args.builtin_worker or not should_launch_composer: + if not should_launch_composer: + print(f"NOTE: launching worker only - no API for composer enabled") + proc_worker = self._spawn_worker() + + if should_launch_composer: + proc_composer = self._spawn_composer(sockets) + + debug_port = os.environ.get('GODEBUG_PORT') + debugger = None + + if debug_port: + # only debug one - either composer or worker if there is no composer + child_pid = proc_composer.pid if proc_composer else proc_worker.pid + debug_target_name = "image-builder-composer" if proc_composer else "image-builder-worker" + + debugger_cmd = [ + "/usr/bin/dlv", + "attach", + "--headless=true", + "--api-version", "2", + "--listen", f":{debug_port}", + str(child_pid), + "/usr/libexec/osbuild-composer/osbuild-composer" + ] + + print(f"NOTE: you HAVE to attach the debugger NOW otherwise { debug_target_name } " + f"will not continue running", file=sys.stderr) + debugger = subprocess.Popen(debugger_cmd) + + if proc_composer: + res = proc_composer.wait() + + if proc_worker: + if proc_composer: + proc_worker.terminate() + proc_worker.wait() + + if debugger: + debugger.wait() + + except KeyboardInterrupt: + if proc_composer: + proc_composer.terminate() + res = proc_composer.wait() + if proc_worker: + proc_worker.terminate() + proc_worker.wait() + except: + if proc_worker: + proc_worker.kill() + if proc_composer: + proc_composer.kill() + raise + finally: + liveness.unlink() + + return res + + +if __name__ == "__main__": + with Cli(sys.argv) as global_main: + sys.exit(global_main.run()) diff --git a/repos/osbuild-composer/tools/osbuild-worker-entrypoint.sh b/repos/osbuild-composer/tools/osbuild-worker-entrypoint.sh new file mode 100755 index 0000000..b3f1a15 --- /dev/null +++ b/repos/osbuild-composer/tools/osbuild-worker-entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +APP="/usr/libexec/osbuild-composer/osbuild-worker" +APP_ARGS="${WORKER_ARGS:-}" + +if [[ -n "${GODEBUG_PORT:-}" ]]; then + echo "With golang debugger enabled on port ${GODEBUG_PORT} ..." + echo "NOTE: you HAVE to attach the debugger NOW otherwise the osbuild-worker will not continue running" + /usr/bin/dlv "--listen=:${GODEBUG_PORT}" --headless=true --api-version=2 exec ${APP} -- "${APP_ARGS}" + exit $? +fi + +${APP} "${APP_ARGS}" diff --git a/repos/osbuild/Makefile.getting-started b/repos/osbuild/Makefile.getting-started new file mode 100644 index 0000000..a48e0f3 --- /dev/null +++ b/repos/osbuild/Makefile.getting-started @@ -0,0 +1,72 @@ + +MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +GETTING_STARTED_DIR := $(dir $(MAKEFILE_PATH))../.. +SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= $(GETTING_STARTED_DIR)/.. + +OSBUILD_DIR ?= $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)/osbuild + +include $(OSBUILD_DIR)/Makefile + +# List of all files which trigger a rebuild of the +# development targets and container + +# Directories to search +SRC_DEPS_DIRS := assemblers devices inputs mounts osbuild runners sources stages tools + +# Function to search for Python files in a directory +define find_python_files +$(shell find $(1) -type f -exec file {} \; | grep "Python script" | grep -Eo "^[^:]+") +endef + +# All files to check for rebuild! +SRC_DEPS := $(shell find $(OSBUILD_DIR) -name "*.py") \ + $(foreach dir,$(SRC_DEPS_DIRS),$(call find_python_files,$(OSBUILD_DIR)/$(dir))) \ + $(shell find $(OSBUILD_DIR)/schemas -name "*.json") + +CONTAINER_EXECUTABLE ?= podman +CONTAINER_IMAGE := osbuild_dev +CONTAINERFILE := $(GETTING_STARTED_DIR)/repos/osbuild/tools/Containerfile.dev + +RPM_SPECFILE_DEV=$(OSBUILD_DIR)/rpmbuild/SPECS/osbuild-SRC.spec + +# building rpm for development with a constant name +# to avoid any semblance to a "release rpm" +$(RPM_SPECFILE_DEV): osbuild.spec + mkdir -p $(OSBUILD_DIR)/rpmbuild/SPECS + echo "%global commit SRC" > $@ + echo "%define _rpmfilename %%{NAME}.rpm" >> $@ + echo "%global source_date_epoch_from_changelog 1" >> $@ + echo "%global clamp_mtime_to_source_date_epoch 1" >> $@ + cat $< >> $@ + +RPM_DEV_FILES := osbuild.rpm \ + osbuild-depsolve-dnf.rpm \ + osbuild-luks2.rpm \ + osbuild-ostree.rpm \ + osbuild-selinux.rpm \ + osbuild-tools.rpm \ + python3-osbuild.rpm + +RPM_DEV := $(addprefix $(OSBUILD_DIR)/rpmbuild/RPMS/,$(RPM_DEV_FILES)) + +.PHONY: rpm.dev +rpm.dev: $(RPM_DEV) + +$(RPM_DEV): $(RPM_SPECFILE_DEV) $(SRC_DEPS) + export SOURCE_DATE_EPOCH=$(shell stat -c %Y $(RPM_SPECFILE_DEV)) + rpmbuild -bb --build-in-place \ + --define "_topdir $(OSBUILD_DIR)/rpmbuild" \ + $(RPM_SPECFILE_DEV) + +clean.dev: clean + rm -rf $(OSBUILD_DIR)/rpmbuild + rm -f $(OSBUILD_DIR)/container_built.info + + +container_built.info: $(CONTAINERFILE) osbuild.spec $(SRC_DEPS) + $(CONTAINER_EXECUTABLE) build -t $(CONTAINER_IMAGE) -f $(CONTAINERFILE) $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR) + echo "Container last built on" > $@ + date >> $@ + +.PHONY: container.dev +container.dev: container_built.info \ No newline at end of file diff --git a/repos/osbuild/tools/Containerfile.dev b/repos/osbuild/tools/Containerfile.dev new file mode 100644 index 0000000..d33f244 --- /dev/null +++ b/repos/osbuild/tools/Containerfile.dev @@ -0,0 +1,36 @@ +FROM registry.fedoraproject.org/fedora:latest + +ARG OSBUILD_DIR=osbuild +ENV OSBUILD_DIR=${OSBUILD_DIR} + +ARG GETTING_STARTED_DIR=osbuild-getting-started +ENV GETTING_STARTED_DIR=${GETTING_STARTED_DIR} + +RUN dnf install -y @rpm-development-tools git + +RUN mkdir /prepare +WORKDIR /prepare + +COPY ${OSBUILD_DIR}/osbuild.spec . + +RUN dnf -y builddep ./osbuild.spec + +RUN pip install pydevd_pycharm debugpy + +RUN mkdir /app +WORKDIR /app + +COPY ${GETTING_STARTED_DIR}/repos/osbuild/Makefile.getting-started Makefile.getting-started + +COPY ${OSBUILD_DIR} /app + +COPY ${GETTING_STARTED_DIR}/repos/osbuild/tools/main_cli_dev.py /app/osbuild/ +COPY ${GETTING_STARTED_DIR}/repos/osbuild/tools/Containerfile.dev.patch /app/tools/ +RUN git apply tools/Containerfile.dev.patch + +# OSBUILD_DIR is different in the container than on the host +RUN OSBUILD_DIR=/app make -f Makefile.getting-started rpm.dev + +RUN dnf install -y ./rpmbuild/RPMS/*.rpm + +ENTRYPOINT [ "/usr/bin/osbuild" ] diff --git a/repos/osbuild/tools/Containerfile.dev.patch b/repos/osbuild/tools/Containerfile.dev.patch new file mode 100644 index 0000000..007d41f --- /dev/null +++ b/repos/osbuild/tools/Containerfile.dev.patch @@ -0,0 +1,13 @@ +diff --git a/setup.py b/setup.py +index cedc5e5d..b5953aa7 100644 +--- a/setup.py ++++ b/setup.py +@@ -18,7 +18,7 @@ setuptools.setup( + ], + entry_points={ + "console_scripts": [ +- "osbuild = osbuild.main_cli:osbuild_cli" ++ "osbuild = osbuild.main_cli_dev:osbuild_dev" + ] + }, + scripts=[ diff --git a/repos/osbuild/tools/main_cli_dev.py b/repos/osbuild/tools/main_cli_dev.py new file mode 100644 index 0000000..49c8256 --- /dev/null +++ b/repos/osbuild/tools/main_cli_dev.py @@ -0,0 +1,28 @@ +"""Development entry point for osbuild + +This is used while building a container with `make container.dev` +This allows to attach the debugger even when more services +of the stack are running in parallel +""" +import os + +import debugpy +import pydevd_pycharm + +from osbuild.main_cli import osbuild_cli + + +def osbuild_dev(): + debug_port = os.getenv("OSBUILD_PYCHARM_DEBUG_PORT") + if debug_port: + debug_host = os.getenv("OSBUILD_PYCHARM_DEBUG_HOST", 'host.docker.internal') + print("Connecting to debugger...") + pydevd_pycharm.settrace(debug_host, port=int(debug_port), stdoutToServer=True, stderrToServer=True) + + debug_port = os.getenv("OSBUILD_DEBUGPY_DEBUG_PORT") + if debug_port: + debug_host = os.getenv("OSBUILD_DEBUGPY_DEBUG_HOST", 'host.docker.internal') + print("Connecting to debugger...") + debugpy.listen(debug_host, port=int(debug_port)) + + osbuild_cli() diff --git a/service/README.md b/service/README.md index 94dc66e..8f495e8 100644 --- a/service/README.md +++ b/service/README.md @@ -1,24 +1,45 @@ # devtools Development Tools for Image Builder +Here are some tools of "Image Builder" to start up a development environment for the +[hosted deployment](https://osbuild.org/docs/hosted/architecture/). +The reason for all the `sudo` below is that the stack has to perform may operations +with loop devices etc. where root privileges are required. ## Setup - -To start local development, first clone the image builder stack: + +To start local development, first clone the image builder stack and all it's dependent source. ```bash +git clone git@github.com:osbuild/images.git git clone git@github.com:osbuild/osbuild-composer.git -git clone git@github.com:osbuild/image-builder.git git clone git@github.com:osbuild/osbuild-getting-started.git +git clone git@github.com:osbuild/osbuild.git +git clone git@github.com:osbuild/pulp-client.git +``` + +For starting the service you need additionally: +```shell +git clone git@github.com:osbuild/community-gateway.git +git clone git@github.com:osbuild/image-builder-frontend.git +git clone git@github.com:osbuild/image-builder.git +``` + +For starting the on-prem version you need additionally: +```shell +git clone git@github.com:osbuild/weldr-client.git ``` -The folder structure should look like: +The folder structure should look like this: ``` . -├── image-builder +├── images +├── osbuild +├── osbuild-composer ├── osbuild-getting-started -└── osbuild-composer +… +└── pulp-client ``` Secondly redirect a stage and prod domains to localhost. If you are outside @@ -29,24 +50,6 @@ echo "127.0.0.1 prod.foo.redhat.com" >> /etc/hosts echo "127.0.0.1 stage.foo.redhat.com" >> /etc/hosts ``` -Lastly, you will need to ensure that the folders for the config volume and the local S3 bucket -have been created: - -> This can be modified in the volumes -> section of the `docker-compose.yaml` -> file - -```bash -sudo mkdir -p /scratch/podman/image-builder-config -``` - -> The name of the bucket used is `service`, -> you can change this in the `.env` file - -```bash -sudo mkdir -p /scratch/data/s3/service -``` - ## Docker compose notes As per the [docker compose cli](https://docs.docker.com/compose/reference/) docs, the new syntax for running docker compose changed from @@ -55,7 +58,7 @@ command. ## Upload Targets -Upload targets need to be configued for the Image Builder backend to upload successfully. +Upload targets need to be configured for the Image Builder backend to upload successfully. This stack comes pre-configured with a generic S3 bucket which can be accessed at: `http://localhost:9000` @@ -95,49 +98,60 @@ The config variables for the worker can be found [here](https://github.com/osbui *NOTE:* If you change the config files, you will either need to modify the worker config in the `/scratch/podman/image-builder-config` file and restart the containers. Alternatively, you will need to remove the named volume and rebuild the config container. The steps for this are as follows: -Run the following from **this directory**. +Run the following from the main **osbuild-getting-started directory** (one level above where this README.md is). + +```bash +make help +``` + - stop the containers and remove volumes ```bash -docker compose down -v +make prune-service ``` -- rebuild the config container +- rebuild containers ```bash -docker compose build config +make service-containers ``` - start the containers again ```bash -docker compose up +make run-service ``` -## Run the backend +## Run the frontend separately -To build the containers run the following command from **this directory**: +The command above (`make run-service`) also starts the frontend by bind-mounting the source +from your computer, so hot-reload of `npm` should work. +If you want to run the frontend yourself (see the [README.md](https://github.com/osbuild/image-builder-frontend/blob/main/README.md) there) +you can use the make target: ```bash -docker compose build +make run-service-no-fronted ``` -To run the containers: +You have to have a "staging account" in order to run this setup (although it's local). +Check if you can log in at https://console.stage.redhat.com/ -```bash -docker compose up -``` +Access the service through the GUI: +[https://stage.foo.redhat.com:1337/beta/insights/image-builder](https://stage.foo.redhat.com:1337/beta/insights/image-builder), or +directly through the API: +[https://stage.foo.redhat.com:1337/docs/api/image-builder](https://stage.foo.redhat.com:1337/docs/api/image-builder). -## Run the frontend +## Debugging +When you want to use your IDE (Integrated Development Environment) to debug the go-code you can check out the +`GODEBUG_PORT` variables in [docker-composer.yml](./docker-composer.yml) and enable debugging for the supported +layers. -The frontend has been removed as a container in favour of running it separately in order to leverage hot-reloading -capabilities. In order to run the frontend with the backend you can run the following command: +## Known Problems -```bash -npm run devel +If you encounter problems with the certificate when you browse to +[https://stage.foo.redhat.com:1337/preview/insights/image-builder](https://stage.foo.redhat.com:1337/preview/insights/image-builder) +you might need to rebuild the certificates with: +``` +make wipe-config ``` -Access the service through the GUI: -[https://stage.foo.redhat.com:1337/beta/insights/image-builder](https://stage.foo.redhat.com:1337/beta/insights/image-builder), or -directly through the API: -[https://stage.foo.redhat.com:1337/docs/api/image-builder](https://stage.foo.redhat.com:1337/docs/api/image-builder). diff --git a/service/config/Dockerfile-config b/service/config/Dockerfile-config index d876df4..fbd2664 100644 --- a/service/config/Dockerfile-config +++ b/service/config/Dockerfile-config @@ -1,15 +1,21 @@ FROM fedora:38 RUN dnf install -y openssl -RUN mkdir -p /config +RUN mkdir -p /config_build COPY ./tools/gen-certs.sh . COPY ./config/x509/openssl.cnf . -COPY ./config/composer/acl.yml /config/. -COPY ./config/composer/osbuild-composer.toml /config/. -COPY ./config/worker/osbuild-worker.toml /config/. -COPY ./config/worker/s3-credentials /config/. -COPY ./config/worker/secret /config/. -COPY ./config/backend/quotas.json /config/. - -RUN ./gen-certs.sh ./openssl.cnf /config /config/ca +COPY ./config/composer/acl.yml /config_build/. +COPY ./config/composer/osbuild-composer.toml /config_build/. +COPY ./config/worker/osbuild-worker.toml /config_build/. +COPY ./config/worker/s3-credentials /config_build/. +COPY ./config/worker/secret /config_build/. +COPY ./config/backend/quotas.json /config_build + +# Helper to re-run gen-certs.sh +ARG CONFIG_BUILD_DATE=NOT_SET +ENV CONFIG_BUILD_DATE=$CONFIG_BUILD_DATE + +RUN ./gen-certs.sh ./openssl.cnf /config_build /config_build/ca + +CMD cp -r /config_build/* /config/ && echo "Done. Provided the config to all other containers." diff --git a/service/config/Dockerfile-config-onprem b/service/config/Dockerfile-config-onprem new file mode 100644 index 0000000..f351d05 --- /dev/null +++ b/service/config/Dockerfile-config-onprem @@ -0,0 +1,21 @@ +FROM fedora:38 + +RUN dnf install -y openssl +RUN mkdir -p /config_build + +COPY ./tools/gen-certs.sh . +COPY ./config/x509/openssl.cnf . +COPY ./config/composer/acl.yml /config_build/. +COPY ./config/composer/osbuild-composer-onprem.toml /config_build/osbuild-composer.toml +# COPY ./config/worker/osbuild-worker-onprem.toml /config_build/osbuild-worker.toml +COPY ./config/worker/s3-credentials /config_build/. +COPY ./config/worker/secret /config_build/. +COPY ./config/backend/quotas.json /config_build + +# Helper to re-run gen-certs.sh +ARG CONFIG_BUILD_DATE=NOT_SET +ENV CONFIG_BUILD_DATE=$CONFIG_BUILD_DATE + +RUN ./gen-certs.sh ./openssl.cnf /config_build /config_build/ca + +CMD cp -r /config_build/* /config/ && cp -r /repositories /config/ && echo "Done. Provided the config to all other containers." diff --git a/service/config/backend/quotas.json b/service/config/backend/quotas.json index c3a2a04..d0b5af9 100644 --- a/service/config/backend/quotas.json +++ b/service/config/backend/quotas.json @@ -3,4 +3,4 @@ "quota":200, "slidingWindow":1209600000000000 } -} + } diff --git a/service/config/cli/data/blueprint.toml b/service/config/cli/data/blueprint.toml new file mode 100644 index 0000000..9971ac3 --- /dev/null +++ b/service/config/cli/data/blueprint.toml @@ -0,0 +1,9 @@ +name = "example-image" +description = "An example image for osbuild-getting-started." +version = "0.0.1" +modules = [] +groups = [] + +[[packages]] +name = "tmux" +version = "*" diff --git a/service/config/composer/osbuild-composer-onprem.toml b/service/config/composer/osbuild-composer-onprem.toml new file mode 100644 index 0000000..6cfff0a --- /dev/null +++ b/service/config/composer/osbuild-composer-onprem.toml @@ -0,0 +1,7 @@ +[koji] +allowed_domains = [ "localhost", "client.osbuild.org" ] +ca = "/etc/osbuild-composer/ca-crt.pem" + +[worker] +allowed_domains = [ "localhost", "worker.osbuild.org", "172.31.0.1" ] +ca = "/etc/osbuild-composer/ca-crt.pem" diff --git a/service/docker-compose-onprem.yml b/service/docker-compose-onprem.yml new file mode 100644 index 0000000..410a859 --- /dev/null +++ b/service/docker-compose-onprem.yml @@ -0,0 +1,155 @@ +networks: + onpremnet: + ipam: + driver: default + config: + - subnet: 172.32.0.0/16 + +volumes: + config_volume: + driver_opts: + type: none + # this can be modified as desired + device: ${SCRATCH_DIR:-/scratch}/image-builder-config + o: bind + weldr_volume: + driver_opts: + type: none + # this can be modified as desired + device: ${SCRATCH_DIR:-/scratch}/weldr + o: bind + cloudapi_volume: + driver_opts: + type: none + # this can be modified as desired + device: ${SCRATCH_DIR:-/scratch}/cloudapi + o: bind + dnfjson_volume: + driver_opts: + type: none + # this can be modified as desired + device: ${SCRATCH_DIR:-/scratch}/dnf-json + o: bind + +services: + config: + image: local/config + build: + context: . + dockerfile: ./config/Dockerfile-config-onprem + volumes: + - config_volume:/config:z + - ../../osbuild-composer/repositories:/repositories:z + + composer: + image: osbuild-composer_dev + entrypoint: + [ + "python3", + "/opt/entrypoint.py", + "--weldr-api", + "--remote-worker-api", + "--composer-api", + "--composer-api-port", "8000", + ] + healthcheck: + test: ["CMD-SHELL", "curl -k -s --cert /etc/osbuild-composer/worker-crt.pem --key /etc/osbuild-composer/worker-key.pem -o /dev/null https://localhost:8700/api/worker/v1/status"] + interval: 2s + timeout: 2s + retries: 100 + environment: + - PGHOST=postgres_composer + - PGPORT=5432 + - PGDATABASE=postgres + - PGUSER=postgres + - PGPASSWORD=postgres + - PGSSLMODE=disable + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7730 + volumes: + - config_volume:/etc/osbuild-composer + - weldr_volume:/run/weldr:rw,z + - cloudapi_volume:/run/cloudapi:rw,z + - dnfjson_volume:/run/osbuild-dnf-json:rw,z + ports: + - 8080:8080 + - 8700:8700 + - 7730:7730 + depends_on: + config: + condition: service_completed_successfully + postgres_composer: + condition: service_healthy + + networks: + onpremnet: + ipv4_address: 172.32.0.10 + + worker: + image: osbuild-worker_dev + volumes: + - config_volume:/etc/osbuild-composer:z + - config_volume:/etc/osbuild-worker:z + environment: + - CACHE_DIRECTORY=/var/cache/osbuild-composer + - WORKER_ARGS=composer:8700 + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7740 + # only enable the OSBUILD_PYTHON_DEBUG_PORT if you want live debugging. + # you HAVE to attach a pydev debugger to continue execution (check the logs after starting a build) + # it will connect to your localhost:5679 and expects a python debug server there + # - OSBUILD_PYTHON_DEBUG_PORT=5679 + ports: + - 7740:7740 + privileged: true + cap_add: + - CAP_MKNOD + - SYS_ADMIN + - NET_ADMIN + depends_on: + config: + condition: service_completed_successfully + composer: + condition: service_healthy + restart: on-failure + networks: + onpremnet: + ipv4_address: 172.32.0.20 + extra_hosts: + - "host.docker.internal:host-gateway" + + + postgres_composer: + image: docker.io/postgres:10.5 + healthcheck: + test: [ "CMD", "pg_isready", "-U", "postgres", "-d", "postgres" ] + interval: 2s + timeout: 2s + retries: 10 + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + volumes: + - ../../osbuild-composer/pkg/jobqueue/dbjobqueue/schemas/:/docker-entrypoint-initdb.d/:z + networks: + onpremnet: + ipv4_address: 172.32.0.30 + + cli: + image: osbuild-cli_dev + volumes: + - config_volume:/etc/osbuild-composer:z + - weldr_volume:/run/weldr:rw,z + - dnfjson_volume:/run/osbuild-dnf-json:rw,z + - ./config/cli/data:/data + #environment: + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7750 + networks: + onpremnet: + ipv4_address: 172.32.0.40 + entrypoint: + [ "/usr/bin/true" ] # will be started interactively from Makefile diff --git a/service/docker-compose.yml b/service/docker-compose.yml index 01af030..86cb958 100644 --- a/service/docker-compose.yml +++ b/service/docker-compose.yml @@ -1,18 +1,16 @@ -version: '3.9' - networks: - net: + servicenet: ipam: driver: default config: - subnet: 172.31.0.0/16 volumes: - config: + config_volume: driver_opts: type: none # this can be modified as desired - device: /scratch/podman/image-builder-config + device: ${SCRATCH_DIR:-/scratch}/image-builder-config o: bind services: @@ -22,14 +20,27 @@ services: context: . dockerfile: ./config/Dockerfile-config volumes: - - config:/config + - config_volume:/config:z - ../../osbuild-composer/repositories:/config/repositories:z + postgres_composer: + image: docker.io/postgres:16.8 + healthcheck: + test: [ "CMD", "pg_isready", "-h", "localhost", "-U", "postgres", "-d", "postgres" ] + interval: 2s + timeout: 2s + retries: 10 + environment: + - POSTGRES_USER=postgres + - POSTGRES_PASSWORD=postgres + volumes: + - ../../osbuild-composer/pkg/jobqueue/dbjobqueue/schemas/:/docker-entrypoint-initdb.d/:z + networks: + servicenet: + ipv4_address: 172.31.0.80 + composer: - image: local/osbuild-composer - build: - context: ../../osbuild-composer - dockerfile: ./distribution/Dockerfile-ubi + image: osbuild-composer_dev entrypoint: [ "python3", @@ -37,46 +48,77 @@ services: "--remote-worker-api", "--composer-api" ] + healthcheck: + test: ["CMD-SHELL", "if curl -k -s -o /dev/null -w '%{http_code}' https://localhost:8700/ | grep -q 401; then exit 0; else exit 1; fi"] + interval: 2s + timeout: 2s + retries: 100 + environment: + - PGHOST=postgres_composer + - PGPORT=5432 + - PGDATABASE=postgres + - PGUSER=postgres + - PGPASSWORD=postgres + - PGSSLMODE=disable + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7730 volumes: - - config:/etc/osbuild-composer + - config_volume:/etc/osbuild-composer ports: - 8080:8080 - 8700:8700 + - 7730:7730 depends_on: - - config + config: + condition: service_completed_successfully + postgres_composer: + condition: service_healthy + networks: - net: + servicenet: ipv4_address: 172.31.0.10 worker: - image: local/osbuild-worker - build: - context: ../../osbuild-composer - dockerfile: ./distribution/Dockerfile-worker - # override the entrypoint to specify composer hostname and port - entrypoint: [ "/usr/libexec/osbuild-composer/osbuild-worker", "composer:8700" ] + image: osbuild-worker_dev volumes: - - config:/etc/osbuild-composer - - config:/etc/osbuild-worker + - config_volume:/etc/osbuild-composer:z + - config_volume:/etc/osbuild-worker:z environment: - CACHE_DIRECTORY=/var/cache/osbuild-composer + - WORKER_ARGS=composer:8700 + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7740 + # only enable the OSBUILD_PYCHARM_DEBUG_PORT if you want live debugging. + # you HAVE to attach a pydev debugger to continue execution (check the logs after starting a build) + # it will connect to your localhost:5679 and expects a python debug server there + # - OSBUILD_PYCHARM_DEBUG_PORT=5679 + # or use OSBUILD_DEBUGPY_DEBUG_PORT for e.g. vscode + # - OSBUILD_DEBUGPY_DEBUG_PORT=5679 + ports: + - 7740:7740 privileged: true cap_add: - CAP_MKNOD - - SYS_ADMIN - NET_ADMIN + - SYS_ADMIN depends_on: - - config - - composer + config: + condition: service_completed_successfully + composer: + condition: service_healthy restart: on-failure networks: - net: + servicenet: ipv4_address: 172.31.0.20 + extra_hosts: + - "host.docker.internal:host-gateway" - postgres: - image: docker.io/postgres:10.5 + postgres_image_builder_crc: + image: docker.io/postgres:16.8 healthcheck: - test: [ "CMD", "pg_isready", "-U", "postgres", "-d", "postgres" ] + test: [ "CMD", "pg_isready", "-h", "localhost", "-U", "postgres", "-d", "postgres" ] interval: 2s timeout: 2s retries: 10 @@ -84,30 +126,28 @@ services: - POSTGRES_USER=postgres - POSTGRES_PASSWORD=postgres volumes: - - ../../image-builder/internal/db/migrations-tern/:/docker-entrypoint-initdb.d/:Z + - ../../image-builder/internal/db/migrations-tern/:/docker-entrypoint-initdb.d/:z networks: - net: + servicenet: ipv4_address: 172.31.0.30 - backend: - image: local/image-builder - build: - context: ../../image-builder - dockerfile: ./distribution/Dockerfile-ubi + image_builder_crc: + image: image-builder-crc_dev ports: - 8086:8086 + - 7720:7720 healthcheck: test: [ "CMD", "curl", "localhost:8086/status" ] interval: 2s timeout: 2s - retries: 10 + retries: 100 volumes: - - config:/etc/image-builder + - config_volume:/etc/image-builder environment: - - LISTEN_ADDRESS=backend:8086 + - LISTEN_ADDRESS=image_builder_crc:8086 - LOG_LEVEL=DEBUG - ALLOWED_ORG_IDS=* - - PGHOST=postgres + - PGHOST=postgres_image_builder_crc - PGPORT=5432 - PGDATABASE=postgres - PGUSER=postgres @@ -119,48 +159,75 @@ services: - COMPOSER_CA_PATH=/etc/image-builder/ca-crt.pem - DISTRIBUTIONS_DIR=/app/distributions - QUOTA_FILE=/etc/image-builder/quotas.json + # only enable the GODEBUG_PORT if you want live debugging. + # you HAVE to attach a golang debugger to continue execution (check the logs after startup) + # - GODEBUG_PORT=7720 depends_on: - - config - - composer - - postgres + config: + condition: service_completed_successfully + composer: + condition: service_healthy + postgres_image_builder_crc: + condition: service_healthy restart: on-failure networks: - net: + servicenet: ipv4_address: 172.31.0.40 fauxauth: - image: local/osbuild-fauxauth + image: local/osbuild-fauxauth_dev build: - context: ../../osbuild-composer - dockerfile: ./distribution/Dockerfile-fauxauth + context: ../../ + dockerfile: ./osbuild-getting-started/repos/osbuild-composer/tools/Dockerfile-fauxauth.dev + args: + GOMODARGS: "-modfile=go.local.mod" entrypoint: [ "/opt/fauxauth.py", "-a", "0.0.0.0", "-p", "8888" ] volumes: - - config:/etc/osbuild-composer + - config_volume:/etc/osbuild-composer ports: - 8888:8888 depends_on: - config networks: - net: + servicenet: ipv4_address: 172.31.0.50 minio: - image: minio/minio:latest + image: docker.io/minio/minio:latest volumes: # this can be modified as desired - - /scratch/data:/data:z + - ${SCRATCH_DIR:-/scratch}/data/s3:/data:z command: server /data --console-address ":9090" + working_dir: /data environment: - - MINIO_ROOT_USER=admin - - MINIO_ROOT_PASSWORD=password42 + - MINIO_ROOT_USER=admin + - MINIO_ROOT_PASSWORD=password42 ports: - 9000:9000 - - 9090:9090 + - 9091:9090 healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ] interval: 30s timeout: 20s retries: 3 networks: - net: + servicenet: ipv4_address: 172.31.0.60 + + frontend: + image: image-builder-frontend_dev + security_opt: + - seccomp=unconfined + environment: + - BACKEND_HOSTNAME=image_builder_crc + ports: + - 1337:1337 + - 8002:8002 + volumes: + - ../../image-builder-frontend/api:/app/api:z + - ../../image-builder-frontend/config:/app/config:z + - ../../image-builder-frontend/src:/app/src:z + networks: + servicenet: + ipv4_address: 172.31.0.70 + diff --git a/service/tools/gen-certs.sh b/service/tools/gen-certs.sh index c603279..9b7a95a 100755 --- a/service/tools/gen-certs.sh +++ b/service/tools/gen-certs.sh @@ -1,4 +1,13 @@ #!/bin/bash + +if [ -f /config ] ; then + echo "ERROR: /config should not be a file. Your setup seems to be broken!" + echo "Please check your volume mounts of the container runtime (podman/docker)" + exit 1 +fi + +ls -la /config/ + if (( $# != 3 )); then echo "Usage: $0 " echo diff --git a/tools/git_stack.sh b/tools/git_stack.sh new file mode 100755 index 0000000..4d40e75 --- /dev/null +++ b/tools/git_stack.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +# repos to respect +BASEDIR=$(dirname $(readlink -f $0))/../.. +REPOS="osbuild-getting-started osbuild osbuild-composer images image-builder-crc image-builder-frontend weldr-client" +REPOS="$REPOS pulp-client community-gateway" + +ARGS=${*:-status --branch --short} + +NUM="$(echo $REPOS | wc -w)" +I=0 +running_jobs=0 +MAX_PARALLEL=4 +tmp_files=() + +RED='\033[0;31m' +NO_COLOR='\033[0m' + +function run_git_command { + local dir=$1 + local args=$2 + local tmp_file=$3 + git -c color.status=always -C "$dir" $args >> "$tmp_file" 2>&1 || echo -e "${RED}ERROR code: $?${NO_COLOR}" >> "$tmp_file" & +} + +for D in $REPOS; do + I=$(( $I + 1 )) + FULL_DIR="$BASEDIR/$D" + if [ "$ARGS" == "walk" ]; then + echo " ------------- ( $I / $NUM ) $D -------------" + pushd $FULL_DIR + git status + bash + popd + else + tmp_file=$(mktemp) + tmp_files+=("$tmp_file") + echo " ------------- ( $I / $NUM ) $D -------------" > "$tmp_file" + echo " -- Started -- ( $I / $NUM ) $D -------------" + run_git_command "$FULL_DIR" "$ARGS" "$tmp_file" + + (( running_jobs++ )) + if (( running_jobs >= MAX_PARALLEL )); then + wait -n + (( running_jobs-- )) + fi + fi +done + +if [ "$ARGS" != "walk" ]; then + wait +fi + +echo " ------------- DONE -------------" + +# Print the buffered output +for tmp_file in "${tmp_files[@]}"; do + echo "" + cat "$tmp_file" + rm "$tmp_file" +done +