Skip to content

Commit 561c2e4

Browse files
schuellerfschuellerf
committed
Makefile: implement "fully source containers" HMS-3883
also changes compose file to be more robust on startup use docker as default for now podman does not seem to run stable. will need to investigate more in docs: document makefile overview
1 parent bbd8b91 commit 561c2e4

19 files changed

+1255
-167
lines changed

‎.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,6 @@ on-prem/build/config/*.pem
44
on-prem/build/config/*.toml
55
on-prem/build/config/ca
66
on-prem/build/config/repositories
7+
service_images_built.info
8+
onprem_images_built.info
9+

‎Makefile

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
.PHONY: help
2+
help:
3+
@echo "make [TARGETS...]"
4+
@echo
5+
@echo "This is the makefile of osbuild-getting-started. The following"
6+
@echo "targets are available:"
7+
@echo
8+
@echo " service-containers: Build all needed containers from source to be able to run the service"
9+
@echo " run-service: Run all containers needed for the 'service'"
10+
@echo " This is running in foreground. Use CTRL-C to stop the containers."
11+
@echo " run-service-no-frontend: Run all containers except for the frontend"
12+
@echo " stop-service: Usually the containers get stopped by CTRL-C. So 'stop-service'"
13+
@echo " is only needed if something strange happened and not all are stopped."
14+
@echo " prune-service: Remove the containers, including the test-data!"
15+
@echo " If you want empty databases"
16+
@echo " clean: Clean all subprojects to assure a rebuild"
17+
18+
# source where the other repos are locally
19+
# has to end with a trailing slash
20+
SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= ../
21+
22+
# either "docker" or "sudo podman"
23+
# podman needs to build as root as it also needs to run as root afterwards
24+
CONTAINER_EXECUTABLE ?= docker
25+
CONTAINER_COMPOSE_EXECUTABLE := $(CONTAINER_EXECUTABLE) compose
26+
27+
MAKE_SUB_CALL := make CONTAINER_EXECUTABLE="$(CONTAINER_EXECUTABLE)"
28+
29+
# osbuild is indirectly used by osbuild-composer
30+
# but we'll mention it here too for better error messages and usability
31+
COMMON_SRC_DEPS_NAMES := osbuild osbuild-composer pulp-client community-gateway
32+
COMMON_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(COMMON_SRC_DEPS_NAMES))
33+
34+
ONPREM_SRC_DEPS_NAMES := weldr-client
35+
ONPREM_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(ONPREM_SRC_DEPS_NAMES))
36+
37+
SERVICE_SRC_DEPS_NAMES := image-builder image-builder-frontend
38+
SERVICE_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(SERVICE_SRC_DEPS_NAMES))
39+
40+
# should be set if we are already sudo - otherwise we set to "whoami"
41+
SUDO_USER ?= $(shell whoami)
42+
43+
$(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN):
44+
@for DIR in $@; do if ! [ -d $$DIR ]; then echo "Please checkout $$DIR so it is available at $$DIR"; exit 1; fi; done
45+
46+
COMPARE_TO_BRANCH ?= origin/main
47+
48+
SCRATCH_DIR := /scratch/
49+
COMMON_DIR := podman/image-builder-config
50+
CLI_DIRS := podman/weldr podman/cloudapi podman/dnf-json
51+
DATA_DIR := data/s3
52+
ALL_SCRATCH_DIRS := $(addprefix $(SCRATCH_DIR),$(COMMON_DIR) $(CLI_DIRS) $(DATA_DIR))
53+
54+
# internal rule for sub-calls
55+
# NOTE: This chowns all directories back - as we expect to run partly as root
56+
# also we "git fetch origin" to get the current state!
57+
.PHONY: common_sub_makes
58+
common_sub_makes:
59+
@echo "We need to build everything as root, as the target also needs to run as root."
60+
@echo "At least for podman the password as already needed now"
61+
62+
# creating container image from osbuild as a basis for worker
63+
$(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)osbuild-composer container_worker container_composer
64+
65+
.PHONY: service_sub_makes
66+
service_sub_makes:
67+
# building the backend
68+
$(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder container
69+
# building the frontend
70+
$(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder-frontend container
71+
@for DIR in $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN); do echo "Giving directory permissions in '$$DIR' back to '$(SUDO_USER)'"; sudo chown -R $(SUDO_USER): $$DIR; done
72+
@echo "Your current versions are (comparing to origin/main):"
73+
./git_stack.sh
74+
75+
.PHONY: onprem_sub_makes
76+
onprem_sub_makes:
77+
# building the cli
78+
$(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)weldr-client container
79+
@for DIR in $(COMMON_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN); do echo "Giving directory permissions in '$$DIR' back to '$(SUDO_USER)'"; sudo chown -R $(SUDO_USER): $$DIR; done
80+
@echo "Your current versions are (comparing to origin/main):"
81+
./git_stack.sh
82+
83+
.PHONY: service-containers
84+
service-containers: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) common_sub_makes service_sub_makes service_images_built.info
85+
86+
onprem-containers: $(COMMON_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN) common_sub_makes onprem_sub_makes onprem_images_built.info
87+
88+
service_images_built.info: service/config/Dockerfile-config service/config/composer/osbuild-composer.toml $(ALL_SCRATCH_DIRS)
89+
# building remaining containers (config, fauxauth)
90+
$(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml build --build-arg CONFIG_BUILD_DATE=$(shell date -r $(SCRATCH_DIR)$(COMMON_DIR) +%Y%m%d_%H%M%S)
91+
echo "Images last built on" > $@
92+
date >> $@
93+
94+
onprem_images_built.info: service/config/Dockerfile-config-onprem service/config/composer/osbuild-composer-onprem.toml $(ALL_SCRATCH_DIRS)
95+
# building remaining containers (config)
96+
$(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml build --build-arg CONFIG_BUILD_DATE=$(shell date -r $(SCRATCH_DIR)$(COMMON_DIR) +%Y%m%d_%H%M%S)
97+
echo "Images last built on" > $@
98+
date >> $@
99+
100+
$(ALL_SCRATCH_DIRS):
101+
@echo "Creating directory: $@"
102+
mkdir -p $@ || ( echo "Trying as root" ; sudo mkdir -p $@ )
103+
104+
.PHONY: wipe-config
105+
wipe-config:
106+
sudo rm -rf $(SCRATCH_DIR)$(COMMON_DIR)
107+
rm -f $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder-frontend/node_modules/.cache/webpack-dev-server/server.pem
108+
109+
.PHONY: clean
110+
clean: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN) prune-service prune-onprem wipe-config
111+
rm -f service_images_built.info
112+
rm -f onprem_images_built.info
113+
sudo rm -rf $(SCRATCH_DIR)
114+
for DIR in $*; do $(MAKE_SUB_CALL) -C $$DIR clean; done
115+
$(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml rm
116+
$(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml rm
117+
118+
# for compatibility of relative volume mount paths
119+
# between docker and podman we have to change to the directory
120+
.PHONY: run-service
121+
.ONESHELL:
122+
run-service: $(addprefix $(SCRATCH_DIR),$(COMMON_DIRS)) service-containers
123+
pushd service
124+
$(CONTAINER_COMPOSE_EXECUTABLE) up
125+
popd
126+
127+
# only for strange crashes - should shut down properly in normal operation
128+
.PHONY: stop-service
129+
.ONESHELL:
130+
stop-service:
131+
pushd service
132+
$(CONTAINER_COMPOSE_EXECUTABLE) stop
133+
popd
134+
135+
.PHONY: prune-service
136+
.ONESHELL:
137+
prune-service:
138+
pushd service
139+
$(CONTAINER_COMPOSE_EXECUTABLE) down
140+
popd
141+
142+
# if you want to run the frontend yourself - outside the docker environment
143+
.PHONY: run-service
144+
run-service-no-frontend: service-containers
145+
$(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml up backend fauxauth worker composer minio postgres_backend postgres_composer
146+
147+
.PHONY: prune-onprem
148+
.ONESHELL:
149+
prune-onprem:
150+
pushd service
151+
$(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml down
152+
popd
153+
154+
# for compatibility of relative volume mount paths
155+
# between docker and podman we have to change to the directory
156+
.PHONY: run-onprem
157+
.ONESHELL:
158+
run-onprem: $(addprefix $(SCRATCH_DIR),$(COMMON_DIRS) $(CLI_DIRS)) onprem-containers
159+
pushd service
160+
# remove dangling sockets
161+
sudo rm -f $(addsuffix /api.sock*, $(addprefix $(SCRATCH_DIR), $(CLI_DIRS)))
162+
$(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml up -d
163+
@echo "------ Welcome to osbuild! You can now use 'composer-cli' to build images"
164+
@echo " … and 'exit' afterwards"
165+
@echo " You might also be interested to open up a second terminal and"
166+
@echo " run 'docker compose -f $(shell readlink -f service/docker-compose-onprem.yml) logs --follow' to see possible problems"
167+
168+
$(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml run --entrypoint /bin/bash -ti cli
169+
$(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml stop
170+
popd
171+
172+
%.svg: %.dot
173+
dot -T svg $< > $@
174+
175+
.PHONY: docs
176+
docs: docs/src_compile_service.svg docs/src_compile_onprem.svg
177+
178+
.PHONY: overview
179+
overview:
180+
@echo "Fetching all repos for better overview if rebase will be necessary:"
181+
ifeq (${SUDO_USER},$(shell whoami))
182+
./git_stack.sh fetch --all
183+
else
184+
sudo -u ${SUDO_USER} ./git_stack.sh fetch --all
185+
endif
186+
@./git_stack.sh

‎README.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ provide more in-depth explainations of these products, their architecture and ho
4949

5050
## Getting started
5151

52+
You can develop and install the individual components on your local computer, or you can run the
53+
whole stack as containers fully from source.
54+
5255
For further instructions on how to run containerized versions of the CLI and service locally:
53-
- the [on-premise README](./on-prem/README.md) offers explainations on how to run the on-premise CLI tool.
56+
- the [on-premise README](./on-prem/README.md) offers explanations on how to run the on-premise CLI tool.
5457
- the [service README](./service/README.md) provides instructions on how to get started with a stack of the hosted service.

‎docs/src_compile_onprem.dot

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
digraph src_compile {
2+
newrank=true;
3+
subgraph cluster_osbuild_getting_started {
4+
node [ shape=box; fillcolor=white; style=filled; ]
5+
style=filled; fillcolor=lightgrey;
6+
7+
label = "osbuild-\ngetting-started";
8+
osbuild_getting_started_makefile [
9+
label = "make\nonprem-containers"
10+
width = 2;
11+
];
12+
osbuild_getting_started_run [
13+
label = "make\nrun-onprem"
14+
width = 2;
15+
];
16+
}
17+
18+
subgraph cluster_osbuild {
19+
node [ shape=box; fillcolor=white; style=filled; ]
20+
style=filled; fillcolor=lightgrey;
21+
22+
label = "osbuild\n&nbsp;";
23+
osbuild_makefile [
24+
label = "make\ncontainer";
25+
width = 1.5;
26+
];
27+
}
28+
29+
subgraph cluster_osbuild_composer {
30+
node [ shape=box; fillcolor=white; style=filled; ]
31+
style=filled; fillcolor=lightgrey;
32+
33+
label = "osbuild-composer\n&nbsp;";
34+
osbuild_composer_makefile_composer [
35+
label = "make\ncontainer_composer"
36+
width = 2;
37+
];
38+
osbuild_composer_makefile_worker [
39+
label = "make\ncontainer_worker"
40+
width = 2;
41+
];
42+
43+
osbuild_composer_go [ label = "go code"; ];
44+
}
45+
46+
subgraph cluster_weldr_client {
47+
node [ shape=box; fillcolor=white; style=filled; ]
48+
style=filled; fillcolor=lightgrey;
49+
50+
label = "weldr-client";
51+
weldr_client_makefile [
52+
label = "make\ncontainer"
53+
width = 1.5;
54+
];
55+
}
56+
subgraph cluster_pulp_client {
57+
node [ shape=box; fillcolor=white; style=filled; ]
58+
style=filled; fillcolor=lightgrey;
59+
60+
label = "pulp-client";
61+
pulp_client_go [
62+
label = "go code";
63+
width = 1.5;
64+
];
65+
}
66+
subgraph cluster_images {
67+
node [ shape=box; fillcolor=white; style=filled; ]
68+
style=filled; fillcolor=lightgrey;
69+
70+
label = "images";
71+
images_go [label = "go code";];
72+
}
73+
{ rank=same;
74+
osbuild_makefile;
75+
osbuild_composer_makefile_worker;
76+
osbuild_composer_makefile_composer;
77+
weldr_client_makefile;
78+
79+
}
80+
{ rank=same;
81+
edge [style=dashed; headport=s; tailport=s];
82+
pulp_client_go;
83+
images_go;
84+
}
85+
osbuild_composer_go -> pulp_client_go [style=dashed;];
86+
osbuild_composer_go -> images_go [style=dashed;];
87+
88+
osbuild_getting_started_run -> osbuild_getting_started_makefile;
89+
90+
osbuild_getting_started_makefile -> osbuild_makefile;
91+
osbuild_getting_started_makefile -> osbuild_composer_makefile_worker;
92+
osbuild_getting_started_makefile -> osbuild_composer_makefile_composer;
93+
94+
95+
osbuild_getting_started_makefile -> weldr_client_makefile;
96+
osbuild_composer_makefile_worker -> osbuild_composer_go [style=dashed];
97+
osbuild_composer_makefile_composer -> osbuild_composer_go [style=dashed];
98+
99+
osbuild_composer_makefile_worker -> osbuild_makefile [style=dashed; headport=s; tailport=s];
100+
101+
}

0 commit comments

Comments
 (0)