Skip to content

Commit d2cc5a0

Browse files
schuellerfschuellerf
committed
Makefile: implement "fully source containers" HMS-3883
also changes compose file to be more robust on startup use docker as default for now podman does not seem to run stable. will need to investigate more in docs: document makefile overview also fix execution on MAC
1 parent 1ac2af3 commit d2cc5a0

20 files changed

+1287
-169
lines changed

‎.gitignore

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,3 +4,6 @@ on-prem/build/config/*.pem
44
on-prem/build/config/*.toml
55
on-prem/build/config/ca
66
on-prem/build/config/repositories
7+
service_images_built.info
8+
onprem_images_built.info
9+

‎Makefile

Lines changed: 202 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,202 @@
1+
.PHONY: help
2+
help:
3+
@echo "make [TARGETS...]"
4+
@echo
5+
@echo "This is the makefile of osbuild-getting-started. The following"
6+
@echo "targets are available:"
7+
@echo
8+
@echo " service_containers: Build all needed containers from source to be able to run the service"
9+
@echo " run_service: Run all containers needed for the 'service'"
10+
@echo " This is running in foreground. Use CTRL-C to stop the containers."
11+
@echo " run_service_no_frontend: Run all containers except for the frontend"
12+
@echo " stop_service: Usually the containers get stopped by CTRL-C. So 'stop-service'"
13+
@echo " is only needed if something strange happened and not all are stopped."
14+
@echo " prune_service: Remove the containers, including the test-data!"
15+
@echo " If you want empty databases"
16+
@echo " clean: Clean all subprojects to assure a rebuild"
17+
18+
# source where the other repos are locally
19+
# has to end with a trailing slash
20+
SRC_DEPS_EXTERNAL_CHECKOUT_DIR ?= ../
21+
22+
# either "docker" or "sudo podman"
23+
# podman needs to build as root as it also needs to run as root afterwards
24+
CONTAINER_EXECUTABLE ?= docker
25+
CONTAINER_COMPOSE_EXECUTABLE := $(CONTAINER_EXECUTABLE) compose
26+
27+
MAKE_SUB_CALL := make CONTAINER_EXECUTABLE="$(CONTAINER_EXECUTABLE)"
28+
29+
# osbuild is indirectly used by osbuild-composer
30+
# but we'll mention it here too for better error messages and usability
31+
COMMON_SRC_DEPS_NAMES := osbuild osbuild-composer pulp-client community-gateway
32+
COMMON_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(COMMON_SRC_DEPS_NAMES))
33+
34+
ONPREM_SRC_DEPS_NAMES := weldr-client
35+
ONPREM_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(ONPREM_SRC_DEPS_NAMES))
36+
37+
SERVICE_SRC_DEPS_NAMES := image-builder image-builder-frontend
38+
SERVICE_SRC_DEPS_ORIGIN := $(addprefix $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR),$(SERVICE_SRC_DEPS_NAMES))
39+
40+
# should be set if we are already sudo - otherwise we set to "whoami"
41+
SUDO_USER ?= $(shell whoami)
42+
43+
$(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN):
44+
@for DIR in $@; do if ! [ -d $$DIR ]; then echo "Please checkout $$DIR so it is available at $$DIR"; exit 1; fi; done
45+
46+
COMPARE_TO_BRANCH ?= origin/main
47+
48+
SCRATCH_DIR := $(HOME)/.cache/osbuild-getting-started/scratch
49+
export SCRATCH_DIR
50+
COMMON_DIR := podman/image-builder-config
51+
CLI_DIRS := podman/weldr podman/cloudapi podman/dnf-json
52+
DATA_DIR := data/s3/service
53+
ALL_SCRATCH_DIRS := $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIR) $(CLI_DIRS) $(DATA_DIR))
54+
55+
# internal rule for sub-calls
56+
# NOTE: This chowns all directories back - as we expect to run partly as root
57+
# also we "git fetch origin" to get the current state!
58+
.PHONY: common_sub_makes
59+
common_sub_makes:
60+
@echo "We need to build everything as root, as the target also needs to run as root."
61+
@echo "At least for podman the password as already needed now"
62+
63+
# creating container image from osbuild as a basis for worker
64+
$(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)osbuild-composer container_worker.dev container_composer.dev
65+
66+
.PHONY: service_sub_make_backend
67+
service_sub_make_backend:
68+
$(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder container.dev
69+
70+
.PHONY: service_sub_make_frontend
71+
service_sub_make_frontend:
72+
$(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder-frontend container.dev
73+
74+
.PHONY: service_sub_make_cleanup
75+
service_sub_make_cleanup:
76+
@for DIR in $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN); do echo "Giving directory permissions in '$$DIR' back to '$(SUDO_USER)'"; chown -R $(SUDO_USER): $$DIR || sudo chown -R $(SUDO_USER): $$DIR; done
77+
@echo "Your current versions are (comparing to origin/main):"
78+
bash -c './tools/git_stack.sh'
79+
80+
.PHONY: service_sub_makes_no_frontend
81+
service_sub_makes_no_frontend: service_sub_make_backend service_sub_make_cleanup
82+
83+
.PHONY: service_sub_makes
84+
service_sub_makes: service_sub_make_backend service_sub_make_frontend service_sub_make_cleanup
85+
86+
.PHONY: onprem_sub_makes
87+
onprem_sub_makes:
88+
# building the cli
89+
$(MAKE_SUB_CALL) -C $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)weldr-client container.dev
90+
@for DIR in $(COMMON_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN); do echo "Giving directory permissions in '$$DIR' back to '$(SUDO_USER)'"; chown -R $(SUDO_USER): $$DIR || sudo chown -R $(SUDO_USER): $$DIR; done
91+
@echo "Your current versions are (comparing to origin/main):"
92+
bash -c './tools/git_stack.sh'
93+
94+
.PHONY: service_containers
95+
service_containers: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) common_sub_makes service_sub_makes service_images_built.info
96+
97+
.PHONY: service_containers_no_frontend
98+
service_containers_no_frontend: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) common_sub_makes service_sub_makes_no_frontend service_images_built.info
99+
100+
onprem_containers: $(COMMON_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN) common_sub_makes onprem_sub_makes onprem_images_built.info
101+
102+
service_images_built.info: service/config/Dockerfile-config service/config/composer/osbuild-composer.toml $(ALL_SCRATCH_DIRS)
103+
# building remaining containers (config, fauxauth)
104+
$(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml build --build-arg CONFIG_BUILD_DATE=$(shell date -r $(SCRATCH_DIR)/$(COMMON_DIR) +%Y%m%d_%H%M%S)
105+
echo "Images last built on" > $@
106+
date >> $@
107+
108+
onprem_images_built.info: service/config/Dockerfile-config-onprem service/config/composer/osbuild-composer-onprem.toml $(ALL_SCRATCH_DIRS)
109+
# building remaining containers (config)
110+
$(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml build --build-arg CONFIG_BUILD_DATE=$(shell date -r $(SCRATCH_DIR)/$(COMMON_DIR) +%Y%m%d_%H%M%S)
111+
echo "Images last built on" > $@
112+
date >> $@
113+
114+
$(ALL_SCRATCH_DIRS):
115+
@echo "Creating directory: $@"
116+
mkdir -p $@ || ( echo "Trying as root" ; sudo mkdir -p $@ )
117+
118+
.PHONY: wipe_config
119+
wipe_config:
120+
sudo rm -rf $(SCRATCH_DIR)/$(COMMON_DIR)
121+
rm -f $(SRC_DEPS_EXTERNAL_CHECKOUT_DIR)image-builder-frontend/node_modules/.cache/webpack-dev-server/server.pem
122+
123+
.PHONY: clean
124+
clean: $(COMMON_SRC_DEPS_ORIGIN) $(SERVICE_SRC_DEPS_ORIGIN) $(ONPREM_SRC_DEPS_ORIGIN) prune_service prune_onprem wipe_config
125+
rm -f service_images_built.info
126+
rm -f onprem_images_built.info
127+
rm -rf $(SCRATCH_DIR) || (echo "Trying as root" ;sudo rm -rf $(SCRATCH_DIR))
128+
for DIR in $*; do $(MAKE_SUB_CALL) -C $$DIR clean; done
129+
$(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml down --volumes
130+
$(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml rm --volumes
131+
$(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml down --volumes
132+
$(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose-onprem.yml rm --volumes
133+
134+
# for compatibility of relative volume mount paths
135+
# between docker and podman we have to change to the directory
136+
.PHONY: run_service
137+
.ONESHELL:
138+
run_service: $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIRS)) service_containers
139+
export PORTS="$(shell docker compose -f service/docker-compose.yml config|grep -Po '(?<=published: ")[0-9]+')"
140+
echo "-- Checking if any of our ports are used: $$PORTS"
141+
sudo netstat -lntp|grep -E "$$(echo "$$PORTS"|tr ' ' ':')"
142+
echo "-- Check done"
143+
cd service
144+
$(CONTAINER_COMPOSE_EXECUTABLE) up
145+
146+
# if you want to run the frontend yourself - outside the docker environment
147+
.PHONY: run_service_no_frontend
148+
run_service_no_frontend: service_containers_no_frontend
149+
$(CONTAINER_COMPOSE_EXECUTABLE) -f service/docker-compose.yml up backend fauxauth worker composer minio postgres_backend postgres_composer
150+
151+
# only for strange crashes - should shut down properly in normal operation
152+
.PHONY: stop_service
153+
.ONESHELL:
154+
stop_service:
155+
cd service
156+
$(CONTAINER_COMPOSE_EXECUTABLE) stop
157+
158+
.PHONY: prune_service
159+
.ONESHELL:
160+
prune_service:
161+
cd service
162+
$(CONTAINER_COMPOSE_EXECUTABLE) down
163+
164+
.PHONY: prune_onprem
165+
.ONESHELL:
166+
prune_onprem:
167+
cd service
168+
$(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml down
169+
170+
# for compatibility of relative volume mount paths
171+
# between docker and podman we have to change to the directory
172+
.PHONY: run_onprem
173+
.ONESHELL:
174+
run_onprem: $(addprefix $(SCRATCH_DIR)/,$(COMMON_DIRS) $(CLI_DIRS)) onprem_containers
175+
cd service
176+
echo "Remove dangling sockets as root"
177+
sudo rm -f $(addsuffix /api.sock*, $(addprefix $(SCRATCH_DIR)/, $(CLI_DIRS)))
178+
$(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml up -d
179+
@echo "------ Welcome to osbuild! You can now use 'composer-cli' to build images"
180+
@echo " … and 'exit' afterwards"
181+
@echo " You might also be interested to open up a second terminal and"
182+
@echo " run 'docker compose -f $(shell readlink -f service/docker-compose-onprem.yml) logs --follow' to see possible problems"
183+
184+
$(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml run --entrypoint /bin/bash -ti cli
185+
$(CONTAINER_COMPOSE_EXECUTABLE) -f docker-compose-onprem.yml stop
186+
187+
%.svg: %.dot
188+
dot -T svg $< > $@
189+
190+
.PHONY: docs
191+
docs: docs/src_compile_service.svg docs/src_compile_onprem.svg
192+
193+
.PHONY: overview
194+
overview:
195+
@echo "Fetching all repos for better overview if rebase will be necessary:"
196+
ifeq (${SUDO_USER},$(shell whoami))
197+
bash -c './tools/git_stack.sh fetch --all'
198+
else
199+
sudo -u ${SUDO_USER} bash -c './tools/git_stack.sh fetch --all'
200+
endif
201+
@bash -c './tools/git_stack.sh'
202+

‎README.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@ provide more in-depth explainations of these products, their architecture and ho
4949

5050
## Getting started
5151

52+
You can develop and install the individual components on your local computer, or you can run the
53+
whole stack as containers fully from source.
54+
5255
For further instructions on how to run containerized versions of the CLI and service locally:
53-
- the [on-premise README](./on-prem/README.md) offers explainations on how to run the on-premise CLI tool.
56+
- the [on-premise README](./on-prem/README.md) offers explanations on how to run the on-premise CLI tool.
5457
- the [service README](./service/README.md) provides instructions on how to get started with a stack of the hosted service.

‎docs/src_compile_onprem.dot

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
digraph src_compile {
2+
newrank=true;
3+
subgraph cluster_osbuild_getting_started {
4+
node [ shape=box; fillcolor=white; style=filled; ]
5+
style=filled; fillcolor=lightgrey;
6+
7+
label = "osbuild-\ngetting-started";
8+
osbuild_getting_started_makefile [
9+
label = "make\nonprem_containers"
10+
width = 2;
11+
];
12+
osbuild_getting_started_run [
13+
label = "make\nrun_onprem"
14+
width = 2;
15+
];
16+
}
17+
18+
subgraph cluster_osbuild {
19+
node [ shape=box; fillcolor=white; style=filled; ]
20+
style=filled; fillcolor=lightgrey;
21+
22+
label = "osbuild\n&nbsp;";
23+
osbuild_makefile [
24+
label = "make\ncontainer";
25+
width = 1.5;
26+
];
27+
}
28+
29+
subgraph cluster_osbuild_composer {
30+
node [ shape=box; fillcolor=white; style=filled; ]
31+
style=filled; fillcolor=lightgrey;
32+
33+
label = "osbuild-composer\n&nbsp;";
34+
osbuild_composer_makefile_composer [
35+
label = "make\ncontainer_composer"
36+
width = 2;
37+
];
38+
osbuild_composer_makefile_worker [
39+
label = "make\ncontainer_worker"
40+
width = 2;
41+
];
42+
43+
osbuild_composer_go [ label = "go code"; ];
44+
}
45+
46+
subgraph cluster_weldr_client {
47+
node [ shape=box; fillcolor=white; style=filled; ]
48+
style=filled; fillcolor=lightgrey;
49+
50+
label = "weldr-client";
51+
weldr_client_makefile [
52+
label = "make\ncontainer"
53+
width = 1.5;
54+
];
55+
}
56+
subgraph cluster_pulp_client {
57+
node [ shape=box; fillcolor=white; style=filled; ]
58+
style=filled; fillcolor=lightgrey;
59+
60+
label = "pulp-client";
61+
pulp_client_go [
62+
label = "go code";
63+
width = 1.5;
64+
];
65+
}
66+
subgraph cluster_images {
67+
node [ shape=box; fillcolor=white; style=filled; ]
68+
style=filled; fillcolor=lightgrey;
69+
70+
label = "images";
71+
images_go [label = "go code";];
72+
}
73+
{ rank=same;
74+
osbuild_makefile;
75+
osbuild_composer_makefile_worker;
76+
osbuild_composer_makefile_composer;
77+
weldr_client_makefile;
78+
79+
}
80+
{ rank=same;
81+
edge [style=dashed; headport=s; tailport=s];
82+
pulp_client_go;
83+
images_go;
84+
}
85+
osbuild_composer_go -> pulp_client_go [style=dashed;];
86+
osbuild_composer_go -> images_go [style=dashed;];
87+
88+
osbuild_getting_started_run -> osbuild_getting_started_makefile;
89+
90+
osbuild_getting_started_makefile -> osbuild_makefile;
91+
osbuild_getting_started_makefile -> osbuild_composer_makefile_worker;
92+
osbuild_getting_started_makefile -> osbuild_composer_makefile_composer;
93+
94+
95+
osbuild_getting_started_makefile -> weldr_client_makefile;
96+
osbuild_composer_makefile_worker -> osbuild_composer_go [style=dashed];
97+
osbuild_composer_makefile_composer -> osbuild_composer_go [style=dashed];
98+
99+
osbuild_composer_makefile_worker -> osbuild_makefile [style=dashed; headport=s; tailport=s];
100+
101+
}

0 commit comments

Comments
 (0)