Skip to content

Commit c7ac189

Browse files
pgeng-nvpgeng-nv
authored
Improve GitHub CI robustness for ARM builds (#287)
Our GitHub CI build uses ARM64 containes for ARM based rpm/deb build, while GitHub-hosted runners are x86_64. When we run an ARM64 container, Docker typically uses QEMU for emulation. QEMU can be slow and occasionally unstable, especially for memory-intensive tasks like compiling with gcc, leading to error like this: ``` configure:4941: gcc -o conftest -fno-lto -Wdate-time -D_FORTIFY_SOURCE=2 -fno-lto conftest.c -lusb-1.0 >&5 gcc: internal compiler error: Segmentation fault signal terminated program cc1 ``` These are transient errors that will usually be fixed in a repeated Docker run. So we added logic to retry up to three times in case of an ARM docker run failure.
1 parent cca6323 commit c7ac189

File tree

1 file changed

+108
-45
lines changed

1 file changed

+108
-45
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 108 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -100,36 +100,71 @@ jobs:
100100
101101
- name: Build & Test ARM64 .deb package in Docker
102102
run: |
103-
set -e
103+
set -euo pipefail
104+
104105
docker run --rm --privileged multiarch/qemu-user-static \
105106
--reset -p yes
106-
docker run --rm --platform linux/arm64 \
107-
-e HOST_UID="${{ env.HOST_UID }}" \
108-
-e HOST_GID="${{ env.HOST_GID }}" \
109-
-v "${{ github.workspace }}:/workspace" \
110-
-w /workspace \
111-
arm64v8/ubuntu:22.04 \
112-
/bin/bash -c "
113-
. /workspace/retry_apt_update.sh && \
114-
apt-get update || retry_apt_get_update && \
115-
apt-get install -y build-essential debhelper devscripts \
116-
fakeroot gcc git libpci-dev libusb-1.0-0-dev libfuse-dev \
117-
pkg-config && \
118-
git config --global --add safe.directory /workspace && \
119-
version=\$(/workspace/scripts/get-ver) && \
120-
sed -i \"1s/(\(.*\))/(\$version)/\" \
121-
/workspace/debian/changelog && \
122-
mkdir -p /workspace/build && \
123-
DEB_BUILD_OPTIONS='nocheck nostrip nolto' \
124-
CFLAGS="-fno-lto" LDFLAGS="-fno-lto" \
125-
dpkg-buildpackage -us -uc -a arm64 && \
126-
mv -v ../*.deb /workspace/build/ && \
127-
apt-get install -y libpci3 libusb-1.0-0 fuse || true && \
128-
dpkg -i /workspace/build/*.deb || apt-get -f install -y && \
129-
echo 'Running rshim --version:' && \
130-
rshim --version && \
131-
chown -R \$HOST_UID:\$HOST_GID /workspace
132-
"
107+
108+
# Retry function for Docker build
109+
retry_count=0
110+
max_retries=3
111+
delay=20
112+
113+
while true; do
114+
echo "Docker build attempt $((retry_count + 1))/$max_retries"
115+
116+
if docker run --rm --platform linux/arm64 \
117+
-e HOST_UID="${{ env.HOST_UID }}" \
118+
-e HOST_GID="${{ env.HOST_GID }}" \
119+
-v "${{ github.workspace }}:/workspace" \
120+
-w /workspace \
121+
arm64v8/ubuntu:22.04 \
122+
/bin/bash -c "
123+
set -euo pipefail
124+
. /workspace/retry_apt_update.sh
125+
apt-get update || retry_apt_get_update
126+
apt-get install -y \
127+
autoconf automake libtool build-essential debhelper \
128+
devscripts fakeroot gcc git libpci-dev libusb-1.0-0-dev \
129+
libfuse-dev libsystemd-dev libpci3 libusb-1.0-0 fuse \
130+
pkg-config
131+
git config --global --add safe.directory /workspace && \
132+
version=\$(/workspace/scripts/get-ver)
133+
sed -i \"1s/(\(.*\))/(\$version)/\" /workspace/debian/changelog
134+
mkdir -p /workspace/build
135+
DEB_BUILD_OPTIONS='nocheck nostrip nolto' \
136+
CFLAGS="-fno-lto" LDFLAGS="-fno-lto" \
137+
dpkg-buildpackage -us -uc -a arm64 || {
138+
echo '--- pkg-config failure diagnostics ---'
139+
pkg-config --modversion libpci || true
140+
pkg-config --modversion libusb-1.0 || true
141+
pkg-config --modversion fuse || true
142+
pkg-config --variable=systemdsystemunitdir systemd || true
143+
echo '--- config.log tails ---'
144+
find . -name config.log -exec sh -c \
145+
'echo ==== {}; tail -n 200 {}' \;
146+
exit 1
147+
}
148+
mv -v ../*.deb /workspace/build/ && \
149+
dpkg -i /workspace/build/*.deb || apt-get -f install -y
150+
echo 'Running rshim --version:'
151+
rshim --version
152+
chown -R \$HOST_UID:\$HOST_GID /workspace
153+
"; then
154+
echo "Docker build succeeded on attempt $((retry_count + 1))"
155+
break
156+
else
157+
retry_count=$((retry_count + 1))
158+
if [ $retry_count -lt $max_retries ]; then
159+
echo "Docker build failed, waiting ${delay}s before retry..."
160+
sleep $delay
161+
else
162+
echo "Docker build failed after $max_retries attempts"
163+
exit 1
164+
fi
165+
fi
166+
done
167+
133168
mkdir -p dist
134169
mv -v build/*.deb dist/
135170
ls -lh dist
@@ -143,6 +178,7 @@ jobs:
143178
-w /workspace \
144179
quay.io/rockylinux/rockylinux:8 \
145180
/bin/bash -c "
181+
set -euo pipefail
146182
yum -y install gcc make rpm-build git autoconf automake \
147183
libtool pkgconfig pciutils-devel libusb1-devel fuse-devel && \
148184
git config --global --add safe.directory /workspace && \
@@ -159,25 +195,52 @@ jobs:
159195
160196
- name: Build & Test arm64 RPM using Rocky Linux
161197
run: |
198+
set -euo pipefail
199+
200+
# Prepare QEMU for ARM64 emulation
162201
docker run --rm --privileged multiarch/qemu-user-static \
163202
--reset -p yes
164-
docker run --rm --platform linux/arm64 \
165-
-e HOST_UID="${{ env.HOST_UID }}" \
166-
-e HOST_GID="${{ env.HOST_GID }}" \
167-
-v "${{ github.workspace }}:/workspace" \
168-
-w /workspace \
169-
quay.io/rockylinux/rockylinux:8 \
170-
/bin/bash -c "
171-
yum -y install gcc make rpm-build git autoconf automake \
172-
libtool pkgconfig pciutils-devel libusb1-devel fuse-devel && \
173-
git config --global --add safe.directory /workspace && \
174-
ARCH=aarch64 CFLAGS="-fno-lto" LDFLAGS="-fno-lto" \
175-
./scripts/build-rpm && \
176-
yum localinstall -y /workspace/rpmbuild/RPMS/aarch64/*.rpm && \
177-
echo 'Running rshim --version:' && \
178-
rshim --version && \
179-
chown -R \$HOST_UID:\$HOST_GID /workspace
180-
"
203+
204+
# Retry function for Docker build
205+
retry_count=0
206+
max_retries=3
207+
delay=20
208+
209+
while true; do
210+
echo "Docker build attempt $((retry_count + 1))/$max_retries"
211+
212+
if docker run --rm --platform linux/arm64 \
213+
-e HOST_UID="${{ env.HOST_UID }}" \
214+
-e HOST_GID="${{ env.HOST_GID }}" \
215+
-v "${{ github.workspace }}:/workspace" \
216+
-w /workspace \
217+
quay.io/rockylinux/rockylinux:8 \
218+
/bin/bash -c "
219+
set -euo pipefail
220+
yum -y install gcc make rpm-build git autoconf automake \
221+
libtool pkgconfig pciutils-devel libusb1-devel fuse-devel && \
222+
git config --global --add safe.directory /workspace && \
223+
ARCH=aarch64 CFLAGS=\"-fno-lto\" LDFLAGS=\"-fno-lto\" \
224+
./scripts/build-rpm && \
225+
yum localinstall -y /workspace/rpmbuild/RPMS/aarch64/*.rpm && \
226+
echo 'Running rshim --version:' && \
227+
rshim --version && \
228+
chown -R \$HOST_UID:\$HOST_GID /workspace
229+
"; then
230+
echo "Docker build succeeded on attempt $((retry_count + 1))"
231+
break
232+
else
233+
retry_count=$((retry_count + 1))
234+
if [ $retry_count -lt $max_retries ]; then
235+
echo "Docker build failed, waiting ${delay}s before retry..."
236+
sleep $delay
237+
else
238+
echo "Docker build failed after $max_retries attempts"
239+
exit 1
240+
fi
241+
fi
242+
done
243+
181244
mkdir -p dist
182245
mv -v $(find rpmbuild/RPMS -name '*.rpm') dist/
183246
ls -lh dist

0 commit comments

Comments
 (0)