From c331446fe3547bdb0ac3d8c90d5d69514c99edcd Mon Sep 17 00:00:00 2001 From: Anders Roxell Date: Mon, 27 Jul 2026 14:30:05 +0200 Subject: [PATCH 1/2] build: drop our own KCFLAGS from the reproducer command line We set KCFLAGS to -ffile-prefix-map=/= to keep the build path out of __FILE__. The build dir is local, and the reproducer does not pass --build-dir. So the next build gets a different build dir but the old path in KCFLAGS. The prefix never matches, and __FILE__ keeps the full path. In the kernel that string shows up in WARN and BUG messages, so the build is not bit by bit identical. Leave it out and let the next build set its own value. Keep KCFLAGS if the user asked for it with --environment. Signed-off-by: Anders Roxell --- test/test_cmdline.py | 10 ++++++++++ tuxmake/build.py | 9 +++++++++ tuxmake/cmdline.py | 5 ++++- 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/test/test_cmdline.py b/test/test_cmdline.py index 286179c..42dabc0 100644 --- a/test/test_cmdline.py +++ b/test/test_cmdline.py @@ -55,6 +55,16 @@ def test_environment(self, cmdline): cmd = cmdline.reproduce(build) assert "--environment=FOO=BAR" in cmd + def test_environment_without_local_kcflags(self, cmdline): + build = Build() + cmd = cmdline.reproduce(build) + assert [o for o in cmd if o.startswith("--environment=KCFLAGS=")] == [] + + def test_environment_with_kcflags_from_the_user(self, cmdline): + build = Build(environment={"KCFLAGS": "-Werror"}) + cmd = cmdline.reproduce(build) + assert "--environment=KCFLAGS=-Werror" in cmd + def test_kconfig_add(self, cmdline): build = Build(kconfig_add=["foo.config", "bar.config"]) cmd = cmdline.reproduce(build) diff --git a/tuxmake/build.py b/tuxmake/build.py index 5e0bec8..210c5c1 100644 --- a/tuxmake/build.py +++ b/tuxmake/build.py @@ -446,6 +446,15 @@ def environment(self): self.__environment__ = env return self.__environment__ + @property + def reproducible_environment(self): + # Our KCFLAGS points at the local build dir, so the next build has + # to set its own. + env = dict(self.environment) + if "KCFLAGS" not in self.__environment_input__: + del env["KCFLAGS"] + return env + def get_silent(self): if self.verbose: return [] diff --git a/tuxmake/cmdline.py b/tuxmake/cmdline.py index 89bca24..904ece4 100644 --- a/tuxmake/cmdline.py +++ b/tuxmake/cmdline.py @@ -326,7 +326,10 @@ def reproduce(self, build): if option.key in self.ignore: continue if hasattr(build, option.key): - value = getattr(build, option.key) + if option.key == "environment": + value = build.reproducible_environment + else: + value = getattr(build, option.key) if not value: continue for c in option.expand(value): From 3c2c50065426e08782eb226429f3009a8fa6334c Mon Sep 17 00:00:00 2001 From: Anders Roxell Date: Mon, 27 Jul 2026 13:10:06 +0200 Subject: [PATCH 2/2] build: save reproducer.sh in the output directory The reproducer command line is already in metadata.json and in the top of build.log. Neither can be run. The line has to be copied out of the log first. Write it to reproducer.sh instead. The file is executable, so it runs directly. It is written before the build starts, so it is there also when the build fails. Signed-off-by: Anders Roxell --- docs/metadata.md | 3 ++- docs/reproducible_builds.md | 16 ++++++++++++++++ test/test_build.py | 19 +++++++++++++++++++ tuxmake/build.py | 15 ++++++++++++++- tuxmake/utils.py | 4 ++-- 5 files changed, 53 insertions(+), 4 deletions(-) diff --git a/docs/metadata.md b/docs/metadata.md index 6e615d6..cc7d2f9 100644 --- a/docs/metadata.md +++ b/docs/metadata.md @@ -59,7 +59,8 @@ available. * **status**: target status: "PASS", "FAIL", or "SKIP" (string). * **duration**: duration of this target build, in seconds (number). - **artifacts**: key/value with target names (string) as keys, and list of - artifacts built for that target (list of strings). + artifacts built for that target (list of strings). Two keys are not + targets: `log` for the build logs, and `reproducer` for `reproducer.sh`. - **errors**: number of errors in the build (integer). - **warnings**: number of warnings in the build (integer). - **sccache**: sccache statistics. diff --git a/docs/reproducible_builds.md b/docs/reproducible_builds.md index baca3af..51641d3 100644 --- a/docs/reproducible_builds.md +++ b/docs/reproducible_builds.md @@ -61,6 +61,22 @@ the one Alice did on their side. The only artifact that will be different is `metadata.json` and `build.log`, because both include data about their local systems. +## The reproducer script + +The same command line is also saved as `reproducer.sh` in the output directory. +It is executable, so person A can just send the file to person B. B runs it +from the root of the same kernel tree: + +``` +linux (master) $ /path/to/artifacts/reproducer.sh +``` + +The script does not set up the kernel tree. You have to be in the tree you want +to build. + +The reproducer.sh is written before the build starts, so you also get it when +the build fails. + ## On container images As described above, the easiest way of ensuring a consistent build environment diff --git a/test/test_build.py b/test/test_build.py index d89cbc3..5d67233 100644 --- a/test/test_build.py +++ b/test/test_build.py @@ -3,6 +3,7 @@ import os import pytest import re +import shlex import subprocess import shutil import urllib @@ -968,6 +969,24 @@ def test_targets(self, metadata): def test_command_line(self, metadata): assert type(metadata["build"]["reproducer_cmdline"]) is list + @pytest.fixture(scope="class") + def reproducer(self, build): + return build.output_dir / "reproducer.sh" + + def test_reproducer_script_written_for_failed_build(self, reproducer): + assert reproducer.read_text().startswith("#!/bin/sh\n") + + def test_reproducer_script_is_executable(self, reproducer): + assert os.access(reproducer, os.X_OK) + + def test_reproducer_script_runs_the_reproducer_cmdline(self, reproducer, metadata): + command = reproducer.read_text().split("\nexec ", 1)[1] + command = command.replace("\\\n", "") + assert shlex.split(command) == metadata["build"]["reproducer_cmdline"] + + def test_reproducer_script_is_an_artifact(self, metadata): + assert metadata["results"]["artifacts"]["reproducer"] == ["reproducer.sh"] + class TestParseLog: @pytest.fixture(scope="class") diff --git a/tuxmake/build.py b/tuxmake/build.py index 210c5c1..b777751 100644 --- a/tuxmake/build.py +++ b/tuxmake/build.py @@ -290,7 +290,10 @@ def __init__( self.offline = False - self.artifacts = {"log": ["build.log", "build-debug.log"]} + self.artifacts = { + "log": ["build.log", "build-debug.log"], + "reproducer": ["reproducer.sh"], + } self.__status__ = {} self.__durations__ = {} self.metadata_collector = MetadataCollector(self) @@ -735,6 +738,15 @@ def save_metadata(self): tmp.write_text(json.dumps(self.metadata, indent=4, sort_keys=True) + "\n") os.replace(tmp, path) + def save_reproducer(self): + # Atomic rename, so a hard kill mid-write can't leave a half-written file. + path = self.output_dir / "reproducer.sh" + tmp = self.output_dir / "reproducer.sh.tmp" + command = quote_command_line(self.cmdline.reproduce(self), " \\\n ") + tmp.write_text(f"#!/bin/sh\nset -eu\nexec {command}\n") + tmp.chmod(0o755) + os.replace(tmp, path) + def parse_log(self): parser = LogParser() parser.parse(self.output_dir / "build.log") @@ -858,6 +870,7 @@ def run(self): with self.measure_duration("Early Metadata Extraction"): self.collect_metadata_early() self.save_metadata() + self.save_reproducer() version_full = self.metadata.get("compiler", {}).get("version_full") if version_full: diff --git a/tuxmake/utils.py b/tuxmake/utils.py index e14421e..56fa4e0 100644 --- a/tuxmake/utils.py +++ b/tuxmake/utils.py @@ -15,8 +15,8 @@ PROGRESS_REPORT_CHUNK_SIZE = 1 * MB # Report progress every 1MB for responsiveness -def quote_command_line(cmd: List[str]) -> str: - return " ".join([shlex.quote(c) for c in cmd]) +def quote_command_line(cmd: List[str], separator: str = " ") -> str: + return separator.join([shlex.quote(c) for c in cmd]) def get_directory_timestamp(directory):