diff --git a/docs/reproducible_builds.md b/docs/reproducible_builds.md index 51641d3..71eeed1 100644 --- a/docs/reproducible_builds.md +++ b/docs/reproducible_builds.md @@ -37,6 +37,27 @@ configuration, tarballs containing headers/dtbs/modules/kselftest, etc. If you find any artifact that cannot be reproduced (minus the exceptions documented above), please send a bug report. +## Paths in the debug info + +The build directory and the source tree would otherwise end up in the debug +info, and they are different on every machine. TuxMake maps them away with +`-ffile-prefix-map` in `KCFLAGS` and `KAFLAGS`, and `--remap-path-prefix` in +`KRUSTFLAGS`. The build directory becomes `/tuxmake`, and the source files +get names relative to the tree, the same as an in tree build. + +You can pass your own `KCFLAGS`, `KAFLAGS` or `KRUSTFLAGS` with +`--environment`. TuxMake then puts the maps in front of your flags, so the +paths stay out of the debug info anyway. The maps only make sense on this +machine, so the reproducer carries your flags and not the maps. You cannot +turn the maps off. An empty value only means that nothing comes after them. + +So a debugger does not find the sources on its own. Run it from the root of +the kernel tree, or point it there: + +``` +(gdb) dir /path/to/linux +``` + ## Example Alice does a local arm64 build, using a given kernel configuration. At the top diff --git a/docs/targets.md b/docs/targets.md index b2f0e95..312e204 100644 --- a/docs/targets.md +++ b/docs/targets.md @@ -112,6 +112,10 @@ When this target is built, the `kernel` target is not. This target builds the Kernel modules. The modules are compressed in a tarball, which is copied into the output directory as `modules.tar.xz`. +The `build` and `source` symlinks that `modules_install` creates are left out +of the tarball. They point at the local build and source directories, so they +are broken anywhere else, and they made the tarball different on every build. + ## headers diff --git a/test/test_build.py b/test/test_build.py index 5d67233..261239e 100644 --- a/test/test_build.py +++ b/test/test_build.py @@ -1243,6 +1243,67 @@ def test_reproducible_sets_constant_values(self, linux): ts = "KBUILD_BUILD_TIMESTAMP" assert build1.environment[ts] == build2.environment[ts] + def test_maps_the_build_dir_and_the_source_tree(self, linux): + build = Build(tree=linux) + # the build dir map comes last, the last map that matches wins + assert build.environment["KCFLAGS"] == ( + f"-ffile-prefix-map={build.source_tree}/= " + f"-ffile-prefix-map={build.build_dir}=/tuxmake" + ) + + def test_maps_the_same_for_rust(self, linux): + build = Build(tree=linux) + assert build.environment["KRUSTFLAGS"] == ( + f"--remap-path-prefix={build.source_tree}/= " + f"--remap-path-prefix={build.build_dir}=/tuxmake" + ) + + def test_maps_the_same_for_assembly(self, linux): + env = Build(tree=linux).environment + assert env["KAFLAGS"] == env["KCFLAGS"] + + def test_keeps_the_maps_when_the_user_passes_flags(self, linux): + build = Build(tree=linux, environment={"KCFLAGS": "-Werror"}) + assert build.environment["KCFLAGS"] == ( + f"-ffile-prefix-map={build.source_tree}/= " + f"-ffile-prefix-map={build.build_dir}=/tuxmake " + "-Werror" + ) + + def test_reproducer_keeps_only_the_flags_the_user_passed(self, linux): + build = Build(tree=linux, environment={"KCFLAGS": "-Werror"}) + env = build.reproducible_environment + assert env["KCFLAGS"] == "-Werror" + assert "KAFLAGS" not in env + assert "KRUSTFLAGS" not in env + + def test_keeps_an_empty_value_the_user_passed(self, linux): + build = Build(tree=linux, environment={"KCFLAGS": ""}) + # nothing to put after the map, but the reproducer still says what + # the user asked for + assert build.environment["KCFLAGS"] == ( + f"-ffile-prefix-map={build.source_tree}/= " + f"-ffile-prefix-map={build.build_dir}=/tuxmake" + ) + assert build.reproducible_environment["KCFLAGS"] == "" + + +class TestGitWorktree: + @pytest.fixture + def worktree(self, linux_rw, tmp_path, mocker): + mocker.patch("tuxmake.build.get_directory_timestamp", return_value="1") + git_dir = tmp_path / "main" / ".git" + (git_dir / "worktrees" / "wt").mkdir(parents=True) + (git_dir / "worktrees" / "wt" / "commondir").write_text("../..\n") + (linux_rw / ".git").write_text(f"gitdir: {git_dir}/worktrees/wt\n") + return linux_rw, git_dir + + def test_mounts_the_git_dir(self, worktree, mocker, Popen): + tree, git_dir = worktree + add_volume = mocker.patch("tuxmake.runtime.Runtime.add_volume") + Build(tree=tree).prepare() + assert mocker.call(git_dir, ro=True) in add_volume.call_args_list + class TestTerminated: def test_signal_handler_raises_exception(self): diff --git a/test/test_cmdline.py b/test/test_cmdline.py index 42dabc0..1408bcc 100644 --- a/test/test_cmdline.py +++ b/test/test_cmdline.py @@ -55,10 +55,11 @@ def test_environment(self, cmdline): cmd = cmdline.reproduce(build) assert "--environment=FOO=BAR" in cmd - def test_environment_without_local_kcflags(self, cmdline): + @pytest.mark.parametrize("var", ["KCFLAGS", "KAFLAGS", "KRUSTFLAGS"]) + def test_environment_without_local_prefix_map(self, cmdline, var): build = Build() cmd = cmdline.reproduce(build) - assert [o for o in cmd if o.startswith("--environment=KCFLAGS=")] == [] + assert [o for o in cmd if o.startswith(f"--environment={var}=")] == [] def test_environment_with_kcflags_from_the_user(self, cmdline): build = Build(environment={"KCFLAGS": "-Werror"}) diff --git a/test/test_target.py b/test/test_target.py index 270a90e..c1eb648 100644 --- a/test/test_target.py +++ b/test/test_target.py @@ -97,6 +97,11 @@ def test_strip_modules(self, modules): def test_depends_on_config(self, modules): assert modules.dependencies == ["config"] + def test_leaves_out_the_build_dir_symlinks(self, modules): + tar = modules.commands[2] + assert "--exclude=lib/modules/*/build" in tar + assert "--exclude=lib/modules/*/source" in tar + class TestDtbs: def test_commands(self, build): diff --git a/test/test_utils.py b/test/test_utils.py index 457af46..60955ad 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -3,6 +3,7 @@ import pytest from unittest.mock import patch, MagicMock from tuxmake.utils import get_directory_timestamp +from tuxmake.utils import get_git_dir from tuxmake.utils import retry from tuxmake.utils import download_file_with_progress from tuxmake.utils import prepare_file_from_source @@ -308,3 +309,35 @@ def test_prepare_local_xz_file_without_logger(self, tmp_path): mock_print.assert_called_once() print_call = mock_print.call_args[0][0] assert "Decompressing" in print_call + + +class TestGetGitDir: + def test_worktree(self, tmp_path): + git_dir = tmp_path / "main" / ".git" + (git_dir / "worktrees" / "wt").mkdir(parents=True) + (git_dir / "worktrees" / "wt" / "commondir").write_text("../..\n") + tree = tmp_path / "wt" + tree.mkdir() + (tree / ".git").write_text(f"gitdir: {git_dir}/worktrees/wt\n") + assert get_git_dir(tree) == git_dir + + def test_relative_gitdir(self, tmp_path): + # submodules write a relative gitdir, and no commondir + git_dir = tmp_path / "main" / ".git" / "modules" / "sub" + git_dir.mkdir(parents=True) + tree = tmp_path / "main" / "sub" + tree.mkdir(parents=True) + (tree / ".git").write_text("gitdir: ../.git/modules/sub\n") + assert get_git_dir(tree) == git_dir + + def test_normal_tree(self, tmp_path): + (tmp_path / ".git").mkdir() + assert get_git_dir(tmp_path) is None + + def test_dotgit_file_without_gitdir(self, tmp_path): + (tmp_path / ".git").write_text("something else\n") + assert get_git_dir(tmp_path) is None + + def test_gitdir_that_does_not_exist(self, tmp_path): + (tmp_path / ".git").write_text(f"gitdir: {tmp_path}/gone\n") + assert get_git_dir(tmp_path) is None diff --git a/tuxmake/build.py b/tuxmake/build.py index b777751..073f36d 100644 --- a/tuxmake/build.py +++ b/tuxmake/build.py @@ -36,6 +36,7 @@ from tuxmake.build_utils import defaults from tuxmake.utils import quote_command_line from tuxmake.utils import get_directory_timestamp +from tuxmake.utils import get_git_dir from tuxmake.utils import prepare_file_from_source @@ -166,6 +167,10 @@ class Build: "O", ] + # Set from the local build dir, so they are left out of the reproducer + # command line. The next build sets its own. + LOCAL_ENVIRONMENT = ["KCFLAGS", "KAFLAGS", "KRUSTFLAGS"] + def __init__( self, tree=".", @@ -373,6 +378,9 @@ def prepare(self): self.runtime.source_dir = self.source_tree self.runtime.output_dir = self.output_dir self.runtime.add_volume(self.build_dir) + git_dir = get_git_dir(self.source_tree) + if git_dir: + self.runtime.add_volume(git_dir, ro=True) if self.prepare_korg_gcc: self.runtime.add_volume(self.korg_toolchains_dir) if self.wrapper.path: @@ -444,18 +452,40 @@ def environment(self): env["KBUILD_BUILD_TIMESTAMP"] = "@" + self.timestamp env["KBUILD_BUILD_USER"] = "tuxmake" env["KBUILD_BUILD_HOST"] = "tuxmake" - env["KCFLAGS"] = f"-ffile-prefix-map={self.build_dir}/=" + # The build dir has no trailing slash: the compilation directory is + # the build dir itself, and a map with a slash does not match it. The + # source tree has one, so the file names come out relative to it. + # The build dir comes last, the last map that matches wins, and the + # build dir can be inside the source tree. + prefix_map = ( + f"-ffile-prefix-map={self.source_tree}/= " + f"-ffile-prefix-map={self.build_dir}=/tuxmake" + ) + maps = dict.fromkeys(self.LOCAL_ENVIRONMENT, prefix_map) + # rustc does not take the gcc spelling. + maps["KRUSTFLAGS"] = ( + f"--remap-path-prefix={self.source_tree}/= " + f"--remap-path-prefix={self.build_dir}=/tuxmake" + ) env.update(self.__environment_input__) + # The user can set these too. Keep our map in front, so the paths + # stay out of the debug info either way. + for var, prefix in maps.items(): + given = self.__environment_input__.get(var) + env[var] = f"{prefix} {given}" if given else prefix self.__environment__ = env return self.__environment__ @property def reproducible_environment(self): - # Our KCFLAGS points at the local build dir, so the next build has - # to set its own. env = dict(self.environment) - if "KCFLAGS" not in self.__environment_input__: - del env["KCFLAGS"] + # The maps hold paths from this machine, so drop them here. What the + # user passed can stay. + for var in self.LOCAL_ENVIRONMENT: + if var in self.__environment_input__: + env[var] = self.__environment_input__[var] + else: + del env[var] return env def get_silent(self): diff --git a/tuxmake/target/modules.ini b/tuxmake/target/modules.ini index bb91ae6..f2f1470 100644 --- a/tuxmake/target/modules.ini +++ b/tuxmake/target/modules.ini @@ -4,7 +4,9 @@ dependencies = config preconditions = grep -q CONFIG_MODULES=y {build_dir}/.config commands = rm -rf {build_dir}/modinstall && {make} modules_install - && {tar_caf} {build_dir}/modules.tar{z_ext} -C {build_dir}/modinstall lib + && {tar_caf} {build_dir}/modules.tar{z_ext} + --exclude=lib/modules/*/build --exclude=lib/modules/*/source + -C {build_dir}/modinstall lib [makevars] INSTALL_MOD_STRIP = 1 diff --git a/tuxmake/utils.py b/tuxmake/utils.py index 56fa4e0..6af371f 100644 --- a/tuxmake/utils.py +++ b/tuxmake/utils.py @@ -19,6 +19,28 @@ def quote_command_line(cmd: List[str], separator: str = " ") -> str: return separator.join([shlex.quote(c) for c in cmd]) +def get_git_dir(directory): + # In a git worktree, .git is a file that points at a directory outside + # the tree. Return that directory, so it can be made available to the + # build. Return None for a normal tree, where .git is inside it. + dotgit = directory / ".git" + if not dotgit.is_file(): + return None + _, sep, after = dotgit.read_text().partition("gitdir:") + if not sep: + return None + gitdir = directory / Path(after.strip()) + commondir = gitdir / "commondir" + if commondir.exists(): + gitdir = gitdir / commondir.read_text().strip() + gitdir = gitdir.resolve() + if not gitdir.is_dir(): + # The pointer can be stale. Mounting a path that is not there gives + # the build an empty dir, and then git finds no repo at all. + return None + return gitdir + + def get_directory_timestamp(directory): if (directory / ".git").exists(): try: