Audit files support

doc/manual/audit-trail.rst

@@ -74,6 +74,9 @@ Example of a single audit record::
             "c5b2a8231156f43728af34f3a2dcb731ade2f76a"
          ]
       },
+      "files" : {
+         "hashes" : "0dd432edfab90223f22e49c02e2124f87d6f0a56  ./COPYING"
+      },
       "meta" : {
          "language" : "bash",
          "recipe" : "root",
@@ -309,3 +312,16 @@ found under the ``build`` key and contains the following fields:
    information. The ``os-release`` field, if present, is more reliable in this
    case.
 
+Audit files
+~~~~~~~~~~~
+
+Additional files can be included in the audit trail by using
+:ref:`configuration-recipes-auditfiles`.  Essentially, they are included as is
+as strings into a key/value mapping under the ``files`` key. Example::
+
+    {
+       "files" : {
+           "hashes" : "0dd432edfab90223f22e49c02e2124f87d6f0a56  ./COPYING"
+       },
+    }
+

doc/manual/configuration.rst

@@ -489,6 +489,60 @@ can be configured.
 Recipe and class keywords
 -------------------------
 
+.. _configuration-recipes-auditfiles:
+
+{checkout,build,package}AuditFiles
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Type: Dictionary (String -> String | AuditFileDefinition)
+
+The :ref:`audit-trail` records where and when a package was built, the state of
+the recipes and the checked out sources. Additionally, selected files of a step
+can be included into the audit trail too. Example::
+
+    # Create a checksum of all files except the ".bob" folder.
+    checkoutDeterministic: True
+    checkoutScript: |
+        ...
+        mkdir .bob
+        find . -path ./.bob -prune -o \( -type f -print \) | xargs sha1sum > .bob/file-hashes
+
+    checkoutAuditFiles:
+        FILE_HASHES: .bob/file-hashes
+
+This will include the content of ``.bob/file-hashes`` into the audit trail::
+
+    {
+        "files" : {
+            "FILE_HASHES" : "0dd432edfab90223f22e49c02e2124f87d6f0a56  ./COPYING"
+        },
+    }
+
+By default, the named file(s) must be present and are read with UTF-8 encoding.
+Both properties can be changed with the long format::
+
+    packageAuditFiles:
+        COPYING:
+            filename: COPYING
+            encoding: latin1
+            if: "$INCLUDE_COPYING"
+
+The file is only added to the audit trail when the ``if`` :ref:`condition
+<configuration-principle-booleans>` is true. The file name must always be a
+relative path. File names and encodings can use
+:ref:`configuration-principle-subst`. There is a special encoding ``"base64"``
+which can read binary file and includes them base64 encoded into the audit
+trail. See the `Python standard encodings
+<https://docs.python.org/3/library/codecs.html#standard-encodings>`_ for a list
+of possible encodings.
+
+Note that changing any of the audit files properties does not lead to a rebuild
+of affected packages. These settings do not influence the build result and
+therefore also do not contribute to variant management. If two identical
+packages use different audit file settings it is unspecified which setting is
+applied.  Therefore, keep the audit file settings static or ensure that they
+are configured consistent between package variants.
+
 .. _configuration-recipes-scripts:
 
 {checkout,build,package}Script[{Bash,Pwsh}]

pym/bob/audit.py

@@ -81,6 +81,7 @@ class Artifact:
         },
         "env" : str,
         schema.Optional('metaEnv') : { schema.Optional(str) : str },
+        schema.Optional('files') : { schema.Optional(str) : str },
         "scms" : [ dict ],
         schema.Optional("recipes") : dict,
         schema.Optional("layers") : dict,
@@ -205,6 +206,10 @@ def addMetaEnv(self, var, value):
         self.__data.setdefault("metaEnv", {})[var] = value
         self.__invalidateId()
 
+    def addAuditFile(self, var, value):
+        self.__data.setdefault("files", {})[var] = value
+        self.__invalidateId()
+
     def setSandbox(self, sandboxId):
         self.__data["dependencies"]["sandbox"] = asHexStr(sandboxId)
         self.__invalidateId()
@@ -383,6 +388,9 @@ def addTool(self, name, tool):
     def addMetaEnv(self, var, value):
         self.__artifact.addMetaEnv(var, value)
 
+    def addAuditFile(self, var, value):
+        self.__artifact.addAuditFile(var, value)
+
     def setSandbox(self, sandbox):
         audit = Audit.fromFile(sandbox)
         self.__merge(audit)

pym/bob/builder.py

@@ -17,8 +17,9 @@
     SKIPPED, EXECUTED, INFO, WARNING, DEFAULT, \
     ALWAYS, IMPORTANT, NORMAL, INFO, DEBUG, TRACE
 from .utils import asHexStr, hashDirectory, removePath, emptyDirectory, \
-    isWindows, INVALID_CHAR_TRANS, quoteCmdExe, getPlatformTag, canSymlink
+    isWindows, INVALID_CHAR_TRANS, quoteCmdExe, getPlatformTag, canSymlink, isAbsPath
 from .share import NullShare
+from base64 import b64encode
 from shlex import quote
 from textwrap import dedent
 import argparse
@@ -710,6 +711,21 @@ def auditOf(s):
             for var, val in step.getPackage().getMetaEnv().items():
                 audit.addMetaEnv(var, val)
             audit.setRecipesAudit(await step.getPackage().getRecipe().getRecipeSet().getScmAudit())
+            for var, (fn, encoding) in step.getAuditFileNames().items():
+                if isAbsPath(fn):
+                    raise BuildError(f"Audit: {var}: Path is not relative: {fn}")
+                fn = os.path.join(step.getWorkspacePath(), fn)
+                try:
+                    if encoding == "base64":
+                        with open(fn, "rb") as f:
+                            audit.addAuditFile(var, b64encode(f.read()).decode("ascii"))
+                    else:
+                        with open(fn, encoding=encoding) as f:
+                            audit.addAuditFile(var, f.read())
+                except (OSError, ValueError) as e:
+                    raise BuildError(f"Audit: cannot read '{fn}': {e}")
+                except LookupError:
+                    raise BuildError(f"Audit: error reading '{fn}': encoding '{encoding}' not supported")
 
             # The following things make only sense if we just executed the step
             if executed: