diff --git a/doc/manual/audit-trail.rst b/doc/manual/audit-trail.rst index f6cd4280c..0172d6cc1 100644 --- a/doc/manual/audit-trail.rst +++ b/doc/manual/audit-trail.rst @@ -74,6 +74,9 @@ Example of a single audit record:: "c5b2a8231156f43728af34f3a2dcb731ade2f76a" ] }, + "files" : { + "hashes" : "0dd432edfab90223f22e49c02e2124f87d6f0a56 ./COPYING" + }, "meta" : { "language" : "bash", "recipe" : "root", @@ -309,3 +312,16 @@ found under the ``build`` key and contains the following fields: information. The ``os-release`` field, if present, is more reliable in this case. +Audit files +~~~~~~~~~~~ + +Additional files can be included in the audit trail by using +:ref:`configuration-recipes-auditfiles`. Essentially, they are included as is +as strings into a key/value mapping under the ``files`` key. Example:: + + { + "files" : { + "hashes" : "0dd432edfab90223f22e49c02e2124f87d6f0a56 ./COPYING" + }, + } + diff --git a/doc/manual/configuration.rst b/doc/manual/configuration.rst index 61562d467..9c320649f 100644 --- a/doc/manual/configuration.rst +++ b/doc/manual/configuration.rst @@ -489,6 +489,60 @@ can be configured. Recipe and class keywords ------------------------- +.. _configuration-recipes-auditfiles: + +{checkout,build,package}AuditFiles +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Type: Dictionary (String -> String | AuditFileDefinition) + +The :ref:`audit-trail` records where and when a package was built, the state of +the recipes and the checked out sources. Additionally, selected files of a step +can be included into the audit trail too. Example:: + + # Create a checksum of all files except the ".bob" folder. + checkoutDeterministic: True + checkoutScript: | + ... + mkdir .bob + find . -path ./.bob -prune -o \( -type f -print \) | xargs sha1sum > .bob/file-hashes + + checkoutAuditFiles: + FILE_HASHES: .bob/file-hashes + +This will include the content of ``.bob/file-hashes`` into the audit trail:: + + { + "files" : { + "FILE_HASHES" : "0dd432edfab90223f22e49c02e2124f87d6f0a56 ./COPYING" + }, + } + +By default, the named file(s) must be present and are read with UTF-8 encoding. +Both properties can be changed with the long format:: + + packageAuditFiles: + COPYING: + filename: COPYING + encoding: latin1 + if: "$INCLUDE_COPYING" + +The file is only added to the audit trail when the ``if`` :ref:`condition +` is true. The file name must always be a +relative path. File names and encodings can use +:ref:`configuration-principle-subst`. There is a special encoding ``"base64"`` +which can read binary file and includes them base64 encoded into the audit +trail. See the `Python standard encodings +`_ for a list +of possible encodings. + +Note that changing any of the audit files properties does not lead to a rebuild +of affected packages. These settings do not influence the build result and +therefore also do not contribute to variant management. If two identical +packages use different audit file settings it is unspecified which setting is +applied. Therefore, keep the audit file settings static or ensure that they +are configured consistent between package variants. + .. _configuration-recipes-scripts: {checkout,build,package}Script[{Bash,Pwsh}] diff --git a/pym/bob/audit.py b/pym/bob/audit.py index 6cb89c9dc..45f7a9801 100644 --- a/pym/bob/audit.py +++ b/pym/bob/audit.py @@ -81,6 +81,7 @@ class Artifact: }, "env" : str, schema.Optional('metaEnv') : { schema.Optional(str) : str }, + schema.Optional('files') : { schema.Optional(str) : str }, "scms" : [ dict ], schema.Optional("recipes") : dict, schema.Optional("layers") : dict, @@ -205,6 +206,10 @@ def addMetaEnv(self, var, value): self.__data.setdefault("metaEnv", {})[var] = value self.__invalidateId() + def addAuditFile(self, var, value): + self.__data.setdefault("files", {})[var] = value + self.__invalidateId() + def setSandbox(self, sandboxId): self.__data["dependencies"]["sandbox"] = asHexStr(sandboxId) self.__invalidateId() @@ -383,6 +388,9 @@ def addTool(self, name, tool): def addMetaEnv(self, var, value): self.__artifact.addMetaEnv(var, value) + def addAuditFile(self, var, value): + self.__artifact.addAuditFile(var, value) + def setSandbox(self, sandbox): audit = Audit.fromFile(sandbox) self.__merge(audit) diff --git a/pym/bob/builder.py b/pym/bob/builder.py index 36627f2e5..39e3cd309 100644 --- a/pym/bob/builder.py +++ b/pym/bob/builder.py @@ -17,8 +17,9 @@ SKIPPED, EXECUTED, INFO, WARNING, DEFAULT, \ ALWAYS, IMPORTANT, NORMAL, INFO, DEBUG, TRACE from .utils import asHexStr, hashDirectory, removePath, emptyDirectory, \ - isWindows, INVALID_CHAR_TRANS, quoteCmdExe, getPlatformTag, canSymlink + isWindows, INVALID_CHAR_TRANS, quoteCmdExe, getPlatformTag, canSymlink, isAbsPath from .share import NullShare +from base64 import b64encode from shlex import quote from textwrap import dedent import argparse @@ -710,6 +711,21 @@ def auditOf(s): for var, val in step.getPackage().getMetaEnv().items(): audit.addMetaEnv(var, val) audit.setRecipesAudit(await step.getPackage().getRecipe().getRecipeSet().getScmAudit()) + for var, (fn, encoding) in step.getAuditFileNames().items(): + if isAbsPath(fn): + raise BuildError(f"Audit: {var}: Path is not relative: {fn}") + fn = os.path.join(step.getWorkspacePath(), fn) + try: + if encoding == "base64": + with open(fn, "rb") as f: + audit.addAuditFile(var, b64encode(f.read()).decode("ascii")) + else: + with open(fn, encoding=encoding) as f: + audit.addAuditFile(var, f.read()) + except (OSError, ValueError) as e: + raise BuildError(f"Audit: cannot read '{fn}': {e}") + except LookupError: + raise BuildError(f"Audit: error reading '{fn}': encoding '{encoding}' not supported") # The following things make only sense if we just executed the step if executed: diff --git a/pym/bob/input.py b/pym/bob/input.py index b2560387b..7ee3fab2d 100644 --- a/pym/bob/input.py +++ b/pym/bob/input.py @@ -843,10 +843,11 @@ def getUser(self): class CoreStep(CoreItem): __slots__ = ( "corePackage", "digestEnv", "env", "args", "providedEnv", "providedTools", "providedDeps", "providedSandbox", - "variantId", "deterministic", "isValid", "toolDep", "toolDepWeak" ) + "variantId", "deterministic", "isValid", "toolDep", "toolDepWeak", + "auditFileNames" ) def __init__(self, corePackage, isValid, deterministic, digestEnv, env, args, - toolDep, toolDepWeak): + toolDep, toolDepWeak, auditFileNames): self.corePackage = corePackage self.isValid = isValid self.digestEnv = digestEnv.detach() @@ -861,6 +862,7 @@ def __init__(self, corePackage, isValid, deterministic, digestEnv, env, args, self.providedTools = {} self.providedDeps = [] self.providedSandbox = None + self.auditFileNames = auditFileNames def getPreRunCmds(self): return [] @@ -1305,6 +1307,9 @@ def toolDep(self): def toolDepWeak(self): return self._coreStep.toolDepWeak + def getAuditFileNames(self): + return self._coreStep.auditFileNames + class CoreCheckoutStep(CoreStep): __slots__ = ( "scmList", "__checkoutUpdateIf", "__checkoutUpdateDeterministic", "__checkoutAsserts" ) @@ -1312,7 +1317,7 @@ class CoreCheckoutStep(CoreStep): def __init__(self, corePackage, checkout=None, checkoutSCMs=[], fullEnv=Env(), digestEnv=Env(), env=Env(), args=[], checkoutUpdateIf=[], checkoutUpdateDeterministic=True, - toolDep=set(), toolDepWeak=set()): + toolDep=set(), toolDepWeak=set(), auditFileNames={}): if checkout: recipeSet = corePackage.recipe.getRecipeSet() overrides = recipeSet.scmOverrides() @@ -1353,7 +1358,8 @@ def __init__(self, corePackage, checkout=None, checkoutSCMs=[], self.__checkoutUpdateIf = checkoutUpdateIf self.__checkoutUpdateDeterministic = checkoutUpdateDeterministic deterministic = corePackage.recipe.checkoutDeterministic - super().__init__(corePackage, isValid, deterministic, digestEnv, env, args, toolDep, toolDepWeak) + super().__init__(corePackage, isValid, deterministic, digestEnv, env, args, toolDep, + toolDepWeak, auditFileNames) def refDeref(self, stack, inputTools, inputSandbox, pathsConfig, cache=None): package = self.corePackage.refDeref(stack, inputTools, inputSandbox, pathsConfig) @@ -1438,10 +1444,12 @@ class CoreBuildStep(CoreStep): __slots__ = ["fingerprintMask"] def __init__(self, corePackage, script=(None, None, None), digestEnv=Env(), - env=Env(), args=[], fingerprintMask=0, toolDep=set(), toolDepWeak=set()): + env=Env(), args=[], fingerprintMask=0, toolDep=set(), toolDepWeak=set(), + auditFileNames={}): isValid = script[1] is not None self.fingerprintMask = fingerprintMask - super().__init__(corePackage, isValid, True, digestEnv, env, args, toolDep, toolDepWeak) + super().__init__(corePackage, isValid, True, digestEnv, env, args, toolDep, toolDepWeak, + auditFileNames) def refDeref(self, stack, inputTools, inputSandbox, pathsConfig, cache=None): package = self.corePackage.refDeref(stack, inputTools, inputSandbox, pathsConfig) @@ -1475,10 +1483,11 @@ class CorePackageStep(CoreStep): __slots__ = ["fingerprintMask"] def __init__(self, corePackage, script=(None, None, None), digestEnv=Env(), env=Env(), args=[], - fingerprintMask=0, toolDep=set(), toolDepWeak=set()): + fingerprintMask=0, toolDep=set(), toolDepWeak=set(), auditFileNames={}): isValid = script[1] is not None self.fingerprintMask = fingerprintMask - super().__init__(corePackage, isValid, True, digestEnv, env, args, toolDep, toolDepWeak) + super().__init__(corePackage, isValid, True, digestEnv, env, args, toolDep, toolDepWeak, + auditFileNames) def refDeref(self, stack, inputTools, inputSandbox, pathsConfig, cache=None): package = self.corePackage.refDeref(stack, inputTools, inputSandbox, pathsConfig) @@ -1553,27 +1562,27 @@ def refDeref(self, stack, inputTools, inputSandbox, pathsConfig): def createCoreCheckoutStep(self, checkout, checkoutSCMs, fullEnv, digestEnv, env, args, checkoutUpdateIf, checkoutUpdateDeterministic, - toolDep, toolDepWeak): + toolDep, toolDepWeak, auditFileNames): ret = self.checkoutStep = CoreCheckoutStep(self, checkout, checkoutSCMs, fullEnv, digestEnv, env, args, checkoutUpdateIf, checkoutUpdateDeterministic, - toolDep, toolDepWeak) + toolDep, toolDepWeak, auditFileNames) return ret def createInvalidCoreCheckoutStep(self): ret = self.checkoutStep = CoreCheckoutStep(self) return ret - def createCoreBuildStep(self, script, digestEnv, env, args, fingerprintMask, toolDep, toolDepWeak): + def createCoreBuildStep(self, script, digestEnv, env, args, fingerprintMask, toolDep, toolDepWeak, auditFileNames): ret = self.buildStep = CoreBuildStep(self, script, digestEnv, env, args, - fingerprintMask, toolDep, toolDepWeak) + fingerprintMask, toolDep, toolDepWeak, auditFileNames) return ret def createInvalidCoreBuildStep(self, args): ret = self.buildStep = CoreBuildStep(self, args=args) return ret - def createCorePackageStep(self, script, digestEnv, env, args, fingerprintMask, toolDep, toolDepWeak): - ret = self.packageStep = CorePackageStep(self, script, digestEnv, env, args, fingerprintMask, toolDep, toolDepWeak) + def createCorePackageStep(self, script, digestEnv, env, args, fingerprintMask, toolDep, toolDepWeak, auditFileNames): + ret = self.packageStep = CorePackageStep(self, script, digestEnv, env, args, fingerprintMask, toolDep, toolDepWeak, auditFileNames) return ret def getCorePackageStep(self): @@ -1855,12 +1864,8 @@ def validate(self, data): return LayerSpec(name, RecipeSet.LAYERS_SCM_SCHEMA.validate(_data)[0]) -class VarDefineValidator: +class KeyValDefineValidator: VAR_NAME = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$') - VAR_DEF = schema.Schema({ - 'value' : str, - schema.Optional("if"): schema.Or(str, IfExpression), - }) def __init__(self, keyword, conditional=True): self.__keyword = keyword @@ -1875,31 +1880,73 @@ def validate(self, data): for key,value in sorted(data.items()): if not isinstance(key, str): raise schema.SchemaUnexpectedTypeError( - "{}: bad variable '{}'. Environment variable names must be strings!" - .format(self.__keyword, key), + f"{self.__keyword}: bad variable '{key}'. Variable names must be strings!", None) if key.startswith("BOB_"): raise schema.SchemaWrongKeyError( - "{}: bad variable '{}'. Environment variables starting with 'BOB_' are reserved!" - .format(self.__keyword, key), + f"{self.__keyword}: bad variable '{key}'. Variables starting with 'BOB_' are reserved!", None) if self.VAR_NAME.match(key) is None: raise schema.SchemaWrongKeyError( - "{}: bad variable name '{}'.".format(self.__keyword, key), + f"{self.__keyword}: bad variable name '{key}'.", None) if isinstance(value, dict) and self.__conditional: self.VAR_DEF.validate(value, error=f"{self.__keyword}: {key}: invalid definition!") - data[key] = (value['value'], value.get('if')) + data[key] = self._convertItemDict(value) elif isinstance(value, str): if self.__conditional: - data[key] = (value, None) + data[key] = self._convertItemStr(value) else: raise schema.SchemaUnexpectedTypeError( - "{}: {}: bad variable definition type." - .format(self.__keyword, key), + f"{self.__keyword}: {key}: bad variable definition type.", None) return data +class VarDefineValidator(KeyValDefineValidator): + VAR_DEF = schema.Schema({ + 'value' : str, + schema.Optional("if"): schema.Or(str, IfExpression), + }) + + def _convertItemDict(self, value): + return (value['value'], value.get('if')) + + def _convertItemStr(self, value): + return (value, None) + +class AuditFile: + __slots__ = ('filename', 'condition', 'encoding') + + def __init__(self, filename, condition=None, encoding="utf-8"): + self.filename = filename + self.condition = condition + self.encoding = encoding + + def substitute(self, env): + return (env.substitute(self.filename, "filename"), + env.substitute(self.encoding, "encoding")) + +class AuditFilesValidator(KeyValDefineValidator): + VAR_DEF = schema.Schema({ + 'filename' : str, + schema.Optional("if"): schema.Or(str, IfExpression), + schema.Optional("encoding") : str, + }) + + def _convertItemDict(self, value): + return AuditFile(value['filename'], value.get('if'), value.get('encoding', "utf-8")) + + def _convertItemStr(self, value): + return AuditFile(value) + +def substituteAuditFiles(key, auditFiles, env): + try: + return { key : val.substitute(env) + for key, val in auditFiles.items() + if env.evaluate(val.condition, key) } + except ParseError as e: + raise ParseError(f"{key}: {e.slogan}") from e + RECIPE_NAME_SCHEMA = schema.Regex(r'^[0-9A-Za-z_.+-]+$') MULTIPACKAGE_NAME_SCHEMA = schema.Regex(r'^[0-9A-Za-z_.+-]*$') @@ -2170,14 +2217,17 @@ def __init__(self, recipeSet, recipe, layer, sourceFile, baseDir, packageName, b self.__varSelf = recipe.get("environment", {}) self.__varPrivate = recipe.get("privateEnvironment", {}) self.__metaEnv = recipe.get("metaEnvironment", {}) + self.__checkoutAuditFiles = recipe.get("checkoutAuditFiles", {}) self.__checkoutDeterministic = recipe.get("checkoutDeterministic") self.__checkoutVars = set(recipe.get("checkoutVars", [])) self.__checkoutVarsWeak = set(recipe.get("checkoutVarsWeak", [])) + self.__buildAuditFiles = recipe.get("buildAuditFiles", {}) self.__buildVars = set(recipe.get("buildVars", [])) self.__buildVars |= self.__checkoutVars self.__buildVarsWeak = set(recipe.get("buildVarsWeak", [])) self.__buildVarsWeak |= self.__checkoutVarsWeak self.__packageDepends = recipe.get("packageDepends") + self.__packageAuditFiles = recipe.get("packageAuditFiles", {}) self.__packageVars = set(recipe.get("packageVars", [])) self.__packageVars |= self.__buildVars self.__packageVarsWeak = set(recipe.get("packageVarsWeak", [])) @@ -2360,6 +2410,15 @@ def coDet(r): if self.__packageNetAccess is None: self.__packageNetAccess = cls.__packageNetAccess for (n, p) in self.__properties.items(): p.inherit(cls.__properties[n]) + tmp = cls.__checkoutAuditFiles.copy() + tmp.update(self.__checkoutAuditFiles) + self.__checkoutAuditFiles = tmp + tmp = cls.__buildAuditFiles.copy() + tmp.update(self.__buildAuditFiles) + self.__buildAuditFiles = tmp + tmp = cls.__packageAuditFiles.copy() + tmp.update(self.__packageAuditFiles) + self.__packageAuditFiles = tmp # the package step must always be valid if self.__package[1] is None: @@ -2798,7 +2857,8 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, srcCoreStep = p.createCoreCheckoutStep(self.__checkout, self.__checkoutSCMs, env, checkoutDigestEnv, checkoutEnv, checkoutDeps, checkoutUpdateIf, checkoutUpdateDeterministic, - toolDepCheckout, toolDepCheckoutWeak) + toolDepCheckout, toolDepCheckoutWeak, + substituteAuditFiles("checkoutAuditFiles", self.__checkoutAuditFiles, env)) else: srcCoreStep = p.createInvalidCoreCheckoutStep() @@ -2808,7 +2868,8 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, buildEnv = ( env.prune(self.__buildVars | self.__buildVarsWeak) if self.__buildVarsWeak else buildDigestEnv ) buildCoreStep = p.createCoreBuildStep(self.__build, buildDigestEnv, buildEnv, - [CoreRef(srcCoreStep)] + results, doFingerprintBuild, toolDepBuild, toolDepBuildWeak) + [CoreRef(srcCoreStep)] + results, doFingerprintBuild, toolDepBuild, toolDepBuildWeak, + substituteAuditFiles("buildAuditFiles", self.__buildAuditFiles, env)) else: buildCoreStep = p.createInvalidCoreBuildStep([CoreRef(srcCoreStep)] + results) @@ -2820,7 +2881,8 @@ def prepare(self, inputEnv, sandboxEnabled, inputStates, inputSandbox=None, if self.__packageDepends: packageDeps.extend(results) packageCoreStep = p.createCorePackageStep(self.__package, packageDigestEnv, packageEnv, - packageDeps, doFingerprint, toolDepPackage, toolDepPackageWeak) + packageDeps, doFingerprint, toolDepPackage, toolDepPackageWeak, + substituteAuditFiles("packageAuditFiles", self.__packageAuditFiles, env)) # provide environment packageCoreStep.providedEnv = env.substituteCondDict(self.__provideVars, "provideVars") @@ -4179,6 +4241,9 @@ def __createSchemas(self): schema.Use(ScriptLanguage)), schema.Optional('jobServer') : schema.Or(bool, "pipe", "fifo", "fifo-or-pipe"), schema.Optional('packageDepends') : bool, + schema.Optional('checkoutAuditFiles') : AuditFilesValidator("checkoutAuditFiles"), + schema.Optional('buildAuditFiles') : AuditFilesValidator("buildAuditFiles"), + schema.Optional('packageAuditFiles') : AuditFilesValidator("packageAuditFiles"), } for (name, prop) in self.__properties.items(): classSchemaSpec[schema.Optional(name)] = schema.Schema(prop.validate, diff --git a/pym/bob/intermediate.py b/pym/bob/intermediate.py index 19b7d4d7f..c80e4747c 100644 --- a/pym/bob/intermediate.py +++ b/pym/bob/intermediate.py @@ -96,6 +96,7 @@ def fromStep(cls, step, graph, partial=False): self.__data['scmDirectories'] = { d : (h.hex(), p) for (d, (h, p)) in step.getScmDirectories().items() } self.__data['toolKeysWeak'] = sorted(step._coreStep.toolDepWeak) self.__data['digestEnv'] = step._coreStep.digestEnv + self.__data['auditFileNames'] = step.getAuditFileNames() return self @@ -393,6 +394,9 @@ def getUpdateScriptDigest(self): h.update((key+val).encode('utf8')) return h.digest() + def getAuditFileNames(self): + return self.__data['auditFileNames'] + class PackageIR(AbstractIR): diff --git a/test/black-box/audit/config.yaml b/test/black-box/audit/config.yaml index f72863673..6877518fd 100644 --- a/test/black-box/audit/config.yaml +++ b/test/black-box/audit/config.yaml @@ -1 +1 @@ -bobMinimumVersion: "0.16" +bobMinimumVersion: "1.1" diff --git a/test/black-box/audit/extract.py b/test/black-box/audit/extract.py new file mode 100755 index 000000000..e430b8f6f --- /dev/null +++ b/test/black-box/audit/extract.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +import gzip +import io +import json +import sys + +with gzip.open(sys.argv[1], 'rb') as gzf: + tree = json.load(io.TextIOWrapper(gzf, encoding='utf8')) + print(tree["artifact"]["files"][sys.argv[2]]) diff --git a/test/black-box/audit/recipes/root.yaml b/test/black-box/audit/recipes/root.yaml index 1f5d43ed1..a147a8fc4 100644 --- a/test/black-box/audit/recipes/root.yaml +++ b/test/black-box/audit/recipes/root.yaml @@ -10,5 +10,10 @@ buildScript: | cp $1/root.txt . cp $2/foo.txt . +packageAuditFiles: + ROOT: root.txt + FOO: + filename: foo.txt + encoding: base64 packageScript: | cp $1/* . diff --git a/test/black-box/audit/run.sh b/test/black-box/audit/run.sh index 2daa9f2c5..a9b2fa675 100755 --- a/test/black-box/audit/run.sh +++ b/test/black-box/audit/run.sh @@ -31,3 +31,13 @@ test -z "$(/usr/bin/find "$archiveDir" -type f -name '*.tgz')" # Rebuild forced and upload again. Now it must create artifcats in the archive. run_bob dev -v --upload -f root test -n "$(/usr/bin/find "$archiveDir" -type f -name '*.tgz')" + +# Verify additional files in audit trail +expect_equal "$(./extract.py dev/dist/root/1/audit.json.gz ROOT)" foo +expect_equal "$(./extract.py dev/dist/root/1/audit.json.gz FOO)" "$(echo foo | base64)" + +# Provoke audit errors +expect_fail run_bob dev audit-absolute +expect_fail run_bob dev audit-missing +expect_fail run_bob dev audit-encoding-error +expect_fail run_bob dev audit-invalid-encoding diff --git a/test/unit/test_input_recipeset.py b/test/unit/test_input_recipeset.py index 2d37ffd11..aa79bfdca 100644 --- a/test/unit/test_input_recipeset.py +++ b/test/unit/test_input_recipeset.py @@ -2615,3 +2615,139 @@ def testPrecedence(self): ScmOverride({ "set" : { "url" : "include_l1" }}), ScmOverride({ "set" : { "url" : "include_l2" }}) ], cfg.scmOverrides()) + +class TestAuditFiles(RecipesTmp, TestCase): + """Test packageAuditFiles """ + + def testSimple(self): + """Simple packageAuditFiles""" + self.writeRecipe("root", """\ + root: True + checkoutScript: "true" + checkoutAuditFiles: + CHECKOUT: some + buildScript: "true" + buildAuditFiles: + BUILD: files + packageAuditFiles: + FOO: foo + BAR: bar + """) + + p = self.generate().walkPackagePath("root") + + af = p.getCheckoutStep().getAuditFileNames() + self.assertEqual(set(af.keys()), {"CHECKOUT"}) + self.assertEqual(af["CHECKOUT"], ("some", "utf-8")) + + af = p.getBuildStep().getAuditFileNames() + self.assertEqual(set(af.keys()), {"BUILD"}) + self.assertEqual(af["BUILD"], ("files", "utf-8")) + + for name, (filename, encoding) in p.getPackageStep().getAuditFileNames().items(): + self.assertIn(name, {"FOO", "BAR"}) + self.assertEqual(name.lower(), filename) + self.assertEqual(encoding, "utf-8") + + def testInherit(self): + """Inherited classes are merged on a key-by-key basis""" + self.writeRecipe("root", """\ + root: True + inherit: [cls] + checkoutScript: "true" + buildScript: "true" + buildAuditFiles: + BUILD: files + packageAuditFiles: + BAR: bar + """) + self.writeClass("cls", """\ + checkoutAuditFiles: + CHECKOUT: some + buildAuditFiles: + BUILD: xxxxx + packageAuditFiles: + FOO: "$FOO" + BAR: baz + """) + + p = self.generate(env={"FOO" : "foo"}).walkPackagePath("root") + + af = p.getCheckoutStep().getAuditFileNames() + self.assertEqual(set(af.keys()), {"CHECKOUT"}) + self.assertEqual(af["CHECKOUT"], ("some", "utf-8")) + + af = p.getBuildStep().getAuditFileNames() + self.assertEqual(set(af.keys()), {"BUILD"}) + self.assertEqual(af["BUILD"], ("files", "utf-8")) + + for name, (filename, encoding) in p.getPackageStep().getAuditFileNames().items(): + self.assertIn(name, {"FOO", "BAR"}) + self.assertEqual(name.lower(), filename) + self.assertEqual(encoding, "utf-8") + + def testCustomEncoding(self): + """A custom encoding can be set per audit file""" + self.writeRecipe("root", """\ + root: True + packageAuditFiles: + FOO: + filename: foo + encoding: custom + """) + + p = self.generate().walkPackagePath("root") + filename, encoding = p.getPackageStep().getAuditFileNames()["FOO"] + self.assertEqual(filename, "foo") + self.assertEqual(encoding, "custom") + + def testSubstitution(self): + """Audit filename and encoding are string substituted""" + self.writeRecipe("root", """\ + root: True + packageAuditFiles: + FOO: + filename: "${FN:-foo}" + encoding: "${ENC:-utf8}" + """) + + p = self.generate().walkPackagePath("root") + filename, encoding = p.getPackageStep().getAuditFileNames()["FOO"] + self.assertEqual(filename, "foo") + self.assertEqual(encoding, "utf8") + + p = self.generate(env={"FN" : "bar", "ENC" : "binary"}).walkPackagePath("root") + filename, encoding = p.getPackageStep().getAuditFileNames()["FOO"] + self.assertEqual(filename, "bar") + self.assertEqual(encoding, "binary") + + def testConditional(self): + self.writeRecipe("root", """\ + root: True + packageAuditFiles: + FOO: + filename: foo + if: "${FOO}" + BAR: + filename: bar + if: !expr >- + "$BAR" == "enabled" + """) + + with self.assertRaises(ParseError): + self.generate().walkPackagePath("root") + + p = self.generate(env={"FOO" : "0"}).walkPackagePath("root") + self.assertEqual(set(p.getPackageStep().getAuditFileNames().keys()), set()) + + p = self.generate(env={"FOO" : "1"}).walkPackagePath("root") + self.assertEqual(set(p.getPackageStep().getAuditFileNames().keys()), {'FOO'}) + filename, encoding = p.getPackageStep().getAuditFileNames()["FOO"] + self.assertEqual(filename, "foo") + self.assertEqual(encoding, "utf-8") + + p = self.generate(env={"FOO" : "true", "BAR" : "enabled"}).walkPackagePath("root") + self.assertEqual(set(p.getPackageStep().getAuditFileNames().keys()), {'FOO', 'BAR'}) + filename, encoding = p.getPackageStep().getAuditFileNames()["BAR"] + self.assertEqual(filename, "bar") + self.assertEqual(encoding, "utf-8")