Merge pull request #599 from jkloetzke/managed-layers-audit

Add managed layers to audit trail

Author: jkloetzke

pym/bob/audit.py

@@ -42,6 +42,8 @@ def digestData(d, h):
     elif isinstance(d, bytes):
         h.update(struct.pack("<BI", 6, len(d)))
         h.update(d)
+    elif d is None:
+        h.update(struct.pack("<B", 7))
     else:
         assert False, "Cannot digest " + str(type(d))
 
@@ -80,6 +82,7 @@ class Artifact:
         schema.Optional('metaEnv') : { schema.Optional(str) : str },
         "scms" : [ dict ],
         schema.Optional("recipes") : dict,
+        schema.Optional("layers") : dict,
         "dependencies" : {
             schema.Optional('args') : [ HexValidator() ],
             schema.Optional('tools') : { str : HexValidator() },
@@ -122,6 +125,7 @@ def reset(self, variantId, buildId, resultHash):
         self.__buildId = buildId
         self.__resultHash = resultHash
         self.__recipes = None
+        self.__layers = {}
         self.__defines = {}
         u = platform.uname()
         self.__build = {
@@ -155,6 +159,13 @@ def load(self, data):
         else:
             self.__recipes = None
 
+        layers = data.get("layers")
+        if layers:
+            self.__layers = { name : (audit and auditFromData(audit))
+                              for name, audit in layers.items() }
+        else:
+            self.__layers = {}
+
         self.__defines = data["meta"]
         self.__build = data["build"]
         self.__env = data["env"]
@@ -206,11 +217,18 @@ def __dump(self):
         if self.__recipes is not None:
             ret["recipes"] = self.__recipes.dump()
 
+        if self.__layers: # Explicitly filter empty layers
+            ret["layers"] = { name : (audit and audit.dump())
+                              for name, audit in self.__layers.items() }
+
         return ret
 
     def setRecipes(self, recipes):
         self.__recipes = recipes
 
+    def setLayers(self, layers):
+        self.__layers = layers
+
     def setEnv(self, env):
         try:
             with open(env) as f:
@@ -338,7 +356,8 @@ def load(self, file, name):
                 r["artifact-id"] : Artifact.fromData(r) for r in tree["references"]
             }
         except schema.SchemaError as e:
-            raise ParseError(name + ": Invalid audit record: " + str(e))
+            raise ParseError(name + ": Invalid audit record: " + str(e),
+                             help="Try updating to the latest Bob version. The audit probably contains new record types.")
         except ValueError as e:
             raise ParseError(name + ": Invalid json: " + str(e))
         self.__validate()
@@ -385,11 +404,11 @@ def getReferencedBuildIds(self):
                 refs.update(artifact.getReferences())
         return sorted(ret)
 
-    def setRecipesAudit(self, recipes):
-        self.__artifact.setRecipes(recipes)
-
-    def setRecipesData(self, xml):
-        self.__artifact.setRecipes(auditFromData(xml))
+    def setRecipesAudit(self, recipesAudit):
+        self.__artifact.setRecipes(recipesAudit.get(""))
+        self.__artifact.setLayers({
+            layer : audit for layer, audit in recipesAudit.items() if layer != ""
+        })
 
     def setEnv(self, env):
         self.__artifact.setEnv(env)

pym/bob/cmds/graph.py

@@ -59,7 +59,7 @@ def getColorId(package, level):
            yield ('link', d.getPackage(), package)
         yield from findPackages(d.getPackage(), excludes, highlights, done, maxdepth, donePackages, level+1)
 
-def makeD3Graph(packages, p, filename, options, excludes, highlights, maxdepth):
+def makeD3Graph(recipes, packages, p, filename, options, excludes, highlights, maxdepth):
     def getHover(package, options):
         hover = collections.OrderedDict(package.getMetaEnv())
         if options.get('d3.showScm', False) and package.getCheckoutStep().isValid():
@@ -469,7 +469,7 @@ def getHover(package, options):
 simulation.force("link").links(links)
 </script>
 <div class='info' style="width: 100%; text-align: center;">
-  <div id='innerLeft' style="float: left"> Recipes: """ + runInEventLoop(RecipeSet().getScmStatus()) + """</div>
+  <div id='innerLeft' style="float: left"> Recipes: """ + runInEventLoop(recipes.getScmStatus()) + """</div>
   <div id='innerRight' style="float: right">Bob version: """ + BOB_VERSION +"""</div>
   <div id='innerMiddle' style="display: inline-block">Generated using <a href="https://www.d3js.org">D3JS</a></div>
 </div>
@@ -584,6 +584,6 @@ def doGraph(argv, bobRoot):
         print(colorize("   GRAPH   {} ({})".format(p, args.type), "32"))
 
         if args.type == 'd3':
-            makeD3Graph(packages, p, os.path.join(destination, filename), options, excludes, highlights, args.max_depth)
+            makeD3Graph(recipes, packages, p, os.path.join(destination, filename), options, excludes, highlights, args.max_depth)
         elif args.type == 'dot':
             makeDotGraph(packages, p, os.path.join(destination, filename), excludes, highlights, args.max_depth)

pym/bob/cmds/jenkins/exec.py

@@ -86,7 +86,7 @@ def recipesAudit(self):
         if self.__recipesAudit:
             return json.loads("".join(self.__recipesAudit))
         else:
-            return None
+            return {}
 
     @property
     def execIR(self):

pym/bob/cmds/jenkins/intermediate.py

@@ -74,7 +74,8 @@ class PartialRecipeSet(PartialIRBase, RecipeSetIR):
     @classmethod
     def fromData(cls, data, scmAudit):
         self = super(PartialRecipeSet, cls).fromData(data)
-        self.__scmAudit = scmAudit and auditFromData(scmAudit)
+        self.__scmAudit = scmAudit and { name : (audit and auditFromData(audit))
+                                         for name, audit in scmAudit.items() }
         return self
 
     async def getScmAudit(self):
@@ -94,7 +95,7 @@ def __init__(self):
         self.packages = {}
         self.recipes = {}
         self.recipeSet = None
-        self.scmAudit = None
+        self.scmAudit = {}
 
     @classmethod
     def fromData(cls, data):

pym/bob/cmds/jenkins/jenkins.py

@@ -1281,8 +1281,10 @@ def doJenkinsPush(recipes, argv):
     date = str(datetime.datetime.now())
 
     recipesAudit = runInEventLoop(recipes.getScmAudit())
-    if recipesAudit is not None:
-        recipesAudit = json.dumps(recipesAudit.dump(), sort_keys=True)
+    if recipesAudit:
+        recipesAudit = json.dumps({ name : (audit and audit.dump())
+                                    for name, audit in recipesAudit.items() },
+                                  sort_keys=True)
 
     def printLine(level, job, *args):
         if level <= verbose:

pym/bob/input.py

@@ -8,7 +8,7 @@
 from .languages import getLanguage, ScriptLanguage, BashLanguage, PwshLanguage
 from .pathspec import PackageSet
 from .scm import CvsScm, GitScm, ImportScm, SvnScm, UrlScm, ScmOverride, \
-    auditFromDir, getScm, SYNTHETIC_SCM_PROPS
+    auditFromDir, auditFromProperties, getScm, SYNTHETIC_SCM_PROPS
 from .state import BobState
 from .stringparser import checkGlobList, Env, DEFAULT_STRING_FUNS, IfExpression
 from .tty import InfoOnce, Warn, WarnOnce, setColorMode, setParallelTUIThreshold
@@ -1739,13 +1739,28 @@ def getScm(self):
         return self.__scm
 
 class LayerValidator:
+    @staticmethod
+    def __validateName(name):
+        if name == "":
+            raise schema.SchemaError("Layer name must not be empty")
+        if any((c in name) for c in '\\/'):
+            raise schema.SchemaError("Invalid character in layer name")
+
     def validate(self, data):
-        if isinstance(data,str):
+        if isinstance(data, str):
+            self.__validateName(data)
             return LayerSpec(data)
+        elif not isinstance(data, dict):
+            raise schema.SchemaUnexpectedTypeError("Layer entry must be a string or a dict", None)
+
         if 'name' not in data:
             raise schema.SchemaMissingKeyError("Missing 'name' key in {}".format(data), None)
+        elif not isinstance(data['name'], str):
+            raise schema.SchemaUnexpectedTypeError("Layer name must be a string", None)
+
         _data = data.copy();
         name = _data.get('name')
+        self.__validateName(name)
         del _data['name']
 
         return LayerSpec(name, RecipeSet.LAYERS_SCM_SCHEMA.validate(_data)[0])
@@ -2957,11 +2972,12 @@ class RecipeSet:
         })
 
     # We do not support the "import" SCM for layers. It just makes no sense.
+    # Also, all SCMs lack the "dir" and "if" attributes.
     LAYERS_SCM_SCHEMA = ScmValidator({
-        'git' : GitScm.SCHEMA,
-        'svn' : SvnScm.SCHEMA,
-        'cvs' : CvsScm.SCHEMA,
-        'url' : UrlScm.SCHEMA,
+        'git' : GitScm.LAYERS_SCHEMA,
+        'svn' : SvnScm.LAYERS_SCHEMA,
+        'cvs' : CvsScm.LAYERS_SCHEMA,
+        'url' : UrlScm.LAYERS_SCHEMA,
     })
 
     SCM_SCHEMA = ScmValidator({
@@ -3101,7 +3117,7 @@ def __init__(self):
         self.__commandConfig = {}
         self.__uiConfig = {}
         self.__shareConfig = {}
-        self.__layers = []
+        self.__layers = {}
         self.__buildHooks = {}
         self.__sandboxOpts = {}
         self.__scmDefaults = {}
@@ -3424,20 +3440,34 @@ async def getScmAudit(self):
         try:
             ret = self.__recipeScmAudit
         except AttributeError:
+            ret = {}
             try:
-                ret = await auditFromDir(".")
+                ret[""] = await auditFromDir(".")
             except BobError as e:
                 Warn("could not determine recipes state").warn(e.slogan)
-                ret = None
+
+            # We look into every layers directory. If the Bob state knows it,
+            # use the SCM information from there. Otherwise make a best guess.
+            for layer, path in sorted(self.__layers.items()):
+                state = None
+                try:
+                    scmProps = (BobState().getLayerState(path) or {}).get("prop")
+                    if scmProps is None:
+                        state = await auditFromDir(path)
+                    else:
+                        state = await auditFromProperties(path, scmProps)
+                except BobError as e:
+                    Warn(f"could not determine layer '{layer}' state").warn(e.slogan)
+                ret[layer] = state
+
             self.__recipeScmAudit = ret
         return ret
 
     async def getScmStatus(self):
-        audit = await self.getScmAudit()
-        if audit is None:
-            return "unknown"
-        else:
-            return audit.getStatusLine()
+        scmAudit = await self.getScmAudit()
+        return ", ".join(( ((f"layer {name}: " if name else "")
+                            + ("unknown" if audit is None else audit.getStatusLine()))
+                           for name, audit in sorted(scmAudit.items()) ))
 
     def getBuildHook(self, name):
         return self.__buildHooks.get(name)
@@ -3481,7 +3511,7 @@ def __parse(self, envOverrides, platform, recipesRoot=""):
         if platform not in ('cygwin', 'darwin', 'linux', 'msys', 'win32'):
             raise ParseError("Invalid platform: " + platform)
         self.__platform = platform
-        self.__layers = []
+        self.__layers = {}
         self.__whiteList = getPlatformEnvWhiteList(platform)
         self.__pluginPropDeps = b''
         self.__pluginSettingsDeps = b''
@@ -3585,7 +3615,6 @@ def __parseLayer(self, layerSpec, maxVer, recipesRoot, upperLayer):
 
             if layer in self.__layers:
                 return
-            self.__layers.append(layer)
 
             if managedLayers:
                 # SCM backed layers are in build dir, regular layers are in
@@ -3602,6 +3631,8 @@ def __parseLayer(self, layerSpec, maxVer, recipesRoot, upperLayer):
                                                        for l in layer.split("/") ))
                 if not os.path.isdir(rootDir):
                     raise ParseError(f"Layer '{layer}' does not exist!")
+
+            self.__layers[layer] = rootDir
         else:
             rootDir = recipesRoot
 

pym/bob/layers.py

@@ -100,7 +100,6 @@ def __init__(self, name, upperConfig, defines, projectRoot, scm=None):
     async def __checkoutTask(self, verbose, attic):
         if self.__scm is None:
             return
-        dir = self.__scm.getProperties(False).get("dir")
 
         invoker = Invoker(spec=LayerStepSpec(self.__layerDir, self.__upperConfig.envWhiteList()),
                           preserveEnv= False,

pym/bob/scm/__init__.py

@@ -21,6 +21,21 @@ async def auditFromDir(dir):
     else:
         return None
 
+async def auditFromProperties(baseDir, props):
+    auditSpec = getScm(props).getAuditSpec()
+    if auditSpec is None:
+        return None
+
+    SCMS = {
+        'git' : GitAudit,
+        'svn' : SvnAudit,
+        'url' : UrlAudit,
+        'import' : ImportAudit,
+    }
+
+    (typ, subDir, extra) = auditSpec
+    return await SCMS[typ].fromDir(baseDir, subDir, extra)
+
 def auditFromData(data):
     typ = data.get("type")
     if typ == "git":

pym/bob/scm/cvs.py

@@ -19,11 +19,17 @@ class CvsScm(Scm):
         'scm' : 'cvs',
         'cvsroot' : str,
         'module' : str,
-        schema.Optional('if') : str,
         schema.Optional('rev') : str
     }
 
-    SCHEMA = schema.Schema({**__SCHEMA, **DEFAULTS})
+    SCHEMA = schema.Schema({
+        **__SCHEMA,
+        **DEFAULTS,
+        schema.Optional('if') : str,
+    })
+
+    # Layers have no "dir" and no "if"
+    LAYERS_SCHEMA = schema.Schema({ **__SCHEMA })
 
     # Checkout using CVS
     # - mandatory parameters: cvsroot, module