feat: optimize build time by removing unnecessary I/O

This PR contains two optimizations:

* reorder steps to make variant + dependencies layers cacheable
* promote first dependency to base image for 'scratch' builds

First optimization is pretty simple, given that many packages contain
same top dependencies:

```yaml
dependencies:
  - stage: base
```

It makes sure LLB is ordered so that base part of the LLB up to the
dependencies is identical across packages builds and can be eliminated
to a single instance.

Second optimization is more tricky, for `scratch` packages:

```yaml
variant: scratch
dependencies:
  - stage: tools
  - stage: libressl
```

It promotes `tools` to be base image (instead of scratch). To compare,
in `Dockerfile` syntax it means replacing:

```
FROM scratch
COPY --from=tools / /
COPY --from=libressl / /
```

with:

```
FROM tools
COPY --from=libressl / /
```

If the `tools` layer is huge (which is often the case, in order of 0.5
GiB), it removes a lot of file copy operations and improves build time.

Signed-off-by: Andrey Smirnov <[email protected]>

Author: smira

internal/pkg/convert/graph.go

@@ -27,9 +27,12 @@ type GraphLLB struct {
 	Checksummer  llb.State
 	LocalContext llb.State
 
-	cache map[*solver.PackageNode]llb.State
+	baseImageProcessor llbProcessor
+	cache              map[*solver.PackageNode]llb.State
 }
 
+type llbProcessor func(llb.State) llb.State
+
 // NewGraphLLB creates new GraphLLB and initializes shared images.
 func NewGraphLLB(graph *solver.PackageGraph, options *environment.Options) *GraphLLB {
 	result := &GraphLLB{
@@ -72,7 +75,11 @@ func (graph *GraphLLB) buildBaseImages() {
 		return root
 	}
 
-	graph.BaseImages[v1alpha2.Alpine] = addEnv(addPkg(llb.Image(
+	graph.baseImageProcessor = func(root llb.State) llb.State {
+		return addEnv(addPkg(root))
+	}
+
+	graph.BaseImages[v1alpha2.Alpine] = graph.baseImageProcessor(llb.Image(
 		constants.DefaultBaseImage,
 		llb.WithCustomName(graph.Options.CommonPrefix+"base"),
 	).Run(
@@ -81,9 +88,9 @@ func (graph *GraphLLB) buildBaseImages() {
 	).Run(
 		llb.Args([]string{"ln", "-svf", "/bin/bash", "/bin/sh"}),
 		llb.WithCustomName(graph.Options.CommonPrefix+"base-symlink"),
-	).Root()))
+	).Root())
 
-	graph.BaseImages[v1alpha2.Scratch] = addEnv(addPkg(llb.Scratch()))
+	graph.BaseImages[v1alpha2.Scratch] = graph.baseImageProcessor(llb.Scratch())
 }
 
 func (graph *GraphLLB) buildChecksummer() {

internal/pkg/convert/node.go

@@ -33,6 +33,8 @@ type NodeLLB struct {
 
 	Graph  *GraphLLB
 	Prefix string
+
+	promotedDependency string
 }
 
 // NewNodeLLB wraps PackageNode for LLB conversion.
@@ -45,8 +47,18 @@ func NewNodeLLB(node *solver.PackageNode, graph *GraphLLB) *NodeLLB {
 	}
 }
 
-func (node *NodeLLB) base() llb.State {
-	return node.Graph.BaseImages[node.Pkg.Variant]
+func (node *NodeLLB) base() (llb.State, error) {
+	if node.Pkg.Variant == v1alpha2.Scratch && len(node.Dependencies) > 0 {
+		// pull the first dependency as base image if the package build is from scratch
+		promotedDep := node.Dependencies[0]
+		node.promotedDependency = promotedDep.ID()
+
+		depState, _, err := node.convertDependency(promotedDep)
+
+		return node.Graph.baseImageProcessor(depState), err
+	}
+
+	return node.Graph.BaseImages[node.Pkg.Variant], nil
 }
 
 func (node *NodeLLB) install(root llb.State) llb.State {
@@ -70,6 +82,22 @@ func (node *NodeLLB) context(root llb.State) llb.State {
 	)
 }
 
+func (node *NodeLLB) convertDependency(dep solver.PackageDependency) (depState llb.State, srcName string, err error) {
+	if dep.IsInternal() {
+		depState, err = NewNodeLLB(dep.Node, node.Graph).Build()
+		if err != nil {
+			return llb.Scratch(), "", err
+		}
+
+		srcName = dep.Node.Name
+	} else {
+		depState = llb.Image(dep.Image)
+		srcName = dep.Image
+	}
+
+	return
+}
+
 func (node *NodeLLB) dependencies(root llb.State) (llb.State, error) {
 	deps := make([]solver.PackageDependency, 0, len(node.Dependencies))
 
@@ -96,28 +124,19 @@ func (node *NodeLLB) dependencies(root llb.State) (llb.State, error) {
 
 		seen[dep.ID()] = struct{}{}
 
-		var (
-			depState llb.State
-			srcName  string
-		)
-
-		if dep.IsInternal() {
-			var err error
-
-			depState, err = NewNodeLLB(dep.Node, node.Graph).Build()
-			if err != nil {
-				return llb.Scratch(), err
-			}
+		if node.promotedDependency == dep.ID() {
+			// dependency promoted as base image, skip it
+			continue
+		}
 
-			srcName = dep.Node.Name
-		} else {
-			depState = llb.Image(dep.Image)
-			srcName = dep.Image
+		depState, srcName, err := node.convertDependency(dep)
+		if err != nil {
+			return llb.Scratch(), err
 		}
 
 		root = root.File(
 			llb.Copy(depState, dep.Src(), dep.Dest(), defaultCopyOptions),
-			llb.WithCustomNamef(node.Prefix+"copy --from %s %s -> %s", srcName, dep.Src(), dep.Dest()))
+			llb.WithCustomNamef("copy --from %s %s -> %s", srcName, dep.Src(), dep.Dest()))
 	}
 
 	return root, nil
@@ -229,21 +248,23 @@ func (node *NodeLLB) finalize(root llb.State) llb.State {
 
 // Build converts PackageNode to buildkit LLB.
 func (node *NodeLLB) Build() (llb.State, error) {
-	var err error
-
 	if state, ok := node.Graph.cache[node.PackageNode]; ok {
 		return state, nil
 	}
 
-	root := node.base()
-	root = node.install(root)
-	root = node.context(root)
+	root, err := node.base()
+	if err != nil {
+		return llb.Scratch(), err
+	}
 
 	root, err = node.dependencies(root)
 	if err != nil {
 		return llb.Scratch(), err
 	}
 
+	root = node.install(root)
+	root = node.context(root)
+
 	for i, step := range node.Pkg.Steps {
 		root = node.step(root, i, step)
 	}