feat: build bldr for linux/amd64 and linux/arm64

`bldr` frontend auto-detects build/target platform and adjusts for it.

Add integration tests for multi-platform builds.

Bump Go, golangci-lint, dependencies, etc. Clean up linting errors.

Signed-off-by: Andrey Smirnov <[email protected]>

Author: smira

.drone.yml

@@ -27,11 +27,17 @@ steps:
     image: autonomy/build-container:latest
     pull: always
     environment:
-      REGISTRY: registry.ci.svc:5000
-      PLATFORM: linux/amd64
-      BUILDKIT_HOST: ${BUILDKIT_HOST=tcp://buildkitd.ci.svc:1234}
+      PLATFORM: linux/amd64,linux/arm64
+      SSH_KEY:
+        from_secret: ssh_key
+      DOCKER_USERNAME:
+        from_secret: docker_username
+      DOCKER_PASSWORD:
+        from_secret: docker_password
     commands:
-      - docker buildx create --driver docker-container --platform linux/amd64 --name local --config hack/buildkit.conf --use
+      - install-ci-key
+      - setup-buildx-amd64-arm64
+      - docker login --username "$${DOCKER_USERNAME}" --password "$${DOCKER_PASSWORD}"
       - make # build everything
       - make frontend PUSH=true  # push only frontend for integration tests
       - make integration
@@ -47,14 +53,15 @@ steps:
     image: autonomy/build-container:latest
     pull: always
     environment:
-      PLATFORM: linux/amd64
+      PLATFORM: linux/amd64,linux/arm64
       BUILDKIT_HOST: ${BUILDKIT_HOST=tcp://buildkitd.ci.svc:1234}
       DOCKER_USERNAME:
         from_secret: docker_username
       DOCKER_PASSWORD:
         from_secret: docker_password
     commands:
-      - docker buildx create --driver docker-container --platform linux/amd64 --name local --use
+      - install-ci-key
+      - setup-buildx-amd64-arm64
       - docker login --username "$${DOCKER_USERNAME}" --password "$${DOCKER_PASSWORD}"
       - make PUSH=true
       - make integration

Dockerfile

@@ -1,11 +1,11 @@
 # syntax = docker/dockerfile-upstream:1.1.2-experimental
 
-FROM golang:1.13.6-alpine AS base
+FROM golang:1.14-alpine AS base
 ENV GO111MODULE on
 ENV GOPROXY https://proxy.golang.org
 ENV CGO_ENABLED 0
 RUN apk --update --no-cache add bash curl
-RUN curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b /bin v1.20.0
+RUN curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b /bin v1.27.0
 WORKDIR /src
 COPY ./go.mod ./
 COPY ./go.sum ./

Makefile

@@ -4,7 +4,8 @@ TAG ?= $(shell git describe --tag --always --dirty)
 REGISTRY_AND_USERNAME := $(REGISTRY)/$(USERNAME)
 RUN_TESTS ?= TestIntegration
 
-PLATFORM ?= linux/amd64
+BUILD_PLATFORM ?= linux/amd64
+PLATFORM ?= linux/amd64,linux/arm64
 PROGRESS ?= auto
 PUSH ?= false
 
@@ -55,4 +56,4 @@ frontend:
 
 .PHONY: integration
 integration: integration.test bldr
-	cd internal/pkg/integration && PATH="$$PWD/../../../out:$$PATH"  integration.test -test.v -test.run $(RUN_TESTS)
+	cd internal/pkg/integration && PATH="$$PWD/../../../out/$(subst /,_,$(BUILD_PLATFORM)):$$PATH"  integration.test -test.v -test.run $(RUN_TESTS)

cmd/frontend.go

@@ -15,7 +15,7 @@ import (
 	"github.com/talos-systems/bldr/internal/pkg/pkgfile"
 )
 
-// frontendCmd represents the frontend command
+// frontendCmd represents the frontend command.
 var frontendCmd = &cobra.Command{
 	Use:   "frontend",
 	Short: "Buildkit frontend for Pkgfile",

cmd/graph.go

@@ -12,7 +12,7 @@ import (
 	"github.com/talos-systems/bldr/internal/pkg/solver"
 )
 
-// graphCmd represents the graph command
+// graphCmd represents the graph command.
 var graphCmd = &cobra.Command{
 	Use:   "graph",
 	Short: "Graph dependencies between pkgs",

cmd/llb.go

@@ -15,7 +15,7 @@ import (
 	"github.com/talos-systems/bldr/internal/pkg/solver"
 )
 
-// llbCmd represents the llb command
+// llbCmd represents the llb command.
 var llbCmd = &cobra.Command{
 	Use:   "llb",
 	Short: "Dump buildkit LLB for the build",
@@ -38,12 +38,7 @@ and outputs buildkit LLB to stdout. This can be used as 'bldr pack ... | buildct
 			log.Fatal(err)
 		}
 
-		out, err := convert.BuildLLB(graph, options)
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		dt, err := out.Marshal(options.BuildPlatform.LLBPlatform)
+		dt, err := convert.MarshalLLB(graph, options)
 		if err != nil {
 			log.Fatal(err)
 		}

cmd/root.go

@@ -8,11 +8,14 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"runtime"
 
 	"github.com/spf13/cobra"
 	"github.com/talos-systems/bldr/internal/pkg/environment"
 )
 
+const defaultPlatform = (runtime.GOOS + "/" + runtime.GOARCH)
+
 var (
 	pkgRoot string
 	options = &environment.Options{
@@ -21,7 +24,7 @@ var (
 	}
 )
 
-// rootCmd represents the base command when called without any subcommands
+// rootCmd represents the base command when called without any subcommands.
 var rootCmd = &cobra.Command{
 	Use:   "bldr",
 	Short: "A tool to build and manage software via Pkgfile and pkg.yaml",
@@ -35,6 +38,7 @@ output LLB directly which is useful for development or debugging.`,
 }
 
 // Execute adds all child commands to the root command and sets flags appropriately.
+//
 // This is called by main.main(). It only needs to happen once to the rootCmd.
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
@@ -46,4 +50,7 @@ func Execute() {
 func init() {
 	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
 	rootCmd.PersistentFlags().StringVarP(&pkgRoot, "root", "", ".", "The path to a pkg root")
+
+	options.BuildPlatform.Set(defaultPlatform)  //nolint: errcheck
+	options.TargetPlatform.Set(defaultPlatform) //nolint: errcheck
 }

cmd/validate.go

@@ -11,7 +11,7 @@ import (
 	"github.com/talos-systems/bldr/internal/pkg/solver"
 )
 
-// validateCmd represents the graph command
+// validateCmd represents the validate command.
 var validateCmd = &cobra.Command{
 	Use:   "validate",
 	Short: "Validate syntax of pkg.yaml files",

go.mod

@@ -1,26 +1,27 @@
 module github.com/talos-systems/bldr
 
-go 1.13
+go 1.14
 
 require (
 	github.com/Masterminds/goutils v1.1.0 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible
-	github.com/alessio/shellescape v0.0.0-20190409004728-b115ca0f9053
-	github.com/emicklei/dot v0.10.1
-	github.com/google/uuid v1.1.1 // indirect
-	github.com/hashicorp/go-multierror v1.0.0
-	github.com/huandu/xstrings v1.2.0 // indirect
+	github.com/alessio/shellescape v1.2.2
+	github.com/containerd/containerd v1.4.0-0
+	github.com/emicklei/dot v0.11.0
+	github.com/hashicorp/go-multierror v1.1.0
+	github.com/huandu/xstrings v1.3.1 // indirect
 	github.com/mitchellh/copystructure v1.0.0 // indirect
-	github.com/moby/buildkit v0.6.2-0.20190921015714-10cef0c6e178
-	github.com/opencontainers/go-digest v1.0.0-rc1
-	github.com/otiai10/copy v1.0.2
-	github.com/spf13/cobra v0.0.4
-	golang.org/x/text v0.3.2 // indirect
-	golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7
-	gopkg.in/yaml.v2 v2.2.2
+	github.com/moby/buildkit v0.7.1
+	github.com/opencontainers/go-digest v1.0.0
+	github.com/opencontainers/image-spec v1.0.1
+	github.com/otiai10/copy v1.2.0
+	github.com/spf13/cobra v1.0.0
+	golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e
+	gopkg.in/yaml.v2 v2.3.0
 )
 
-replace github.com/docker/docker v1.14.0-0.20190319215453-e7b5f7dbe98c => github.com/docker/docker v1.4.2-0.20190319215453-e7b5f7dbe98c
-
-replace golang.org/x/crypto v0.0.0-20190129210102-0709b304e793 => golang.org/x/crypto v0.0.0-20180904163835-0709b304e793
+replace (
+	github.com/containerd/containerd => github.com/containerd/containerd v1.3.1-0.20200512144102-f13ba8f2f2fd
+	github.com/docker/docker => github.com/docker/docker v17.12.0-ce-rc1.0.20200310163718-4634ce647cf2+incompatible
+)