Skip to content

tools: use docker compose and quiet for pulling when using docker or docker compose #8594

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 6 commits into
base: main
Choose a base branch
from
Open

tools: use docker compose and quiet for pulling when using docker or docker compose #8594

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0b6c617
tools: use docker compose and quiet for pulling
v1v Jun 19, 2025
f6beb7a
-q, --quiet Suppress the pull output
v1v Jun 19, 2025
4abceb9
Update dev-tools/mage/integtest_docker.go
v1v Jun 19, 2025
5b2cb7e
fix other docker commands
v1v Jun 19, 2025
887bf77
Merge branch 'feature/remove-verbose-pulling-containers' of https://g…
v1v Jun 19, 2025
512ba55
Apply suggestions from code review
v1v Jun 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 0 additions & 10 deletions dev-tools/mage/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -209,16 +209,6 @@
return &info, nil
}

// HaveDockerCompose returns an error if docker-compose is not found on the
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Assuming docker compose is available, so by validating docker I think we can remove this condition. It's not available, it will fail regardless.

// PATH.
func HaveDockerCompose() error {
_, err := exec.LookPath("docker-compose")
if err != nil {
return fmt.Errorf("docker-compose is not available")
}
return nil
}

// HaveKubectl returns an error if kind is not found on the PATH.
func HaveKubectl() error {
_, err := exec.LookPath("kubectl")
Expand Down Expand Up @@ -461,7 +451,7 @@

switch header.Typeflag {
case tar.TypeDir:
if err = os.MkdirAll(path, os.FileMode(header.Mode)); err != nil {

Check failure on line 454 in dev-tools/mage/common.go

View workflow job for this annotation

GitHub Actions / lint (ubuntu-latest) 

G115: integer overflow conversion int64 -> uint32 (gosec)
return err
}
case tar.TypeReg:
Expand All @@ -479,7 +469,7 @@
return err
}

if err = os.Chmod(path, os.FileMode(header.Mode)); err != nil {

Check failure on line 472 in dev-tools/mage/common.go

View workflow job for this annotation

GitHub Actions / lint (ubuntu-latest) 

G115: integer overflow conversion int64 -> uint32 (gosec)
return err
}

Expand Down
2 changes: 1 addition & 1 deletion dev-tools/mage/crossbuild.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ func (b GolangCrossBuilder) Build() error {
// container. So we ensure that all path separators are `/`.
buildCmd = filepath.ToSlash(buildCmd)

dockerRun := sh.RunCmd("docker", "run")
dockerRun := sh.RunCmd("docker", "run", "--quiet")
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

According to the docs:

$ docker run --help
...
      --pull string                      Pull image before running ("always", "missing", "never") (default "missing")
      -q, --quiet                            Suppress the pull output

See https://docs.docker.com/reference/cli/docker/container/run/

image

image, err := b.ImageSelector(b.Platform)
if err != nil {
return fmt.Errorf("failed to determine golang-crossbuild image tag: %w", err)
Expand Down
3 changes: 2 additions & 1 deletion dev-tools/mage/dockerbuilder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -217,7 +217,7 @@ func (b *dockerBuilder) expandDockerfile(templatesDir string, data map[string]in
return nil
}

// dockerBuild runs "docker build -t t1 -t t2 ... buildDir"
// dockerBuild runs "docker build --progress=plain -t t1 -t t2 ... buildDir"
// returns the main tag additional tags if specified as part of extra_tags property
// the extra tags are not push to the registry from b.ExtraVars["repository"]
// returns an error if the command fails
Expand All @@ -237,6 +237,7 @@ func (b *dockerBuilder) dockerBuild() (string, []string, error) {

args := []string{
"build",
"--progress=plain",
"-t", mainTag,
}
extraTags := []string{}
Expand Down
39 changes: 19 additions & 20 deletions dev-tools/mage/integtest_docker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,11 @@ func (d *DockerIntegrationTester) Use(dir string) (bool, error) {
return false, nil
}

// HasRequirements ensures that the required docker and docker-compose are installed.
// HasRequirements ensures that the required docker is installed.
func (d *DockerIntegrationTester) HasRequirements() error {
if err := HaveDocker(); err != nil {
return err
}
if err := HaveDockerCompose(); err != nil {
return err
}
return nil
}

Expand All @@ -66,7 +63,7 @@ func (d *DockerIntegrationTester) StepRequirements() IntegrationTestSteps {
return IntegrationTestSteps{&IntegrationTestStep{}}
}

// Test performs the tests with docker-compose.
// Test performs the tests with docker compose.
func (d *DockerIntegrationTester) Test(dir string, mageTarget string, env map[string]string) error {
var err error
d.buildImagesOnce.Do(func() { err = dockerComposeBuildImages() })
Expand All @@ -85,8 +82,8 @@ func (d *DockerIntegrationTester) Test(dir string, mageTarget string, env map[st
goPkgCache := filepath.Join(filepath.SplitList(build.Default.GOPATH)[0], "pkg/mod/cache/download")
dockerGoPkgCache := "/gocache"

// Execute the inside of docker-compose.
args := []string{"-p", dockerComposeProjectName(), "run",
// Execute the inside of docker compose.
args := []string{"compose", "-p", dockerComposeProjectName(), "--progress=plain", "run",
"-e", "DOCKER_COMPOSE_PROJECT_NAME=" + dockerComposeProjectName(),
// Disable strict.perms because we mount host dirs inside containers
// and the UID/GID won't meet the strict requirements.
Expand Down Expand Up @@ -125,13 +122,13 @@ func (d *DockerIntegrationTester) Test(dir string, mageTarget string, env map[st
composeEnv,
os.Stdout,
os.Stderr,
"docker-compose",
"docker",
args...,
)

err = saveDockerComposeLogs(dir, mageTarget, composeEnv)
if err != nil && testErr == nil {
// saving docker-compose logs failed but the test didn't.
// saving docker compose logs failed but the test didn't.
return err
}

Expand All @@ -145,12 +142,13 @@ func (d *DockerIntegrationTester) Test(dir string, mageTarget string, env map[st
composeEnv,
io.Discard,
out,
"docker-compose",
"docker",
"compose",
"-p", dockerComposeProjectName(),
"rm", "--stop", "--force",
)
if err != nil && testErr == nil {
// docker-compose rm failed but the test didn't
// docker compose rm failed but the test didn't
return err
}
return testErr
Expand All @@ -176,13 +174,14 @@ func saveDockerComposeLogs(rootDir string, mageTarget string, composeEnv map[str
composeEnv,
composeLogFile, // stdout
composeLogFile, // stderr
"docker-compose",
"docker",
"compose",
"-p", dockerComposeProjectName(),
"logs",
"--no-color",
)
if err != nil {
return fmt.Errorf("executing docker-compose logs: %w", err)
return fmt.Errorf("executing docker compose logs: %w", err)
}

return nil
Expand All @@ -205,8 +204,8 @@ func (d *DockerIntegrationTester) InsideTest(test func() error) error {
}

// integTestDockerComposeEnvVars returns the environment variables used for
// executing docker-compose (not the variables passed into the containers).
// docker-compose uses these when evaluating docker-compose.yml files.
// executing docker compose (not the variables passed into the containers).
// docker compose uses these when evaluating docker-compose.yml files.
func integTestDockerComposeEnvVars() (map[string]string, error) {
esBeatsDir, err := ElasticBeatsDir()
if err != nil {
Expand All @@ -221,8 +220,8 @@ func integTestDockerComposeEnvVars() (map[string]string, error) {
}, nil
}

// dockerComposeProjectName returns the project name to use with docker-compose.
// It is passed to docker-compose using the `-p` flag. And is passed to our
// dockerComposeProjectName returns the project name to use with docker compose.
// It is passed to docker compose using the `-p` flag. And is passed to our
// Go and Python testing libraries through the DOCKER_COMPOSE_PROJECT_NAME
// environment variable.
func dockerComposeProjectName() string {
Expand Down Expand Up @@ -255,7 +254,7 @@ func dockerComposeBuildImages() error {
return err
}

args := []string{"-p", dockerComposeProjectName(), "build", "--force-rm"}
args := []string{"compose", "-p", dockerComposeProjectName(), "--progress=plain", "build", "--force-rm"}
if _, noCache := os.LookupEnv("DOCKER_NOCACHE"); noCache {
args = append(args, "--no-cache")
}
Expand All @@ -273,7 +272,7 @@ func dockerComposeBuildImages() error {
composeEnv,
out,
os.Stderr,
"docker-compose", args...,
"docker", args...,
)

if err != nil {
Expand All @@ -284,7 +283,7 @@ func dockerComposeBuildImages() error {
composeEnv,
out,
os.Stderr,
"docker-compose", args...,
"docker", args...,
)
}
return err
Expand Down
2 changes: 1 addition & 1 deletion dev-tools/mage/pkgtypes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -785,7 +785,7 @@ func runFPM(spec PackageSpec, packageType PackageType) error {
}
spec.OutputFile = packageType.AddFileExtension(filepath.Join(distributionsDir, outputFile))

dockerRun := sh.RunCmd("docker", "run")
dockerRun := sh.RunCmd("docker", "run", "--quiet")
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See https://github.com/elastic/elastic-agent/pull/8594/files#r2158465644

var args []string

args, err = addUIDGidEnvArgs(args)
Expand Down
1 change: 1 addition & 0 deletions pkg/testing/ogc/provisioner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@
func (p *provisioner) ogcPull(ctx context.Context) error {
args := []string{
"pull",
"--quiet",
"docker.elastic.co/observability-ci/ogc:5.0.1",
}
var output bytes.Buffer
Expand Down Expand Up @@ -296,9 +297,9 @@
batch.OS.Arch,
}
if batch.OS.Type == define.Linux {
tags = append(tags, strings.ToLower(fmt.Sprintf("%s-%s", batch.OS.Distro, strings.Replace(batch.OS.Version, ".", "-", -1))))

Check failure on line 300 in pkg/testing/ogc/provisioner.go

View workflow job for this annotation

GitHub Actions / lint (ubuntu-latest) 

QF1004: could use strings.ReplaceAll instead (staticcheck)
} else {
tags = append(tags, strings.ToLower(fmt.Sprintf("%s-%s", batch.OS.Type, strings.Replace(batch.OS.Version, ".", "-", -1))))

Check failure on line 302 in pkg/testing/ogc/provisioner.go

View workflow job for this annotation

GitHub Actions / lint (ubuntu-latest) 

QF1004: could use strings.ReplaceAll instead (staticcheck)
}
los, _ := findOSLayout(batch.OS.OS)
return Layout{
Expand Down
Loading