Skip to content

Partition build jobs #9076

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: master
Choose a base branch
from
+69 −20
Draft

Partition build jobs #9076

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 69 additions & 20 deletions .gitlab-ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,22 +143,25 @@ default:
CACHE_TYPE: lib #default
RUNTIME_AVAILABLE_PROCESSORS_OVERRIDE: 4 # Runtime.getRuntime().availableProcessors() returns incorrect or very high values in Kubernetes
cache:
- key: '$CI_SERVER_VERSION-$CACHE_TYPE' # Dependencies cache. Reset the cache every time gitlab is upgraded. ~Every couple months
- key: '$CI_SERVER_VERSION-$CACHE_TYPE-$CI_SPLIT' # Dependencies cache. Reset the cache every time gitlab is upgraded. ~Every couple months
paths:
# Cached dependencies and wrappers for gradle
- .gradle/wrapper
- .gradle/caches
- .gradle/notifications
policy: $DEPENDENCY_CACHE_POLICY
fallback_keys: # Use fallback keys because all cache types are not populated. See note under: populate_dep_cache
- '$CI_SERVER_VERSION-base'
- '$CI_SERVER_VERSION-lib'
- key: $CI_PIPELINE_ID-$CACHE_TYPE # Incremental build cache. Shared by all jobs in the pipeline of the same type
#fallback_keys: # Use fallback keys because all cache types are not populated. See note under: populate_dep_cache
#- '$CI_SERVER_VERSION-$CACHE_TYPE'
#- '$CI_SERVER_VERSION-base'
#- '$CI_SERVER_VERSION-lib'
- key: $CI_PIPELINE_ID-$CACHE_TYPE-$CI_SPLIT # Incremental build cache. Shared by all jobs in the pipeline of the same type
paths:
- .gradle/caches/$GRADLE_VERSION
- .gradle/$GRADLE_VERSION/executionHistory
- workspace
policy: $BUILD_CACHE_POLICY
#fallback_keys:
# - '$CI_PIPELINE_ID-$CACHE_TYPE'
before_script:
- source .gitlab/gitlab-utils.sh
- mkdir -p .gradle
Expand Down Expand Up @@ -210,7 +213,7 @@ build:
reports:
dotenv: build.env

build_tests:
.build_tests:
extends: .gradle_build
variables:
BUILD_CACHE_POLICY: push
Expand All @@ -232,13 +235,57 @@ build_tests:
- GRADLE_TARGET: ":smokeTest"
CACHE_TYPE: "smoke"
MAVEN_OPTS: "-Xms64M -Xmx512M -Dorg.slf4j.simpleLogger.defaultLogLevel=debug" # FIXME: Build :smokeTest build fails unless mvn debug logging is on

script:
- *gitlab_base_ref_params
- ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS
- ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS

build_tests_base:
extends: .build_tests
variables:
GRADLE_TARGET: ":baseTest"
CACHE_TYPE: "base"
parallel:
matrix:
- CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]

build_tests_profiling:
extends: .build_tests
variables:
GRADLE_TARGET: ":profilingTest"
CACHE_TYPE: "profiling"
parallel:
matrix:
- CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]

build_tests_inst:
extends: .build_tests
variables:
GRADLE_TARGET: ":instrumentationTest"
CACHE_TYPE: "inst"
parallel:
matrix:
- CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]

build_tests_latestdep:
extends: .build_tests
variables:
GRADLE_TARGET: ":instrumentationLatestDepTest"
CACHE_TYPE: "latestdep"
parallel:
matrix:
- CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]

build_tests_smoke:
extends: .build_tests
variables:
GRADLE_TARGET: ":smokeTest"
CACHE_TYPE: "smoke"
parallel:
matrix:
- CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]

populate_dep_cache:
extends: build_tests
extends: .build_tests
variables:
BUILD_CACHE_POLICY: pull
DEPENDENCY_CACHE_POLICY: push
Expand All @@ -253,16 +300,19 @@ populate_dep_cache:
CACHE_TYPE: "lib"
- GRADLE_TARGET: ":baseTest"
CACHE_TYPE: "base"
CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]
- GRADLE_TARGET: ":profilingTest"
CACHE_TYPE: "profiling"
# FIXME: Gitlab doesn't support s3 based caches >5GB. Fixed in Gitlab 17.5
# See: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26921#note_2132307223
# - GRADLE_TARGET: ":instrumentationTest"
# CACHE_TYPE: "inst"
# - GRADLE_TARGET: ":instrumentationLatestDepTest"
# CACHE_TYPE: "latestdep"
# - GRADLE_TARGET: ":smokeTest"
# CACHE_TYPE: "smoke"
CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]
- GRADLE_TARGET: ":instrumentationTest"
CACHE_TYPE: "inst"
CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]
- GRADLE_TARGET: ":instrumentationLatestDepTest"
CACHE_TYPE: "latestdep"
CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]
- GRADLE_TARGET: ":smokeTest"
CACHE_TYPE: "smoke"
CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"]

publish-artifacts-to-s3:
image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29
Expand Down Expand Up @@ -391,7 +441,7 @@ check_debugger:

muzzle:
extends: .gradle_build
needs: [ build_tests ]
needs: [ build_tests_inst ]
stage: tests
parallel: 8
variables:
Expand All @@ -416,7 +466,7 @@ muzzle:

muzzle-dep-report:
extends: .gradle_build
needs: [ build_tests ]
needs: [ build_tests_inst ]
stage: tests
variables:
CACHE_TYPE: inst
Expand Down Expand Up @@ -451,7 +501,6 @@ muzzle-dep-report:
extends: .gradle_build
image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm
tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers
needs: [ build_tests ]
stage: tests
variables:
KUBERNETES_MEMORY_REQUEST: 17Gi
Expand Down