diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cfd791338ba..391f9ca6f71 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -143,22 +143,25 @@ default: CACHE_TYPE: lib #default RUNTIME_AVAILABLE_PROCESSORS_OVERRIDE: 4 # Runtime.getRuntime().availableProcessors() returns incorrect or very high values in Kubernetes cache: - - key: '$CI_SERVER_VERSION-$CACHE_TYPE' # Dependencies cache. Reset the cache every time gitlab is upgraded. ~Every couple months + - key: '$CI_SERVER_VERSION-$CACHE_TYPE-$CI_SPLIT' # Dependencies cache. Reset the cache every time gitlab is upgraded. ~Every couple months paths: # Cached dependencies and wrappers for gradle - .gradle/wrapper - .gradle/caches - .gradle/notifications policy: $DEPENDENCY_CACHE_POLICY - fallback_keys: # Use fallback keys because all cache types are not populated. See note under: populate_dep_cache - - '$CI_SERVER_VERSION-base' - - '$CI_SERVER_VERSION-lib' - - key: $CI_PIPELINE_ID-$CACHE_TYPE # Incremental build cache. Shared by all jobs in the pipeline of the same type + #fallback_keys: # Use fallback keys because all cache types are not populated. See note under: populate_dep_cache + #- '$CI_SERVER_VERSION-$CACHE_TYPE' + #- '$CI_SERVER_VERSION-base' + #- '$CI_SERVER_VERSION-lib' + - key: $CI_PIPELINE_ID-$CACHE_TYPE-$CI_SPLIT # Incremental build cache. Shared by all jobs in the pipeline of the same type paths: - .gradle/caches/$GRADLE_VERSION - .gradle/$GRADLE_VERSION/executionHistory - workspace policy: $BUILD_CACHE_POLICY + #fallback_keys: + # - '$CI_PIPELINE_ID-$CACHE_TYPE' before_script: - source .gitlab/gitlab-utils.sh - mkdir -p .gradle @@ -210,7 +213,7 @@ build: reports: dotenv: build.env -build_tests: +.build_tests: extends: .gradle_build variables: BUILD_CACHE_POLICY: push @@ -232,13 +235,57 @@ build_tests: - GRADLE_TARGET: ":smokeTest" CACHE_TYPE: "smoke" MAVEN_OPTS: "-Xms64M -Xmx512M -Dorg.slf4j.simpleLogger.defaultLogLevel=debug" # FIXME: Build :smokeTest build fails unless mvn debug logging is on - script: - *gitlab_base_ref_params - - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests $GRADLE_ARGS + - ./gradlew clean $GRADLE_TARGET $GRADLE_PARAMS -PskipTests -PtaskPartitionCount=$NORMALIZED_NODE_TOTAL -PtaskPartition=$NORMALIZED_NODE_INDEX $GRADLE_ARGS + +build_tests_base: + extends: .build_tests + variables: + GRADLE_TARGET: ":baseTest" + CACHE_TYPE: "base" + parallel: + matrix: + - CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] + +build_tests_profiling: + extends: .build_tests + variables: + GRADLE_TARGET: ":profilingTest" + CACHE_TYPE: "profiling" + parallel: + matrix: + - CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] + +build_tests_inst: + extends: .build_tests + variables: + GRADLE_TARGET: ":instrumentationTest" + CACHE_TYPE: "inst" + parallel: + matrix: + - CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] + +build_tests_latestdep: + extends: .build_tests + variables: + GRADLE_TARGET: ":instrumentationLatestDepTest" + CACHE_TYPE: "latestdep" + parallel: + matrix: + - CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] + +build_tests_smoke: + extends: .build_tests + variables: + GRADLE_TARGET: ":smokeTest" + CACHE_TYPE: "smoke" + parallel: + matrix: + - CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] populate_dep_cache: - extends: build_tests + extends: .build_tests variables: BUILD_CACHE_POLICY: pull DEPENDENCY_CACHE_POLICY: push @@ -253,16 +300,19 @@ populate_dep_cache: CACHE_TYPE: "lib" - GRADLE_TARGET: ":baseTest" CACHE_TYPE: "base" + CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] - GRADLE_TARGET: ":profilingTest" CACHE_TYPE: "profiling" -# FIXME: Gitlab doesn't support s3 based caches >5GB. Fixed in Gitlab 17.5 -# See: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/26921#note_2132307223 -# - GRADLE_TARGET: ":instrumentationTest" -# CACHE_TYPE: "inst" -# - GRADLE_TARGET: ":instrumentationLatestDepTest" -# CACHE_TYPE: "latestdep" -# - GRADLE_TARGET: ":smokeTest" -# CACHE_TYPE: "smoke" + CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] + - GRADLE_TARGET: ":instrumentationTest" + CACHE_TYPE: "inst" + CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] + - GRADLE_TARGET: ":instrumentationLatestDepTest" + CACHE_TYPE: "latestdep" + CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] + - GRADLE_TARGET: ":smokeTest" + CACHE_TYPE: "smoke" + CI_SPLIT: ["1/4", "2/4", "3/4", "4/4"] publish-artifacts-to-s3: image: registry.ddbuild.io/images/mirror/amazon/aws-cli:2.4.29 @@ -391,7 +441,7 @@ check_debugger: muzzle: extends: .gradle_build - needs: [ build_tests ] + needs: [ build_tests_inst ] stage: tests parallel: 8 variables: @@ -416,7 +466,7 @@ muzzle: muzzle-dep-report: extends: .gradle_build - needs: [ build_tests ] + needs: [ build_tests_inst ] stage: tests variables: CACHE_TYPE: inst @@ -451,7 +501,6 @@ muzzle-dep-report: extends: .gradle_build image: ghcr.io/datadog/dd-trace-java-docker-build:${BUILDER_IMAGE_VERSION_PREFIX}$testJvm tags: [ "docker-in-docker:amd64" ] # use docker-in-docker runner for testcontainers - needs: [ build_tests ] stage: tests variables: KUBERNETES_MEMORY_REQUEST: 17Gi