From f19f26c7f4c31a1f1fcc0f3b672d798e27789141 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Thu, 10 Apr 2025 14:13:54 -0700
Subject: [PATCH 01/51] Always publish to us-east-1.

---
 ci/input_files/build.yaml.tpl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 769f87f5..a79de0c4 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -138,6 +138,8 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   tags: ["arch:amd64"]
   image: registry.ddbuild.io/images/docker:20.10-py3
   rules:
+    - if: '"{{ $environment_name }}" == "sandbox" && $REGION == "us-east-1" && "{{ $runtime.arch }}" == "amd64"'
+      when: always
     - if: '"{{ $environment_name }}" == "sandbox"'
       when: manual
       allow_failure: true

From abfb1b61ae36318a1bb339a7e63bd6f942681d5c Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Thu, 24 Apr 2025 08:52:58 -0700
Subject: [PATCH 02/51] WIP

---
 ci/input_files/build.yaml.tpl | 23 ++++++++++++++++++++++-
 1 file changed, 22 insertions(+), 1 deletion(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index a79de0c4..2a3a7f02 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -3,6 +3,7 @@ stages:
  - test
  - sign
  - publish
+ - e2e
 
 .python-before-script: &python-before-script
   - pip install virtualenv
@@ -138,7 +139,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   tags: ["arch:amd64"]
   image: registry.ddbuild.io/images/docker:20.10-py3
   rules:
-    - if: '"{{ $environment_name }}" == "sandbox" && $REGION == "us-east-1" && "{{ $runtime.arch }}" == "amd64"'
+    - if: '"{{ $environment_name }}" == "sandbox" && $REGION == "ap-east-1" && "{{ $runtime.arch }}" == "amd64"'
       when: always
     - if: '"{{ $environment_name }}" == "sandbox"'
       when: manual
@@ -174,6 +175,26 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
 
 {{- end }}
 
+run-e2e:
+  stage: e2e
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/docker:20.10-py3
+  needs: {{ range $runtime := (ds "runtimes").runtimes }}
+    - publish-layer-sandbox ({{ $runtime.name }}-amd64): [ap-east-1]
+  {{- end }}
+  trigger:
+    project: "DataDog/serverless-e2e-tests"
+    strategy: depend
+  variables:
+    LANGUAGES_SUBSET: python
+    PYTHON_38_VERSION: latest
+    PYTHON_39_VERSION: latest
+    PYTHON_310_VERSION: latest
+    PYTHON_311_VERSION: latest
+    PYTHON_312_VERSION: latest
+    PYTHON_313_VERSION: latest
+
+
 publish-pypi-package:
   stage: publish
   tags: ["arch:amd64"]

From 2b283fd8e26b28a8eace6f2046ae07e9ba0fd42f Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Fri, 13 Jun 2025 13:40:38 -0700
Subject: [PATCH 03/51] Publishes to us-west-2.

---
 ci/input_files/build.yaml.tpl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 2a3a7f02..c7dafbc5 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -139,7 +139,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   tags: ["arch:amd64"]
   image: registry.ddbuild.io/images/docker:20.10-py3
   rules:
-    - if: '"{{ $environment_name }}" == "sandbox" && $REGION == "ap-east-1" && "{{ $runtime.arch }}" == "amd64"'
+    - if: '"{{ $environment_name }}" == "sandbox" && $REGION == "us-west-2" && "{{ $runtime.arch }}" == "amd64"'
       when: always
     - if: '"{{ $environment_name }}" == "sandbox"'
       when: manual
@@ -180,7 +180,7 @@ run-e2e:
   tags: ["arch:amd64"]
   image: registry.ddbuild.io/images/docker:20.10-py3
   needs: {{ range $runtime := (ds "runtimes").runtimes }}
-    - publish-layer-sandbox ({{ $runtime.name }}-amd64): [ap-east-1]
+    - publish-layer-sandbox ({{ $runtime.name }}-amd64): [us-west-2]
   {{- end }}
   trigger:
     project: "DataDog/serverless-e2e-tests"

From 919adcf6a51a07fa855f4aa70bddb180d0d19312 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Mon, 16 Jun 2025 11:28:25 -0700
Subject: [PATCH 04/51] Fix needs and update things.

---
 ci/input_files/build.yaml.tpl | 136 +++++++++++++++++-----------------
 ci/publish_layers.sh          |   2 +
 2 files changed, 72 insertions(+), 66 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index c7dafbc5..587a1bb7 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -57,52 +57,52 @@ check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }}):
   stage: test
   tags: ["arch:amd64"]
   image: registry.ddbuild.io/images/docker:20.10
-  needs: 
+  needs:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
   dependencies:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-  script: 
+  script:
     - PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/check_layer_size.sh
 
-lint python:
-  stage: test
-  tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
-  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
-  before_script: *python-before-script
-  script: 
-    - source venv/bin/activate
-    - ./scripts/check_format.sh
+#lint python:
+#  stage: test
+#  tags: ["arch:amd64"]
+#  image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
+#  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
+#  before_script: *python-before-script
+#  script:
+#    - source venv/bin/activate
+#    - ./scripts/check_format.sh
 
-unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
-  stage: test
-  tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
-  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
-  before_script: *python-before-script
-  script: 
-    - source venv/bin/activate
-    - pytest -vv
+#unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
+#  stage: test
+#  tags: ["arch:amd64"]
+#  image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
+#  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
+#  before_script: *python-before-script
+#  script:
+#    - source venv/bin/activate
+#    - pytest -vv
 
-integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
-  stage: test
-  tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10-py3
-  needs: 
-    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-  dependencies:
-    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
-  variables:
-    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
-  before_script:
-    - *install-node
-    - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
-    - yarn global add serverless@^3.38.0 --prefix /usr/local
-    - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local
-    - cd integration_tests && yarn install && cd ..
-  script:
-    - RUNTIME_PARAM={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/run_integration_tests.sh
+#integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
+#  stage: test
+#  tags: ["arch:amd64"]
+#  image: registry.ddbuild.io/images/docker:20.10-py3
+#  needs:
+#    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
+#  dependencies:
+#    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
+#  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
+#  variables:
+#    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
+#  before_script:
+#    - *install-node
+#    - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
+#    - yarn global add serverless@^3.38.0 --prefix /usr/local
+#    - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local
+#    - cd integration_tests && yarn install && cd ..
+#  script:
+#    - RUNTIME_PARAM={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/run_integration_tests.sh
 
 sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
   stage: sign
@@ -114,9 +114,9 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
   needs:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
     - check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }})
-    - lint python
-    - unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
-    - integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
+    #- lint python
+    #- unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
+    #- integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
   dependencies:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
   artifacts: # Re specify artifacts so the modified signed file is passed
@@ -145,15 +145,18 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
       when: manual
       allow_failure: true
     - if: '$CI_COMMIT_TAG =~ /^v.*/'
+  artifacts:
+    reports:
+      dotenv: layer-version
   needs:
 {{ if or (eq $environment_name "prod") }}
       - sign-layer ({{ $runtime.name }}-{{ $runtime.arch}})
 {{ else }}
       - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
       - check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }})
-      - lint python
-      - unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
-      - integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
+      #- lint python
+      #- unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
+      #- integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
 {{ end }}
   dependencies:
 {{ if or (eq $environment_name "prod") }}
@@ -169,32 +172,14 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   before_script:
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
   script:
-    - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh
+    - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
+    # Extract the arn from the publish log to be used as envvar in e2e tests
+    - echo "PYTHON_{{ $runtime.python_version }}_VERSION=$(grep -oP 'Published arn \K\.*' publish.log)" > layer-version
 
 {{- end }}
 
 {{- end }}
 
-run-e2e:
-  stage: e2e
-  tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10-py3
-  needs: {{ range $runtime := (ds "runtimes").runtimes }}
-    - publish-layer-sandbox ({{ $runtime.name }}-amd64): [us-west-2]
-  {{- end }}
-  trigger:
-    project: "DataDog/serverless-e2e-tests"
-    strategy: depend
-  variables:
-    LANGUAGES_SUBSET: python
-    PYTHON_38_VERSION: latest
-    PYTHON_39_VERSION: latest
-    PYTHON_310_VERSION: latest
-    PYTHON_311_VERSION: latest
-    PYTHON_312_VERSION: latest
-    PYTHON_313_VERSION: latest
-
-
 publish-pypi-package:
   stage: publish
   tags: ["arch:amd64"]
@@ -255,3 +240,22 @@ signed layer bundle:
     - rm -rf datadog_lambda_py-signed-bundle-${CI_JOB_ID}
     - mkdir -p datadog_lambda_py-signed-bundle-${CI_JOB_ID}
     - cp .layers/datadog_lambda_py-*.zip datadog_lambda_py-signed-bundle-${CI_JOB_ID}
+
+e2e-test:
+  stage: e2e
+  trigger:
+    project: DataDog/serverless-e2e-tests
+    strategy: depend
+  variables:
+    LANGUAGES_SUBSET: python
+    PYTHON_38_VERSION: latest
+    PYTHON_39_VERSION: latest
+    PYTHON_310_VERSION: latest
+    PYTHON_311_VERSION: latest
+    PYTHON_312_VERSION: latest
+    PYTHON_313_VERSION: latest
+  needs: {{ range (ds "runtimes").runtimes }}
+    {{- if eq .arch "amd64" }}
+      - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [us-west-2]"
+    {{- end }}
+  {{- end }}
diff --git a/ci/publish_layers.sh b/ci/publish_layers.sh
index 58257bf1..9bac3114 100755
--- a/ci/publish_layers.sh
+++ b/ci/publish_layers.sh
@@ -199,6 +199,8 @@ fi
 while [ $latest_version -lt $VERSION ]; do
     latest_version=$(publish_layer $REGION $layer $aws_cli_python_version_key $layer_path)
     printf "[$REGION] Published version $latest_version for layer $layer in region $REGION\n"
+    latest_arn=$(aws lambda get-layer-version --layer-name $layer --version-number $latest_version --region $REGION --query 'LayerVersionArn' --output text)
+    printf "[$REGION] Published arn $latest_arn\n"
 
     # This shouldn't happen unless someone manually deleted the latest version, say 28, and
     # then tries to republish 28 again. The published version would actually be 29, because

From 937d48084aa5ff7726ecd798c5cd83866015c58e Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 08:49:32 -0700
Subject: [PATCH 05/51] Use underscore.

---
 ci/input_files/build.yaml.tpl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 587a1bb7..358c0602 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -147,7 +147,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
     - if: '$CI_COMMIT_TAG =~ /^v.*/'
   artifacts:
     reports:
-      dotenv: layer-version
+      dotenv: layer_version
   needs:
 {{ if or (eq $environment_name "prod") }}
       - sign-layer ({{ $runtime.name }}-{{ $runtime.arch}})
@@ -174,7 +174,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   script:
     - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
     # Extract the arn from the publish log to be used as envvar in e2e tests
-    - echo "PYTHON_{{ $runtime.python_version }}_VERSION=$(grep -oP 'Published arn \K\.*' publish.log)" > layer-version
+    - echo "PYTHON_{{ $runtime.python_version }}_VERSION=$(grep -oP 'Published arn \K\.*' publish.log)" > layer_version
 
 {{- end }}
 

From 38c126f74df91ed8ae4cb75891513b5774de3870 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 08:57:50 -0700
Subject: [PATCH 06/51] No dot.

---
 ci/input_files/build.yaml.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 358c0602..ff299b69 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -174,7 +174,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   script:
     - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
     # Extract the arn from the publish log to be used as envvar in e2e tests
-    - echo "PYTHON_{{ $runtime.python_version }}_VERSION=$(grep -oP 'Published arn \K\.*' publish.log)" > layer_version
+    - echo "PYTHON_{{ $runtime.python_version | replace "." "" }}_VERSION=$(grep -oP 'Published arn \K\.*' publish.log)" > layer_version
 
 {{- end }}
 

From 9e6235722ac78d24c24e0ec7c46e02a5aad39015 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 09:03:38 -0700
Subject: [PATCH 07/51] Correct replace.

---
 ci/input_files/build.yaml.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index ff299b69..bc8f6ff4 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -174,7 +174,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   script:
     - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
     # Extract the arn from the publish log to be used as envvar in e2e tests
-    - echo "PYTHON_{{ $runtime.python_version | replace "." "" }}_VERSION=$(grep -oP 'Published arn \K\.*' publish.log)" > layer_version
+    - echo "PYTHON_{{ $runtime.python_version | strings.ReplaceAll "." "" }}_VERSION=$(grep -oP 'Published arn \K\.*' publish.log)" > layer_version
 
 {{- end }}
 

From c59ba03635a2b3acb04a1e0014ad87fc82df6195 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 09:33:39 -0700
Subject: [PATCH 08/51] Use variables.

---
 ci/input_files/build.yaml.tpl | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index bc8f6ff4..381a4c37 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -1,3 +1,5 @@
+{{- $e2e_region := "us-west-2" -}}
+
 stages:
  - build
  - test
@@ -133,21 +135,24 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
     - LAYER_FILE=datadog_lambda_py-{{ $runtime.arch}}-{{ $runtime.python_version }}.zip ./scripts/sign_layers.sh prod
 
 {{ range $environment_name, $environment := (ds "environments").environments }}
+{{ $dotenv := print $runtime.name "_" $runtime.arch "_" $environment_name ".env" }}
 
 publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   stage: publish
   tags: ["arch:amd64"]
   image: registry.ddbuild.io/images/docker:20.10-py3
   rules:
-    - if: '"{{ $environment_name }}" == "sandbox" && $REGION == "us-west-2" && "{{ $runtime.arch }}" == "amd64"'
+    - if: '"{{ $environment_name }}" == "sandbox" && $REGION == "{{ $e2e_region }}" && "{{ $runtime.arch }}" == "amd64"'
       when: always
     - if: '"{{ $environment_name }}" == "sandbox"'
       when: manual
       allow_failure: true
     - if: '$CI_COMMIT_TAG =~ /^v.*/'
   artifacts:
+    paths:
+      - {{ $dotenv }}
     reports:
-      dotenv: layer_version
+      dotenv: {{ $dotenv }}
   needs:
 {{ if or (eq $environment_name "prod") }}
       - sign-layer ({{ $runtime.name }}-{{ $runtime.arch}})
@@ -174,7 +179,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   script:
     - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
     # Extract the arn from the publish log to be used as envvar in e2e tests
-    - echo "PYTHON_{{ $runtime.python_version | strings.ReplaceAll "." "" }}_VERSION=$(grep -oP 'Published arn \K\.*' publish.log)" > layer_version
+    - echo "PYTHON_{{ $runtime.python_version | strings.ReplaceAll "." "" }}_VERSION=$(grep -oP 'Published arn \K\.*' publish.log)" > {{ $dotenv }}
 
 {{- end }}
 
@@ -256,6 +261,6 @@ e2e-test:
     PYTHON_313_VERSION: latest
   needs: {{ range (ds "runtimes").runtimes }}
     {{- if eq .arch "amd64" }}
-      - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [us-west-2]"
+      - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"
     {{- end }}
   {{- end }}

From 88a298ca448f6e933fa0a383c3881f218966745f Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 09:42:47 -0700
Subject: [PATCH 09/51] Properly get arn.

---
 ci/input_files/build.yaml.tpl | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 381a4c37..e2a6067c 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -179,7 +179,9 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   script:
     - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
     # Extract the arn from the publish log to be used as envvar in e2e tests
-    - echo "PYTHON_{{ $runtime.python_version | strings.ReplaceAll "." "" }}_VERSION=$(grep -oP 'Published arn \K\.*' publish.log)" > {{ $dotenv }}
+    - layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')"
+    - echo "Found published arn: $layer_arn"
+    - echo "PYTHON_{{ $runtime.name | strings.TrimLeft "python" }}_VERSION=$layer_arn" > {{ $dotenv }}
 
 {{- end }}
 

From 4eaf0fa31efb06801b055fdd2070998588a55e7f Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 09:57:21 -0700
Subject: [PATCH 10/51] Just Trim.

---
 ci/input_files/build.yaml.tpl | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index e2a6067c..abc241b2 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -181,7 +181,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
     # Extract the arn from the publish log to be used as envvar in e2e tests
     - layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')"
     - echo "Found published arn: $layer_arn"
-    - echo "PYTHON_{{ $runtime.name | strings.TrimLeft "python" }}_VERSION=$layer_arn" > {{ $dotenv }}
+    - echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$layer_arn" > {{ $dotenv }}
 
 {{- end }}
 
@@ -255,12 +255,6 @@ e2e-test:
     strategy: depend
   variables:
     LANGUAGES_SUBSET: python
-    PYTHON_38_VERSION: latest
-    PYTHON_39_VERSION: latest
-    PYTHON_310_VERSION: latest
-    PYTHON_311_VERSION: latest
-    PYTHON_312_VERSION: latest
-    PYTHON_313_VERSION: latest
   needs: {{ range (ds "runtimes").runtimes }}
     {{- if eq .arch "amd64" }}
       - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"

From dd9a83050c08c023315001bd29c53fa76e6dc94f Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 10:06:13 -0700
Subject: [PATCH 11/51] Multiline string.

---
 ci/input_files/build.yaml.tpl | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index abc241b2..0be7ab9e 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -176,12 +176,12 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
         {{- end}}
   before_script:
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
-  script:
-    - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
+  script: |
+    STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
     # Extract the arn from the publish log to be used as envvar in e2e tests
-    - layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')"
-    - echo "Found published arn: $layer_arn"
-    - echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$layer_arn" > {{ $dotenv }}
+    layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')"
+    echo "Found published arn: $layer_arn"
+    echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$layer_arn" > {{ $dotenv }}
 
 {{- end }}
 

From 1c6a7e3996edc3ac91b9bda534b548ab496f611a Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 10:22:11 -0700
Subject: [PATCH 12/51] Change to template comment.

---
 ci/input_files/build.yaml.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 0be7ab9e..73c9317e 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -178,7 +178,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
   script: |
     STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
-    # Extract the arn from the publish log to be used as envvar in e2e tests
+    {{/* Extract the arn from the publish log to be used as envvar in e2e tests */ -}}
     layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')"
     echo "Found published arn: $layer_arn"
     echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$layer_arn" > {{ $dotenv }}

From cff5b212144272a7a8dcea794ec411409e1bcf9c Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 10:30:59 -0700
Subject: [PATCH 13/51] No more multiline.

---
 ci/input_files/build.yaml.tpl | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 73c9317e..c020aa54 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -176,12 +176,12 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
         {{- end}}
   before_script:
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
-  script: |
-    STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
+  script:
+    - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
     {{/* Extract the arn from the publish log to be used as envvar in e2e tests */ -}}
-    layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')"
-    echo "Found published arn: $layer_arn"
-    echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$layer_arn" > {{ $dotenv }}
+    - layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')"
+    - echo "Found published arn: $layer_arn"
+    - echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$layer_arn" > {{ $dotenv }}
 
 {{- end }}
 

From e73e4fe89388e0c4849164947a309674db27c651 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 10:32:59 -0700
Subject: [PATCH 14/51] Remove comment?

---
 ci/input_files/build.yaml.tpl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index c020aa54..575134b8 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -178,7 +178,6 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
   script:
     - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
-    {{/* Extract the arn from the publish log to be used as envvar in e2e tests */ -}}
     - layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')"
     - echo "Found published arn: $layer_arn"
     - echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$layer_arn" > {{ $dotenv }}

From 547c0540e63e876c7b0ca61a697d29d5ba5e8334 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 10:51:06 -0700
Subject: [PATCH 15/51] Combine lines again.

---
 ci/input_files/build.yaml.tpl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 575134b8..551144f5 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -178,9 +178,9 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
   script:
     - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
-    - layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')"
-    - echo "Found published arn: $layer_arn"
-    - echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$layer_arn" > {{ $dotenv }}
+    - echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')" > {{ $dotenv }}
+
+{{/* Extract the arn from the publish log to be used as envvar in e2e tests */}}
 
 {{- end }}
 

From 03cc647917d469bafa5650197006b8e51c35b8a9 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 10:52:22 -0700
Subject: [PATCH 16/51] Move comment.

---
 ci/input_files/build.yaml.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 551144f5..ddcf8cd0 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -178,9 +178,9 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
   script:
     - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
+    # Extract the arn from the publish log to be used as envvar in e2e tests
     - echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')" > {{ $dotenv }}
 
-{{/* Extract the arn from the publish log to be used as envvar in e2e tests */}}
 
 {{- end }}
 

From 988f28065ee8679574aa7d3740a3c75a9590ed62 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 11:14:16 -0700
Subject: [PATCH 17/51] Another multiline.

---
 ci/input_files/build.yaml.tpl | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index ddcf8cd0..89042370 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -177,9 +177,11 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   before_script:
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
   script:
-    - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
-    # Extract the arn from the publish log to be used as envvar in e2e tests
-    - echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')" > {{ $dotenv }}
+    - |
+      STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
+      # Extract the arn from the publish log to be used as envvar in e2e tests
+      echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')" > {{ $dotenv }}
+      cat {{ $dotenv }}
 
 
 {{- end }}

From b47528b289cd01e385410fce7cc78285e3a960a6 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 11:19:04 -0700
Subject: [PATCH 18/51] Exit if not found.

---
 ci/input_files/build.yaml.tpl | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 89042370..d8195ef9 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -180,7 +180,14 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
     - |
       STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
       # Extract the arn from the publish log to be used as envvar in e2e tests
-      echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')" > {{ $dotenv }}
+      layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')"
+      if [ -z "$layer_arn" ]; then
+        echo "Error: Layer ARN not found in publish log"
+        exit 1
+      else
+        echo "Found layer arn: $layer_arn"
+      fi
+      echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$layer_arn" > {{ $dotenv }}
       cat {{ $dotenv }}
 
 

From fb4d6d827b27f8fb9e159f70a5153dff81015c3d Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 11:24:29 -0700
Subject: [PATCH 19/51] Set -x.

---
 ci/input_files/build.yaml.tpl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index d8195ef9..a9016f03 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -178,6 +178,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
   script:
     - |
+      set -x
       STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
       # Extract the arn from the publish log to be used as envvar in e2e tests
       layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')"

From 63569610fecd92290d435752f5e59e8cc6161dd4 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 11:32:18 -0700
Subject: [PATCH 20/51] More debugging.

---
 ci/input_files/build.yaml.tpl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index a9016f03..caa3353f 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -181,6 +181,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
       set -x
       STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
       # Extract the arn from the publish log to be used as envvar in e2e tests
+      cat publish.log
       layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')"
       if [ -z "$layer_arn" ]; then
         echo "Error: Layer ARN not found in publish log"

From 30e79600faf36f6bf187bb4997cb1a6f13e15c36 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 11:41:08 -0700
Subject: [PATCH 21/51] Even more debugging.

---
 ci/input_files/build.yaml.tpl | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index caa3353f..7ed20c87 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -182,6 +182,8 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
       STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
       # Extract the arn from the publish log to be used as envvar in e2e tests
       cat publish.log
+      grep 'Published arn' publish.log
+      grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+'
       layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')"
       if [ -z "$layer_arn" ]; then
         echo "Error: Layer ARN not found in publish log"

From ed3cd64eec1da2b0254fa97e43a634768196fead Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 11:45:01 -0700
Subject: [PATCH 22/51] Broaden regex.

---
 ci/input_files/build.yaml.tpl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 7ed20c87..bb7057ff 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -183,8 +183,8 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
       # Extract the arn from the publish log to be used as envvar in e2e tests
       cat publish.log
       grep 'Published arn' publish.log
-      grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+'
-      layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*:\d+')"
+      grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*'
+      layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*')"
       if [ -z "$layer_arn" ]; then
         echo "Error: Layer ARN not found in publish log"
         exit 1

From 5b8d6167612bd7be87aea5494a9fb5af867f966f Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 11:58:06 -0700
Subject: [PATCH 23/51] Subscript.

---
 ci/input_files/build.yaml.tpl | 34 ++++++++++++++++------------------
 1 file changed, 16 insertions(+), 18 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index bb7057ff..8138d37e 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -33,6 +33,18 @@ default:
   - apt-get install nodejs -y
   - npm install --global yarn
 
+# Extract the arn from the publish log to be used as envvar in e2e tests
+.extract-arn: &extract-arn
+  - |
+    layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*')"
+    if [ -z "$layer_arn" ]; then
+      echo "Error: Layer ARN not found in publish log"
+      exit 1
+    else
+      echo "Found layer arn, $layer_arn"
+    fi
+    echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$layer_arn" > {{ $dotenv }}
+
 {{ range $runtime := (ds "runtimes").runtimes }}
 
 # TODO(astuyve) - figure out python build cache
@@ -149,8 +161,6 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
       allow_failure: true
     - if: '$CI_COMMIT_TAG =~ /^v.*/'
   artifacts:
-    paths:
-      - {{ $dotenv }}
     reports:
       dotenv: {{ $dotenv }}
   needs:
@@ -177,22 +187,8 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   before_script:
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
   script:
-    - |
-      set -x
-      STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
-      # Extract the arn from the publish log to be used as envvar in e2e tests
-      cat publish.log
-      grep 'Published arn' publish.log
-      grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*'
-      layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*')"
-      if [ -z "$layer_arn" ]; then
-        echo "Error: Layer ARN not found in publish log"
-        exit 1
-      else
-        echo "Found layer arn: $layer_arn"
-      fi
-      echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$layer_arn" > {{ $dotenv }}
-      cat {{ $dotenv }}
+    - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
+    - *extract-arn
 
 
 {{- end }}
@@ -267,6 +263,8 @@ e2e-test:
     strategy: depend
   variables:
     LANGUAGES_SUBSET: python
+  before_script:
+    - env
   needs: {{ range (ds "runtimes").runtimes }}
     {{- if eq .arch "amd64" }}
       - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"

From ed82d35823ed34f3f8d2fd006658611bf20fe51a Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 12:07:05 -0700
Subject: [PATCH 24/51] No more subscript.

---
 ci/input_files/build.yaml.tpl | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 8138d37e..8a48809d 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -33,18 +33,6 @@ default:
   - apt-get install nodejs -y
   - npm install --global yarn
 
-# Extract the arn from the publish log to be used as envvar in e2e tests
-.extract-arn: &extract-arn
-  - |
-    layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*')"
-    if [ -z "$layer_arn" ]; then
-      echo "Error: Layer ARN not found in publish log"
-      exit 1
-    else
-      echo "Found layer arn, $layer_arn"
-    fi
-    echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$layer_arn" > {{ $dotenv }}
-
 {{ range $runtime := (ds "runtimes").runtimes }}
 
 # TODO(astuyve) - figure out python build cache
@@ -187,8 +175,18 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   before_script:
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
   script:
-    - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
-    - *extract-arn
+    - |
+      STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
+      # Extract the arn from the publish log to be used as envvar in e2e tests
+      layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*')"
+      if [ -z "$layer_arn" ]; then
+        echo "Error: Layer ARN not found in publish log"
+        exit 1
+      else
+        echo "Found layer arn, $layer_arn"
+      fi
+      echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$layer_arn" > {{ $dotenv }}
+      cat {{ $dotenv }}
 
 
 {{- end }}

From 5acf9127a99481454efe3ee170e6343bae4549ef Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 12:12:37 -0700
Subject: [PATCH 25/51] Artifacts true.

---
 ci/input_files/build.yaml.tpl | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 8a48809d..e0193496 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -259,10 +259,9 @@ e2e-test:
   trigger:
     project: DataDog/serverless-e2e-tests
     strategy: depend
+    artifacts: true
   variables:
     LANGUAGES_SUBSET: python
-  before_script:
-    - env
   needs: {{ range (ds "runtimes").runtimes }}
     {{- if eq .arch "amd64" }}
       - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"

From bdd8a39acda7c7bad7eeca47e32cb33196cd758f Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 13:40:28 -0700
Subject: [PATCH 26/51] Inherit dependencies and variables.

---
 ci/input_files/build.yaml.tpl | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index e0193496..d60961b7 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -259,7 +259,6 @@ e2e-test:
   trigger:
     project: DataDog/serverless-e2e-tests
     strategy: depend
-    artifacts: true
   variables:
     LANGUAGES_SUBSET: python
   needs: {{ range (ds "runtimes").runtimes }}
@@ -267,3 +266,10 @@ e2e-test:
       - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"
     {{- end }}
   {{- end }}
+  dependencies: {{ range (ds "runtimes").runtimes }}
+    {{- if eq .arch "amd64" }}
+      - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"
+    {{- end }}
+  {{- end }}
+  inherit:
+    variables: true

From 6707673ecc1dc0aac592cd703f815c4b4c7c29ef Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 13:40:54 -0700
Subject: [PATCH 27/51] Add testing downstream branch.

---
 ci/input_files/build.yaml.tpl | 1 +
 1 file changed, 1 insertion(+)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index d60961b7..e54dcc7e 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -259,6 +259,7 @@ e2e-test:
   trigger:
     project: DataDog/serverless-e2e-tests
     strategy: depend
+    branch: rey.abolofia/variable-passing
   variables:
     LANGUAGES_SUBSET: python
   needs: {{ range (ds "runtimes").runtimes }}

From 11ebe29ac3d913fc156064357c29bc575bfbb46c Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 13:45:28 -0700
Subject: [PATCH 28/51] Remove dependencies use variables.

---
 ci/input_files/build.yaml.tpl | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index e54dcc7e..508a7b2b 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -262,15 +262,14 @@ e2e-test:
     branch: rey.abolofia/variable-passing
   variables:
     LANGUAGES_SUBSET: python
+    PYTHON_38_VERSION: $PYTHON_38_VERSION
+    PYTHON_39_VERSION: $PYTHON_39_VERSION
+    PYTHON_310_VERSION: $PYTHON_310_VERSION
+    PYTHON_311_VERSION: $PYTHON_311_VERSION
+    PYTHON_312_VERSION: $PYTHON_312_VERSION
+    PYTHON_313_VERSION: $PYTHON_313_VERSION
   needs: {{ range (ds "runtimes").runtimes }}
     {{- if eq .arch "amd64" }}
       - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"
     {{- end }}
   {{- end }}
-  dependencies: {{ range (ds "runtimes").runtimes }}
-    {{- if eq .arch "amd64" }}
-      - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"
-    {{- end }}
-  {{- end }}
-  inherit:
-    variables: true

From e914d4cca45f41153d99f2dafa6322c65ec94157 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 13:55:54 -0700
Subject: [PATCH 29/51] Use loop and main.

---
 ci/input_files/build.yaml.tpl | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 508a7b2b..9976e8aa 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -259,15 +259,13 @@ e2e-test:
   trigger:
     project: DataDog/serverless-e2e-tests
     strategy: depend
-    branch: rey.abolofia/variable-passing
   variables:
     LANGUAGES_SUBSET: python
-    PYTHON_38_VERSION: $PYTHON_38_VERSION
-    PYTHON_39_VERSION: $PYTHON_39_VERSION
-    PYTHON_310_VERSION: $PYTHON_310_VERSION
-    PYTHON_311_VERSION: $PYTHON_311_VERSION
-    PYTHON_312_VERSION: $PYTHON_312_VERSION
-    PYTHON_313_VERSION: $PYTHON_313_VERSION
+    {{- range $runtime := (ds "runtimes").runtimes }}
+    {{- if eq $runtime.arch "amd64" }}
+    PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION: $PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION
+    {{- end }}
+    {{- end }}
   needs: {{ range (ds "runtimes").runtimes }}
     {{- if eq .arch "amd64" }}
       - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"

From 353ad17c5e2ed55e0a1bf4931ed6f6b6d6253f21 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 14:09:13 -0700
Subject: [PATCH 30/51] Uncomment other tests.

---
 ci/input_files/build.yaml.tpl | 86 +++++++++++++++++------------------
 1 file changed, 43 insertions(+), 43 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 9976e8aa..74555b32 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -66,45 +66,45 @@ check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }}):
   script:
     - PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/check_layer_size.sh
 
-#lint python:
-#  stage: test
-#  tags: ["arch:amd64"]
-#  image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
-#  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
-#  before_script: *python-before-script
-#  script:
-#    - source venv/bin/activate
-#    - ./scripts/check_format.sh
+lint python:
+  stage: test
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
+  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
+  before_script: *python-before-script
+  script:
+    - source venv/bin/activate
+    - ./scripts/check_format.sh
 
-#unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
-#  stage: test
-#  tags: ["arch:amd64"]
-#  image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
-#  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
-#  before_script: *python-before-script
-#  script:
-#    - source venv/bin/activate
-#    - pytest -vv
+unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
+  stage: test
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
+  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
+  before_script: *python-before-script
+  script:
+    - source venv/bin/activate
+    - pytest -vv
 
-#integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
-#  stage: test
-#  tags: ["arch:amd64"]
-#  image: registry.ddbuild.io/images/docker:20.10-py3
-#  needs:
-#    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-#  dependencies:
-#    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-#  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
-#  variables:
-#    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
-#  before_script:
-#    - *install-node
-#    - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
-#    - yarn global add serverless@^3.38.0 --prefix /usr/local
-#    - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local
-#    - cd integration_tests && yarn install && cd ..
-#  script:
-#    - RUNTIME_PARAM={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/run_integration_tests.sh
+integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
+  stage: test
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/docker:20.10-py3
+  needs:
+    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
+  dependencies:
+    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
+  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
+  variables:
+    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
+  before_script:
+    - *install-node
+    - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
+    - yarn global add serverless@^3.38.0 --prefix /usr/local
+    - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local
+    - cd integration_tests && yarn install && cd ..
+  script:
+    - RUNTIME_PARAM={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/run_integration_tests.sh
 
 sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
   stage: sign
@@ -116,9 +116,9 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
   needs:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
     - check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }})
-    #- lint python
-    #- unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
-    #- integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
+    - lint python
+    - unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
+    - integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
   dependencies:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
   artifacts: # Re specify artifacts so the modified signed file is passed
@@ -157,9 +157,9 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
 {{ else }}
       - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
       - check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }})
-      #- lint python
-      #- unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
-      #- integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
+      - lint python
+      - unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
+      - integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
 {{ end }}
   dependencies:
 {{ if or (eq $environment_name "prod") }}

From 7676cc91835ecea357ae20981a1811269e70776b Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Tue, 17 Jun 2025 14:21:45 -0700
Subject: [PATCH 31/51] Cleanup version env vars.

---
 ci/input_files/build.yaml.tpl | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 74555b32..1eb51ae3 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -261,9 +261,11 @@ e2e-test:
     strategy: depend
   variables:
     LANGUAGES_SUBSET: python
-    {{- range $runtime := (ds "runtimes").runtimes }}
-    {{- if eq $runtime.arch "amd64" }}
-    PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION: $PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION
+    # These env vars are inherited from the dotenv reports of the publish-layer jobs
+    {{- range (ds "runtimes").runtimes }}
+    {{- if eq .arch "amd64" }}
+    {{- $version := print (.name | strings.Trim "python") }}
+    PYTHON_{{ $version }}_VERSION: $PYTHON_{{ $version }}_VERSION
     {{- end }}
     {{- end }}
   needs: {{ range (ds "runtimes").runtimes }}

From bd8f4d94bcf727f302a42ceef3ea8e803634a3b1 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Wed, 18 Jun 2025 11:28:45 -0700
Subject: [PATCH 32/51] Whitespace cleanups.

---
 ci/input_files/build.yaml.tpl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 1eb51ae3..96bfc95b 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -260,16 +260,16 @@ e2e-test:
     project: DataDog/serverless-e2e-tests
     strategy: depend
   variables:
-    LANGUAGES_SUBSET: python
-    # These env vars are inherited from the dotenv reports of the publish-layer jobs
+      LANGUAGES_SUBSET: python
+      # These env vars are inherited from the dotenv reports of the publish-layer jobs
     {{- range (ds "runtimes").runtimes }}
     {{- if eq .arch "amd64" }}
     {{- $version := print (.name | strings.Trim "python") }}
-    PYTHON_{{ $version }}_VERSION: $PYTHON_{{ $version }}_VERSION
+      PYTHON_{{ $version }}_VERSION: $PYTHON_{{ $version }}_VERSION
     {{- end }}
     {{- end }}
   needs: {{ range (ds "runtimes").runtimes }}
     {{- if eq .arch "amd64" }}
       - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"
     {{- end }}
-  {{- end }}
+    {{- end }}

From 0ae5a4f6447d157dd33acbd89cbc1344505962f1 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Wed, 18 Jun 2025 13:11:21 -0700
Subject: [PATCH 33/51] Only deploy layers on success.

---
 ci/input_files/build.yaml.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 96bfc95b..97caf9b2 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -143,7 +143,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   image: registry.ddbuild.io/images/docker:20.10-py3
   rules:
     - if: '"{{ $environment_name }}" == "sandbox" && $REGION == "{{ $e2e_region }}" && "{{ $runtime.arch }}" == "amd64"'
-      when: always
+      when: on_success
     - if: '"{{ $environment_name }}" == "sandbox"'
       when: manual
       allow_failure: true

From b159dcf9e1239a19e9369ad0bd69ffa4623ac288 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Wed, 18 Jun 2025 13:40:12 -0700
Subject: [PATCH 34/51] Add e2e-status check.

---
 ci/input_files/build.yaml.tpl | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 97caf9b2..4ae60b6b 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -273,3 +273,14 @@ e2e-test:
       - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"
     {{- end }}
     {{- end }}
+
+e2e-status:
+  stage: e2e
+  image: registry.ddbuild.io/images/mirror/alpine:latest
+  tags: ["arch:amd64"]
+  needs:
+    - e2e-test
+  script:
+    - echo "✅ e2e tests completed successfully"
+  rules:
+    - when: on_success

From 0c3f604cf2b801bf8a92341b4039f88aea9a7ecf Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Fri, 20 Jun 2025 09:27:38 -0700
Subject: [PATCH 35/51] Improve output of job.

---
 ci/input_files/build.yaml.tpl | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 4ae60b6b..fa445763 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -281,6 +281,20 @@ e2e-status:
   needs:
     - e2e-test
   script:
-    - echo "✅ e2e tests completed successfully"
-  rules:
-    - when: on_success
+    - echo "Python layer ARNs used in E2E tests:"
+    {{- range (ds "runtimes").runtimes }}
+    {{- if eq .arch "amd64" }}
+    {{- $version := print (.name | strings.Trim "python") }}
+    - echo "  PYTHON_{{ $version }}_VERSION=$PYTHON_{{ $version }}_VERSION"
+    {{- end }}
+    {{- end }}
+    - |
+      # TODO: link to the test results
+      #       make this job start running at same time as e2e-test job
+      #       do not wait around for the scheduled job to complete
+      if [ "${CI_JOB_STATUS}" = "failed" ]; then
+        echo "❌ E2E tests failed"
+        exit 1
+      else
+        echo "✅ E2E tests completed successfully"
+      fi

From 7255a2b86f04c8b24f93ced1af0e90a2aef3c423 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Fri, 20 Jun 2025 09:31:55 -0700
Subject: [PATCH 36/51] Needing publish loads dotenv artifact.

---
 ci/input_files/build.yaml.tpl | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index fa445763..4393985a 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -280,6 +280,11 @@ e2e-status:
   tags: ["arch:amd64"]
   needs:
     - e2e-test
+    {{- range (ds "runtimes").runtimes }}
+    {{- if eq .arch "amd64" }}
+    - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"
+    {{- end }}
+    {{- end }}
   script:
     - echo "Python layer ARNs used in E2E tests:"
     {{- range (ds "runtimes").runtimes }}

From d158eb05271fc375b5b626d5a0d8763d05e7ce7d Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Fri, 20 Jun 2025 14:18:49 -0700
Subject: [PATCH 37/51] Have e2e-status job start right away then poll.

---
 ci/input_files/build.yaml.tpl | 41 ++++++++++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 8 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 4393985a..68243e19 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -279,13 +279,13 @@ e2e-status:
   image: registry.ddbuild.io/images/mirror/alpine:latest
   tags: ["arch:amd64"]
   needs:
-    - e2e-test
     {{- range (ds "runtimes").runtimes }}
     {{- if eq .arch "amd64" }}
     - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"
     {{- end }}
     {{- end }}
   script:
+    - apt-get install -y curl
     - echo "Python layer ARNs used in E2E tests:"
     {{- range (ds "runtimes").runtimes }}
     {{- if eq .arch "amd64" }}
@@ -295,11 +295,36 @@ e2e-status:
     {{- end }}
     - |
       # TODO: link to the test results
-      #       make this job start running at same time as e2e-test job
       #       do not wait around for the scheduled job to complete
-      if [ "${CI_JOB_STATUS}" = "failed" ]; then
-        echo "❌ E2E tests failed"
-        exit 1
-      else
-        echo "✅ E2E tests completed successfully"
-      fi
+      echo "🔄 Waiting for E2E tests to complete..."
+      # Poll for e2e-test job completion
+      while true; do
+        # Get the e2e-test job status
+        E2E_JOB_STATUS=$(curl -s --header "PRIVATE-TOKEN: ${CI_JOB_TOKEN}" \
+          "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs" | \
+          jq -r '.[] | select(.name=="e2e-test") | .status')
+        echo "E2E job status: $E2E_JOB_STATUS"
+        case "$E2E_JOB_STATUS" in
+          "success")
+            echo "✅ E2E tests completed successfully"
+            exit 0
+            ;;
+          "failed")
+            echo "❌ E2E tests failed"
+            echo "💡 Look for pipelines triggered around $(date -u +"%Y-%m-%d %H:%M:%S UTC")"
+            exit 1
+            ;;
+          "canceled")
+            echo "⚠️ E2E tests were canceled"
+            exit 1
+            ;;
+          "running"|"pending"|"created")
+            echo "⏳ E2E tests still running..."
+            sleep 30
+            ;;
+          *)
+            echo "❓ Unknown E2E test status: $E2E_JOB_STATUS"
+            sleep 30
+            ;;
+        esac
+      done

From afda4704119bd5be9a2f4c43fb17a510bb9d6611 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Fri, 20 Jun 2025 14:29:34 -0700
Subject: [PATCH 38/51] Comment out tests for now.

---
 ci/input_files/build.yaml.tpl | 88 +++++++++++++++++------------------
 1 file changed, 44 insertions(+), 44 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 68243e19..cfaaf00d 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -55,16 +55,16 @@ build-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
   script:
     - PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/build_layers.sh
 
-check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }}):
-  stage: test
-  tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10
-  needs:
-    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-  dependencies:
-    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-  script:
-    - PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/check_layer_size.sh
+#check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }}):
+#  stage: test
+#  tags: ["arch:amd64"]
+#  image: registry.ddbuild.io/images/docker:20.10
+#  needs:
+#    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
+#  dependencies:
+#    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
+#  script:
+#    - PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/check_layer_size.sh
 
 lint python:
   stage: test
@@ -76,35 +76,35 @@ lint python:
     - source venv/bin/activate
     - ./scripts/check_format.sh
 
-unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
-  stage: test
-  tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
-  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
-  before_script: *python-before-script
-  script:
-    - source venv/bin/activate
-    - pytest -vv
+#unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
+#  stage: test
+#  tags: ["arch:amd64"]
+#  image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
+#  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
+#  before_script: *python-before-script
+#  script:
+#    - source venv/bin/activate
+#    - pytest -vv
 
-integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
-  stage: test
-  tags: ["arch:amd64"]
-  image: registry.ddbuild.io/images/docker:20.10-py3
-  needs:
-    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-  dependencies:
-    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
-  variables:
-    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
-  before_script:
-    - *install-node
-    - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
-    - yarn global add serverless@^3.38.0 --prefix /usr/local
-    - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local
-    - cd integration_tests && yarn install && cd ..
-  script:
-    - RUNTIME_PARAM={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/run_integration_tests.sh
+#integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
+#  stage: test
+#  tags: ["arch:amd64"]
+#  image: registry.ddbuild.io/images/docker:20.10-py3
+#  needs:
+#    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
+#  dependencies:
+#    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
+#  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
+#  variables:
+#    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
+#  before_script:
+#    - *install-node
+#    - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
+#    - yarn global add serverless@^3.38.0 --prefix /usr/local
+#    - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local
+#    - cd integration_tests && yarn install && cd ..
+#  script:
+#    - RUNTIME_PARAM={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/run_integration_tests.sh
 
 sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
   stage: sign
@@ -115,10 +115,10 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
       when: manual
   needs:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-    - check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }})
+    #- check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }})
     - lint python
-    - unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
-    - integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
+    #- unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
+    #- integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
   dependencies:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
   artifacts: # Re specify artifacts so the modified signed file is passed
@@ -156,10 +156,10 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
       - sign-layer ({{ $runtime.name }}-{{ $runtime.arch}})
 {{ else }}
       - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-      - check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }})
+      #- check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }})
       - lint python
-      - unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
-      - integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
+      #- unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
+      #- integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
 {{ end }}
   dependencies:
 {{ if or (eq $environment_name "prod") }}

From b2b1ba5db9563ef6cd856a8831e1595d3a731b0c Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Fri, 20 Jun 2025 14:29:42 -0700
Subject: [PATCH 39/51] Use apk.

---
 ci/input_files/build.yaml.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index cfaaf00d..a87d73a0 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -285,7 +285,7 @@ e2e-status:
     {{- end }}
     {{- end }}
   script:
-    - apt-get install -y curl
+    - apk add --no-cache curl jq
     - echo "Python layer ARNs used in E2E tests:"
     {{- range (ds "runtimes").runtimes }}
     {{- if eq .arch "amd64" }}

From 92607f253ec5b7a6fca4c499d3a1b7ae604abbe0 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Fri, 20 Jun 2025 14:44:16 -0700
Subject: [PATCH 40/51] Debugging.

---
 ci/input_files/build.yaml.tpl | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index a87d73a0..effa187f 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -300,9 +300,10 @@ e2e-status:
       # Poll for e2e-test job completion
       while true; do
         # Get the e2e-test job status
-        E2E_JOB_STATUS=$(curl -s --header "PRIVATE-TOKEN: ${CI_JOB_TOKEN}" \
-          "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs" | \
-          jq -r '.[] | select(.name=="e2e-test") | .status')
+        RESPONSE=$(curl -s --header "PRIVATE-TOKEN: ${CI_JOB_TOKEN}" \
+          "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs")
+        echo "Response: $RESPONSE"
+        E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r '.[] | select(.name=="e2e-test") | .status')
         echo "E2E job status: $E2E_JOB_STATUS"
         case "$E2E_JOB_STATUS" in
           "success")

From a0c1a76c164241e7d41d8fb42fcd3fa14b77f3f9 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Fri, 20 Jun 2025 15:01:00 -0700
Subject: [PATCH 41/51] Try JOB-TOKEN.

---
 ci/input_files/build.yaml.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index effa187f..d3b36a6a 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -300,7 +300,7 @@ e2e-status:
       # Poll for e2e-test job completion
       while true; do
         # Get the e2e-test job status
-        RESPONSE=$(curl -s --header "PRIVATE-TOKEN: ${CI_JOB_TOKEN}" \
+        RESPONSE=$(curl -s --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \
           "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs")
         echo "Response: $RESPONSE"
         E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r '.[] | select(.name=="e2e-test") | .status')

From 692494733552a2f58cc4ce7d0431c52584ce936c Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Fri, 20 Jun 2025 15:33:53 -0700
Subject: [PATCH 42/51] Print out URL.

---
 ci/input_files/build.yaml.tpl | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index d3b36a6a..afff10fb 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -300,8 +300,9 @@ e2e-status:
       # Poll for e2e-test job completion
       while true; do
         # Get the e2e-test job status
-        RESPONSE=$(curl -s --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \
-          "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs")
+        URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs"
+        echo "Fetching E2E job status from: $URL"
+        RESPONSE=$(curl -s --header "JOB-TOKEN: ${CI_JOB_TOKEN}" "$URL")
         echo "Response: $RESPONSE"
         E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r '.[] | select(.name=="e2e-test") | .status')
         echo "E2E job status: $E2E_JOB_STATUS"

From dfdc51013d699c3d8678addce3ab0b11b8a6ce73 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Fri, 20 Jun 2025 15:50:41 -0700
Subject: [PATCH 43/51] Try bridges api.

---
 ci/input_files/build.yaml.tpl | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index afff10fb..aca09e67 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -53,6 +53,7 @@ build-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
   variables:
     CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
   script:
+    - exit 0
     - PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/build_layers.sh
 
 #check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }}):
@@ -73,6 +74,7 @@ lint python:
   cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
   before_script: *python-before-script
   script:
+    - exit 0
     - source venv/bin/activate
     - ./scripts/check_format.sh
 
@@ -176,6 +178,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
   script:
     - |
+      exit 0
       STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
       # Extract the arn from the publish log to be used as envvar in e2e tests
       layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*')"
@@ -226,6 +229,7 @@ layer bundle:
       - datadog_lambda_py-bundle-${CI_JOB_ID}/
     name: datadog_lambda_py-bundle-${CI_JOB_ID}
   script:
+    - exit 0
     - rm -rf datadog_lambda_py-bundle-${CI_JOB_ID}
     - mkdir -p datadog_lambda_py-bundle-${CI_JOB_ID}
     - cp .layers/datadog_lambda_py-*.zip datadog_lambda_py-bundle-${CI_JOB_ID}
@@ -300,11 +304,11 @@ e2e-status:
       # Poll for e2e-test job completion
       while true; do
         # Get the e2e-test job status
-        URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs"
+        URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges"
         echo "Fetching E2E job status from: $URL"
         RESPONSE=$(curl -s --header "JOB-TOKEN: ${CI_JOB_TOKEN}" "$URL")
         echo "Response: $RESPONSE"
-        E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r '.[] | select(.name=="e2e-test") | .status')
+        E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r '.[] | select(.name=="e2e-test") | .pipeline.status')
         echo "E2E job status: $E2E_JOB_STATUS"
         case "$E2E_JOB_STATUS" in
           "success")

From 20a2b567875bedc79baead6682037604fec52853 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Fri, 20 Jun 2025 16:02:19 -0700
Subject: [PATCH 44/51] Try GITLAB_API_TOKEN.

---
 ci/input_files/build.yaml.tpl | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index aca09e67..d1b2c3b1 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -301,12 +301,13 @@ e2e-status:
       # TODO: link to the test results
       #       do not wait around for the scheduled job to complete
       echo "🔄 Waiting for E2E tests to complete..."
-      # Poll for e2e-test job completion
+      URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges"
+      echo "Fetching E2E job status from: $URL"
+      env
       while true; do
         # Get the e2e-test job status
-        URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges"
-        echo "Fetching E2E job status from: $URL"
-        RESPONSE=$(curl -s --header "JOB-TOKEN: ${CI_JOB_TOKEN}" "$URL")
+        #RESPONSE=$(curl -s --header "JOB-TOKEN: ${CI_JOB_TOKEN}" "$URL")
+        RESPONSE=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN}" "$URL")
         echo "Response: $RESPONSE"
         E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r '.[] | select(.name=="e2e-test") | .pipeline.status')
         echo "E2E job status: $E2E_JOB_STATUS"

From 75357ef168b970c01f65b07371116579bbbae728 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Fri, 20 Jun 2025 16:30:27 -0700
Subject: [PATCH 45/51] Use CI_JOB_STATUS to determine pass/fail.

---
 ci/input_files/build.yaml.tpl | 146 +++++++++++++++-------------------
 1 file changed, 62 insertions(+), 84 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index d1b2c3b1..f11bc268 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -53,19 +53,18 @@ build-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
   variables:
     CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
   script:
-    - exit 0
     - PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/build_layers.sh
 
-#check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }}):
-#  stage: test
-#  tags: ["arch:amd64"]
-#  image: registry.ddbuild.io/images/docker:20.10
-#  needs:
-#    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-#  dependencies:
-#    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-#  script:
-#    - PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/check_layer_size.sh
+check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }}):
+  stage: test
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/docker:20.10
+  needs:
+    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
+  dependencies:
+    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
+  script:
+    - PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/check_layer_size.sh
 
 lint python:
   stage: test
@@ -74,39 +73,38 @@ lint python:
   cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
   before_script: *python-before-script
   script:
-    - exit 0
     - source venv/bin/activate
     - ./scripts/check_format.sh
 
-#unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
-#  stage: test
-#  tags: ["arch:amd64"]
-#  image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
-#  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
-#  before_script: *python-before-script
-#  script:
-#    - source venv/bin/activate
-#    - pytest -vv
+unit-test ({{ $runtime.name }}-{{ $runtime.arch }}):
+  stage: test
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/mirror/python:{{ $runtime.image }}
+  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
+  before_script: *python-before-script
+  script:
+    - source venv/bin/activate
+    - pytest -vv
 
-#integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
-#  stage: test
-#  tags: ["arch:amd64"]
-#  image: registry.ddbuild.io/images/docker:20.10-py3
-#  needs:
-#    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-#  dependencies:
-#    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-#  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
-#  variables:
-#    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
-#  before_script:
-#    - *install-node
-#    - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
-#    - yarn global add serverless@^3.38.0 --prefix /usr/local
-#    - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local
-#    - cd integration_tests && yarn install && cd ..
-#  script:
-#    - RUNTIME_PARAM={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/run_integration_tests.sh
+integration-test ({{ $runtime.name }}-{{ $runtime.arch }}):
+  stage: test
+  tags: ["arch:amd64"]
+  image: registry.ddbuild.io/images/docker:20.10-py3
+  needs:
+    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
+  dependencies:
+    - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
+  cache: &{{ $runtime.name }}-{{ $runtime.arch }}-cache
+  variables:
+    CI_ENABLE_CONTAINER_IMAGE_BUILDS: "true"
+  before_script:
+    - *install-node
+    - EXTERNAL_ID_NAME=integration-test-externalid ROLE_TO_ASSUME=sandbox-integration-test-deployer AWS_ACCOUNT=425362996713 source ./ci/get_secrets.sh
+    - yarn global add serverless@^3.38.0 --prefix /usr/local
+    - yarn global add serverless-python-requirements@^6.1.1 --prefix /usr/local
+    - cd integration_tests && yarn install && cd ..
+  script:
+    - RUNTIME_PARAM={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./scripts/run_integration_tests.sh
 
 sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
   stage: sign
@@ -117,10 +115,10 @@ sign-layer ({{ $runtime.name }}-{{ $runtime.arch }}):
       when: manual
   needs:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-    #- check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }})
+    - check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }})
     - lint python
-    #- unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
-    #- integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
+    - unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
+    - integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
   dependencies:
     - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
   artifacts: # Re specify artifacts so the modified signed file is passed
@@ -158,10 +156,10 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
       - sign-layer ({{ $runtime.name }}-{{ $runtime.arch}})
 {{ else }}
       - build-layer ({{ $runtime.name }}-{{ $runtime.arch }})
-      #- check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }})
+      - check-layer-size ({{ $runtime.name }}-{{ $runtime.arch }})
       - lint python
-      #- unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
-      #- integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
+      - unit-test ({{ $runtime.name }}-{{ $runtime.arch }})
+      - integration-test ({{ $runtime.name }}-{{ $runtime.arch }})
 {{ end }}
   dependencies:
 {{ if or (eq $environment_name "prod") }}
@@ -178,7 +176,6 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
   script:
     - |
-      exit 0
       STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
       # Extract the arn from the publish log to be used as envvar in e2e tests
       layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*')"
@@ -229,7 +226,6 @@ layer bundle:
       - datadog_lambda_py-bundle-${CI_JOB_ID}/
     name: datadog_lambda_py-bundle-${CI_JOB_ID}
   script:
-    - exit 0
     - rm -rf datadog_lambda_py-bundle-${CI_JOB_ID}
     - mkdir -p datadog_lambda_py-bundle-${CI_JOB_ID}
     - cp .layers/datadog_lambda_py-*.zip datadog_lambda_py-bundle-${CI_JOB_ID}
@@ -283,13 +279,13 @@ e2e-status:
   image: registry.ddbuild.io/images/mirror/alpine:latest
   tags: ["arch:amd64"]
   needs:
+    - e2e-test
     {{- range (ds "runtimes").runtimes }}
     {{- if eq .arch "amd64" }}
     - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"
     {{- end }}
     {{- end }}
   script:
-    - apk add --no-cache curl jq
     - echo "Python layer ARNs used in E2E tests:"
     {{- range (ds "runtimes").runtimes }}
     {{- if eq .arch "amd64" }}
@@ -299,39 +295,21 @@ e2e-status:
     {{- end }}
     - |
       # TODO: link to the test results
+      #       make this job start running at same time as e2e-test job
       #       do not wait around for the scheduled job to complete
-      echo "🔄 Waiting for E2E tests to complete..."
-      URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges"
-      echo "Fetching E2E job status from: $URL"
-      env
-      while true; do
-        # Get the e2e-test job status
-        #RESPONSE=$(curl -s --header "JOB-TOKEN: ${CI_JOB_TOKEN}" "$URL")
-        RESPONSE=$(curl -s --header "PRIVATE-TOKEN: ${GITLAB_API_TOKEN}" "$URL")
-        echo "Response: $RESPONSE"
-        E2E_JOB_STATUS=$(echo "$RESPONSE" | jq -r '.[] | select(.name=="e2e-test") | .pipeline.status')
-        echo "E2E job status: $E2E_JOB_STATUS"
-        case "$E2E_JOB_STATUS" in
-          "success")
-            echo "✅ E2E tests completed successfully"
-            exit 0
-            ;;
-          "failed")
-            echo "❌ E2E tests failed"
-            echo "💡 Look for pipelines triggered around $(date -u +"%Y-%m-%d %H:%M:%S UTC")"
-            exit 1
-            ;;
-          "canceled")
-            echo "⚠️ E2E tests were canceled"
-            exit 1
-            ;;
-          "running"|"pending"|"created")
-            echo "⏳ E2E tests still running..."
-            sleep 30
-            ;;
-          *)
-            echo "❓ Unknown E2E test status: $E2E_JOB_STATUS"
-            sleep 30
-            ;;
-        esac
-      done
+      switch "${CI_JOB_STATUS}" in
+        "success")
+          echo "✅ E2E tests completed successfully"
+          ;;
+        "failed")
+          echo "❌ E2E tests failed"
+          exit 1
+          ;;
+        "canceled")
+          echo "❌ E2E tests were canceled"
+          exit 1
+          ;;
+        *)
+          echo "❌ E2E tests unknown status: ${CI_JOB_STATUS}"
+          exit 1
+      esac

From 57056baf8409a94e8ac8fc8d6f84f42a9f295b0d Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Fri, 20 Jun 2025 17:37:20 -0700
Subject: [PATCH 46/51] Remove switch.

---
 ci/input_files/build.yaml.tpl | 31 ++++++++++++-------------------
 1 file changed, 12 insertions(+), 19 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index f11bc268..595459cd 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -294,22 +294,15 @@ e2e-status:
     {{- end }}
     {{- end }}
     - |
-      # TODO: link to the test results
-      #       make this job start running at same time as e2e-test job
-      #       do not wait around for the scheduled job to complete
-      switch "${CI_JOB_STATUS}" in
-        "success")
-          echo "✅ E2E tests completed successfully"
-          ;;
-        "failed")
-          echo "❌ E2E tests failed"
-          exit 1
-          ;;
-        "canceled")
-          echo "❌ E2E tests were canceled"
-          exit 1
-          ;;
-        *)
-          echo "❌ E2E tests unknown status: ${CI_JOB_STATUS}"
-          exit 1
-      esac
+      if [ "$CI_JOB_STATUS" = "success" ]; then
+        echo "✅ E2E tests completed successfully"
+      elif [ "$CI_JOB_STATUS" = "failed" ]; then
+        echo "❌ E2E tests failed"
+        exit 1
+      elif [ "$CI_JOB_STATUS" = "canceled" ]; then
+        echo "❌ E2E tests were canceled"
+        exit 1
+      else
+        echo "❌ E2E tests unknown status: ${CI_JOB_STATUS}"
+        exit 1
+      fi

From 6b9af41f16e51ead5263dbca0927ae50379855a1 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Mon, 30 Jun 2025 11:24:29 -0700
Subject: [PATCH 47/51] Create dotenv file from publish script.

---
 ci/input_files/build.yaml.tpl | 14 +-------------
 ci/publish_layers.sh          |  6 ++++++
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 595459cd..7b652c55 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -175,19 +175,7 @@ publish-layer-{{ $environment_name }} ({{ $runtime.name }}-{{ $runtime.arch }}):
   before_script:
     - EXTERNAL_ID_NAME={{ $environment.external_id }} ROLE_TO_ASSUME={{ $environment.role_to_assume }} AWS_ACCOUNT={{ $environment.account }} source ./ci/get_secrets.sh
   script:
-    - |
-      STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} ./ci/publish_layers.sh | tee publish.log
-      # Extract the arn from the publish log to be used as envvar in e2e tests
-      layer_arn="$(grep 'Published arn' publish.log | grep -oE 'arn:aws:lambda:.*')"
-      if [ -z "$layer_arn" ]; then
-        echo "Error: Layer ARN not found in publish log"
-        exit 1
-      else
-        echo "Found layer arn, $layer_arn"
-      fi
-      echo "PYTHON_{{ $runtime.name | strings.Trim "python" }}_VERSION=$layer_arn" > {{ $dotenv }}
-      cat {{ $dotenv }}
-
+    - STAGE={{ $environment_name }} PYTHON_VERSION={{ $runtime.python_version }} ARCH={{ $runtime.arch }} DOTENV={{ $dotenv }} ./ci/publish_layers.sh
 
 {{- end }}
 
diff --git a/ci/publish_layers.sh b/ci/publish_layers.sh
index 9bac3114..9654582a 100755
--- a/ci/publish_layers.sh
+++ b/ci/publish_layers.sh
@@ -211,4 +211,10 @@ while [ $latest_version -lt $VERSION ]; do
     fi
 done
 
+if [ -n "$DOTENV" ]; then
+    printf "[$REGION] Exporting layer version to $DOTENV file...\n"
+    echo "PYTHON_${PYTHON_VERSION/./}_VERSION=$latest_arn" >> "$DOTENV"
+    cat "$DOTENV"
+fi
+
 printf "[$REGION] Finished publishing layers...\n\n"

From e78f44e6a30a870b7b2293a40b1ee2846a95a55e Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Mon, 30 Jun 2025 11:27:34 -0700
Subject: [PATCH 48/51] Use docker docker image.

---
 ci/input_files/build.yaml.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 7b652c55..e118572c 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -264,7 +264,7 @@ e2e-test:
 
 e2e-status:
   stage: e2e
-  image: registry.ddbuild.io/images/mirror/alpine:latest
+  image: registry.ddbuild.io/images/docker:20.10-py3
   tags: ["arch:amd64"]
   needs:
     - e2e-test

From cb0bdee5dd0f58fa706b05bdba8f0e907f6137ed Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Mon, 30 Jun 2025 11:28:31 -0700
Subject: [PATCH 49/51] This needs not needed.

---
 ci/input_files/build.yaml.tpl | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index e118572c..e8a445f3 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -268,11 +268,6 @@ e2e-status:
   tags: ["arch:amd64"]
   needs:
     - e2e-test
-    {{- range (ds "runtimes").runtimes }}
-    {{- if eq .arch "amd64" }}
-    - "publish-layer-sandbox ({{ .name }}-{{ .arch }}): [{{ $e2e_region }}]"
-    {{- end }}
-    {{- end }}
   script:
     - echo "Python layer ARNs used in E2E tests:"
     {{- range (ds "runtimes").runtimes }}

From 59bee4c472ed3bb2d395bd3c9ef63f1f7b92bcdb Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Mon, 30 Jun 2025 11:44:02 -0700
Subject: [PATCH 50/51] Push check status to file in e2e repo.

---
 ci/input_files/build.yaml.tpl | 23 +++--------------------
 1 file changed, 3 insertions(+), 20 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index e8a445f3..82142b5c 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -269,23 +269,6 @@ e2e-status:
   needs:
     - e2e-test
   script:
-    - echo "Python layer ARNs used in E2E tests:"
-    {{- range (ds "runtimes").runtimes }}
-    {{- if eq .arch "amd64" }}
-    {{- $version := print (.name | strings.Trim "python") }}
-    - echo "  PYTHON_{{ $version }}_VERSION=$PYTHON_{{ $version }}_VERSION"
-    {{- end }}
-    {{- end }}
-    - |
-      if [ "$CI_JOB_STATUS" = "success" ]; then
-        echo "✅ E2E tests completed successfully"
-      elif [ "$CI_JOB_STATUS" = "failed" ]; then
-        echo "❌ E2E tests failed"
-        exit 1
-      elif [ "$CI_JOB_STATUS" = "canceled" ]; then
-        echo "❌ E2E tests were canceled"
-        exit 1
-      else
-        echo "❌ E2E tests unknown status: ${CI_JOB_STATUS}"
-        exit 1
-      fi
+    - git clone -b rey.abolofia/status-check --single-branch https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/serverless-e2e-tests.git
+    - cd ./serverless-e2e-tests
+    - ./scripts/check_e2e_status.sh

From 648de9a2742e471cfd38af4d730e8cef8a3f62b5 Mon Sep 17 00:00:00 2001
From: Rey Abolofia <rey.abolofia@datadoghq.com>
Date: Mon, 30 Jun 2025 13:29:11 -0700
Subject: [PATCH 51/51] Start status checking in test stage.

---
 ci/input_files/build.yaml.tpl | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/ci/input_files/build.yaml.tpl b/ci/input_files/build.yaml.tpl
index 82142b5c..73b1c2a9 100644
--- a/ci/input_files/build.yaml.tpl
+++ b/ci/input_files/build.yaml.tpl
@@ -262,13 +262,11 @@ e2e-test:
     {{- end }}
     {{- end }}
 
-e2e-status:
-  stage: e2e
+e2e-test-status:
+  stage: test
   image: registry.ddbuild.io/images/docker:20.10-py3
   tags: ["arch:amd64"]
-  needs:
-    - e2e-test
   script:
-    - git clone -b rey.abolofia/status-check --single-branch https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/serverless-e2e-tests.git
+    - git clone -b rey.abolofia/status-check --single-branch https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/serverless-e2e-tests
     - cd ./serverless-e2e-tests
     - ./scripts/check_e2e_status.sh