ci: move DI exploration tests to GitLab

.github/workflows/test_frameworks.yml

@@ -33,7 +33,7 @@ jobs:
         run:  |
           git fetch origin ${{ github.event.pull_request.base.sha }}
           export PATHS=$(git diff --name-only HEAD ${{ github.event.pull_request.base.sha }})
-          python -c "import os,sys,fnmatch;sys.exit(not bool([_ for pattern in {'ddtrace/*', 'setup*', 'pyproject.toml', '.github/workflows/test_frameworks.yml', 'tests/debugging/exploration/*'} for _ in fnmatch.filter(os.environ['PATHS'].splitlines(), pattern)]))"
+          python -c "import os,sys,fnmatch;sys.exit(not bool([_ for pattern in {'ddtrace/*', 'setup*', 'pyproject.toml', '.github/workflows/test_frameworks.yml'} for _ in fnmatch.filter(os.environ['PATHS'].splitlines(), pattern)]))"
         continue-on-error: true
 
   bottle-testsuite:
@@ -65,7 +65,6 @@ jobs:
       DD_APPSEC_ENABLED: ${{ matrix.appsec }}
       DD_TESTING_RAISE: true
       CMAKE_BUILD_PARALLEL_LEVEL: 12
-      DD_DEBUGGER_EXPL_OUTPUT_FILE: debugger-expl.txt
       DD_BYTECODE_INJECTION_OUTPUT_FILE: bytecode-injection.txt
 
     defaults:
@@ -107,10 +106,7 @@ jobs:
         if: needs.needs-run.outputs.outcome == 'success'
         # Disable all test_simple tests because they check for
         # log output and it contains phony error messages.
-        run: PYTHONPATH=../ddtrace/tests/debugging/exploration/ ddtrace-run pytest test --continue-on-collection-errors -v -k 'not test_simple'
-      - name: Debugger exploration result
-        if: needs.needs-run.outputs.outcome == 'success'
-        run: cat debugger-expl.txt
+        run: ddtrace-run pytest test --continue-on-collection-errors -v -k 'not test_simple'
       - name: Run Bytecode injection tests
         if: needs.needs-run.outputs.outcome == 'success'
         # Disable all test_simple tests because they check for
@@ -124,34 +120,16 @@ jobs:
     strategy:
       matrix:
         include:
-          #- suffix: DI profiler
-          #  expl_profiler: 1
-          #  expl_coverage: 0
-          #  profiling: 1
-          #  iast: 0
-          #  appsec: 0
-          #- suffix: DI coverage
-          #  expl_profiler: 0
-          #  expl_coverage: 1
-          #  profiling: 1
-          #  iast: 0
-          #  appsec: 0
           # Disabled while the bug is investigated: APPSEC-53222
           # - suffix: IAST
-          #   expl_profiler: 0
-          #   expl_coverage: 0
           #   profiling: 0
           #   iast: 1
           #   appsec: 0
           - suffix: APPSEC
-            expl_profiler: 0
-            expl_coverage: 0
             profiling: 0
             iast: 0
             appsec: 1
           - suffix: Tracer only
-            expl_profiler: 0
-            expl_coverage: 0
             profiling: 0
             iast: 0
             appsec: 0
@@ -166,13 +144,6 @@ jobs:
       DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING: disabled  # To avoid a couple failures due to the extra query
       DD_APPSEC_AUTOMATED_USER_EVENTS_TRACKING_ENABLED: false  # To avoid a couple failures due to the extra query
       DD_TESTING_RAISE: true
-      DD_DEBUGGER_EXPL_ENCODE: 0  # Disabled to speed up
-      DD_DEBUGGER_EXPL_PROFILER_ENABLED: ${{ matrix.expl_profiler }}
-      DD_DEBUGGER_EXPL_PROFILER_DELETE_FUNCTION_PROBES: 1  # Delete to speed up
-      DD_DEBUGGER_EXPL_COVERAGE_ENABLED: ${{ matrix.expl_coverage }}
-      DD_DEBUGGER_EXPL_COVERAGE_DELETE_LINE_PROBES: 1  # Delete to speed up
-      DD_DEBUGGER_EXPL_CONSERVATIVE: 1
-      DD_DEBUGGER_EXPL_OUTPUT_FILE: debugger-expl.txt
       DD_BYTECODE_INJECTION_OUTPUT_FILE: bytecode-injection.txt
       CMAKE_BUILD_PARALLEL_LEVEL: 12
     defaults:
@@ -237,11 +208,7 @@ jobs:
       - name: Run tests
         if: needs.needs-run.outputs.outcome == 'success'
         # django.tests.requests module interferes with requests library patching in the tracer -> disable requests patch
-        run: PYTHONPATH=../ddtrace/tests/debugging/exploration/:. DD_PATCH_MODULES=unittest:no DD_TRACE_REQUESTS_ENABLED=0 ddtrace-run tests/runtests.py --parallel 1
-
-      - name: Debugger exploration results
-        if: needs.needs-run.outputs.outcome == 'success'
-        run: cat debugger-expl.txt
+        run: PYTHONPATH=. DD_PATCH_MODULES=unittest:no DD_TRACE_REQUESTS_ENABLED=0 ddtrace-run tests/runtests.py --parallel 1
 
       - name: Run Bytecode injection tests
         if: needs.needs-run.outputs.outcome == 'success'
@@ -279,8 +246,7 @@ jobs:
       DD_PROFILING_ENABLED: ${{ matrix.profiling }}
       DD_IAST_ENABLED: ${{ matrix.iast }}
       DD_APPSEC_ENABLED: ${{ matrix.appsec }}
-      PYTHONPATH: ../ddtrace/tests/debugging/exploration/:.
-      DD_DEBUGGER_EXPL_OUTPUT_FILE: debugger-expl.txt
+      PYTHONPATH: .
       CMAKE_BUILD_PARALLEL_LEVEL: 12
     defaults:
       run:
@@ -318,9 +284,6 @@ jobs:
       - name: Run tests
         if: needs.needs-run.outputs.outcome == 'success'
         run: ddtrace-run pytest graphene
-      - name: Debugger exploration results
-        if: needs.needs-run.outputs.outcome == 'success'
-        run: cat debugger-expl.txt
 
   fastapi-testsuite-0_92:
     strategy:
@@ -351,7 +314,6 @@ jobs:
       DD_IAST_ENABLED: ${{ matrix.iast }}
       DD_APPSEC_ENABLED: ${{ matrix.appsec }}
       CMAKE_BUILD_PARALLEL_LEVEL: 12
-      DD_DEBUGGER_EXPL_OUTPUT_FILE: debugger-expl.txt
       DD_BYTECODE_INJECTION_OUTPUT_FILE: bytecode-injection.txt
     defaults:
       run:
@@ -388,10 +350,7 @@ jobs:
       - name: Test
         if: needs.needs-run.outputs.outcome == 'success'
         # https://github.com/tiangolo/fastapi/pull/10876
-        run: PYTHONPATH=../ddtrace/tests/debugging/exploration/ ddtrace-run pytest -p no:warnings tests -k 'not test_warn_duplicate_operation_id'
-      - name: Debugger exploration results
-        if: needs.needs-run.outputs.outcome == 'success'
-        run: cat debugger-expl.txt
+        run: ddtrace-run pytest -p no:warnings tests -k 'not test_warn_duplicate_operation_id'
       - name: Bytecode injection Test
         if: needs.needs-run.outputs.outcome == 'success'
         # https://github.com/tiangolo/fastapi/pull/10876
@@ -429,7 +388,6 @@ jobs:
       DD_PROFILING_ENABLED: ${{ matrix.profiling }}
       DD_IAST_ENABLED: ${{ matrix.iast }}
       DD_APPSEC_ENABLED: ${{ matrix.appsec }}
-      PYTHONPATH: ../ddtrace/tests/debugging/exploration/
       CMAKE_BUILD_PARALLEL_LEVEL: 12
     defaults:
       run:
@@ -519,8 +477,6 @@ jobs:
         env:
           # Disabled distributed tracing since there are a lot of tests that assert on headers
           DD_HTTPX_DISTRIBUTED_TRACING: "false"
-          # Debugger exploration testing does not work in CI
-          # PYTHONPATH: ../ddtrace/tests/debugging/exploration/
           DD_IAST_ENABLED: ${{ matrix.iast }}
           DD_APPSEC_ENABLED: ${{ matrix.appsec }}
         # test_pool_timeout raises RuntimeError: The connection pool was closed while 1 HTTP requests/responses were still in-flight
@@ -555,7 +511,6 @@ jobs:
       DD_PROFILING_ENABLED: ${{ matrix.profiling }}
       DD_IAST_ENABLED: ${{ matrix.iast }}
       DD_APPSEC_ENABLED: ${{ matrix.appsec }}
-      PYTHONPATH: ../ddtrace/tests/debugging/exploration/
       CMAKE_BUILD_PARALLEL_LEVEL: 12
     defaults:
       run:
@@ -622,8 +577,6 @@ jobs:
       DD_PROFILING_ENABLED: ${{ matrix.profiling }}
       DD_IAST_ENABLED: ${{ matrix.iast }}
       DD_APPSEC_ENABLED: ${{ matrix.appsec }}
-      PYTHONPATH: ../ddtrace/tests/debugging/exploration/
-      DD_DEBUGGER_EXPL_OUTPUT_FILE: debugger-expl.txt
       CMAKE_BUILD_PARALLEL_LEVEL: 12
     defaults:
       run:
@@ -658,9 +611,6 @@ jobs:
       - name: Run tests
         if: needs.needs-run.outputs.outcome == 'success'
         run: pytest -W ignore --ddtrace-patch-all tests -k 'not test_request_headers and not test_subdomain_route and not test_websocket_headers and not test_staticfiles_with_invalid_dir_permissions_returns_401 and not test_contextvars[asyncio-CustomMiddlewareUsingBaseHTTPMiddleware]'
-      - name: Debugger exploration results
-        if: needs.needs-run.outputs.outcome == 'success'
-        run: cat debugger-expl.txt
 
   requests-testsuite-2_26_0:
     strategy:
@@ -691,7 +641,6 @@ jobs:
       DD_IAST_ENABLED: ${{ matrix.iast }}
       DD_APPSEC_ENABLED: ${{ matrix.appsec }}
       CMAKE_BUILD_PARALLEL_LEVEL: 12
-      DD_DEBUGGER_EXPL_OUTPUT_FILE: debugger-expl.txt
     defaults:
       run:
         working-directory: requests
@@ -726,10 +675,7 @@ jobs:
         run: pip install --upgrade pytest==5.4.3
       - name: Run tests
         if: needs.needs-run.outputs.outcome == 'success'
-        run: PYTHONPATH=../ddtrace/tests/debugging/exploration/ ddtrace-run pytest -p no:warnings tests
-      - name: Debugger exploration results
-        if: needs.needs-run.outputs.outcome == 'success'
-        run: cat debugger-expl.txt
+        run: ddtrace-run pytest -p no:warnings tests
 
   asyncpg-testsuite:
     # https://github.com/MagicStack/asyncpg/blob/v0.25.0/.github/workflows/tests.yml#L125
@@ -816,7 +762,6 @@ jobs:
       DD_TESTING_RAISE: true
       DD_IAST_ENABLED: ${{ matrix.iast }}
       DD_APPSEC_ENABLED: ${{ matrix.appsec }}
-      # PYTHONPATH: ../ddtrace/tests/debugging/exploration/
       CMAKE_BUILD_PARALLEL_LEVEL: 12
     defaults:
       run:
@@ -874,8 +819,6 @@ jobs:
       DD_PROFILING_ENABLED: ${{ matrix.profiling }}
       DD_IAST_ENABLED: ${{ matrix.iast }}
       DD_APPSEC_ENABLED: ${{ matrix.appsec }}
-      PYTHONPATH: ../ddtrace/tests/debugging/exploration/
-      DD_DEBUGGER_EXPL_OUTPUT_FILE: debugger-expl.txt
       CMAKE_BUILD_PARALLEL_LEVEL: 12
     defaults:
       run:
@@ -924,6 +867,3 @@ jobs:
       - name: Run deadlock tests
         if: needs.needs-run.outputs.outcome == 'success'
         run: ddtrace-run ./tests/gh-deadlocks.sh python39
-      - name: Debugger exploration results
-        if: needs.needs-run.outputs.outcome == 'success'
-        run: cat debugger-expl.txt

.gitlab/templates/debugging/exploration.yml

@@ -0,0 +1,24 @@
+"debugging::exploration::boto3":
+  stage: exploration
+  extends: .cached_testrunner
+  variables:
+    DD_DEBUGGER_EXPL_INCLUDE: "boto3"
+    DD_DEBUGGER_EXPL_OUTPUT_FILE: "${{CI_PROJECT_DIR}}/debugger-expl.txt"
+    DD_DEBUGGER_EXPL_STATUS_MESSAGES: 1
+    PYTEST_PLUGINS: "exploration"
+    PYTHONPATH: "${{CI_PROJECT_DIR}}/tests/debugging/exploration/pytest"
+  parallel:
+    matrix:
+      - PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        BOTO3_TAG: 1.38.44
+  script: |
+    python${{PYTHON_VERSION}} -m pip install -e .
+    git clone --depth 1 --branch ${{BOTO3_TAG}} https://github.com/boto/boto3.git
+    cd boto3
+    python${{PYTHON_VERSION}} scripts/ci/install
+    python${{PYTHON_VERSION}} scripts/ci/run-tests -- -svv -W error -W "ignore::dateutil.parser._parser.UnknownTimezoneWarning" -W "ignore::DeprecationWarning"
+    cat ${{DD_DEBUGGER_EXPL_OUTPUT_FILE}}
+  needs: []
+  artifacts:
+    paths:
+      - ${{DD_DEBUGGER_EXPL_OUTPUT_FILE}}

.gitlab/tests.yml

@@ -2,6 +2,7 @@ stages:
   - setup
   - riot
   - hatch
+  - exploration
 
 variables:
   RIOT_RUN_CMD: riot -P -v run --exitfirst --pass-env -s

scripts/gen_gitlab_config.py

@@ -265,6 +265,16 @@ def gen_build_base_venvs() -> None:
         f.write(template("build-base-venvs"))
 
 
+def gen_debugger_exploration() -> None:
+    """Generate the cached testrunner job.
+
+    We need to generate this dynamically from a template because it depends
+    on the cached testrunner job, which is also generated dynamically.
+    """
+    with TESTS_GEN.open("a") as f:
+        f.write(template("debugging/exploration"))
+
+
 # -----------------------------------------------------------------------------
 
 # The code below is the boilerplate that makes the script work. There is

tests/debugging/exploration/_config.py

@@ -29,7 +29,7 @@ class ExplorationConfig(DDConfig):
     encode = DDConfig.v(
         bool,
         "dd.debugger.expl.encode",
-        default=True,
+        default=False,
         help="Whether to encode the snapshots",
     )
 
@@ -40,7 +40,7 @@ class ExplorationConfig(DDConfig):
         help="Whether to print exploration debugger status messages",
     )
 
-    include = DDConfig.v(
+    includes = DDConfig.v(
         list,
         "dd.debugger.expl.include",
         parser=lambda v: [path.split(".") for path in v.split(",")],
@@ -58,7 +58,7 @@ class ExplorationConfig(DDConfig):
     conservative = DDConfig.v(
         bool,
         "dd.debugger.expl.conservative",
-        default=False,
+        default=True,
         help="Use extremely low capture limits to reduce overhead",
     )
 

tests/debugging/exploration/_coverage.py

@@ -1,6 +1,5 @@
 from collections import defaultdict
 from pathlib import Path
-import sys
 from types import ModuleType
 import typing as t
 
@@ -12,7 +11,6 @@
 from debugging.utils import create_snapshot_line_probe
 from output import log
 from utils import COLS
-from utils import CWD
 
 from ddtrace.debugging._function.discovery import FunctionDiscovery
 from ddtrace.debugging._probe.model import LogLineProbe
@@ -22,27 +20,38 @@
 
 
 # Track all the covered modules and its lines. Indexed by module origin.
-_tracked_modules: t.Dict[str, t.Tuple[ModuleType, t.Set[int]]] = {}
+_tracked_modules: t.Dict[Path, t.Tuple[ModuleType, t.Set[int]]] = {}
 
 
 class LineCollector(ModuleCollector):
     def on_collect(self, discovery: FunctionDiscovery) -> None:
         o = origin(discovery._module)
+        if o is None:
+            status("[coverage] cannot collect lines from %s, no origin found" % discovery._module.__name__)
+            return
+
         status("[coverage] collecting lines from %s" % o)
         _tracked_modules[o] = (discovery._module, {_ for _ in discovery.keys()})
-        LineCoverage.add_probes(
-            [
-                create_snapshot_line_probe(
-                    probe_id="@".join([str(hash(f)), str(line)]),
-                    source_file=origin(sys.modules[f.__module__]),
-                    line=line,
-                    rate=0.0,
-                    limits=expl_config.limits,
+        probes = []
+        for line, fcps in discovery.items():
+            for fcp in fcps:
+                try:
+                    fcp.resolve()
+                except ValueError:
+                    # This function-code pair is not from a function, e.g. a
+                    # class.
+                    continue
+                probe_id = f"{o}:{line}"
+                probes.append(
+                    create_snapshot_line_probe(
+                        probe_id=probe_id,
+                        source_file=o,
+                        line=line,
+                        rate=0.0,
+                        limits=expl_config.limits,
+                    )
                 )
-                for line, functions in discovery.items()
-                for f in functions
-            ]
-        )
+        LineCoverage.add_probes(probes)
 
 
 class LineCoverage(ExplorationDebugger):
@@ -55,7 +64,7 @@ def report_coverage(cls) -> None:
             seen_lines_map[t.cast(LogLineProbe, probe).resolved_source_file].add(probe.line)
 
         try:
-            w = max(len(str(o.relative_to(CWD))) for o in _tracked_modules)
+            w = max(len(str(o)) for o in _tracked_modules)
         except ValueError:
             w = int(COLS * 0.75)
         log(("{:=^%ds}" % COLS).format(" Line coverage "))
@@ -72,7 +81,7 @@ def report_coverage(cls) -> None:
             total_covered += len(seen_lines)
             log(
                 ("{:<%d} {:>5} {: 6.0f}%%" % w).format(
-                    str(o.relative_to(CWD)),
+                    str(o),
                     len(lines),
                     len(seen_lines) * 100.0 / len(lines) if lines else 0,
                 )