Replace template harness with CLI E2E coverage

.github/CODEOWNERS

@@ -23,10 +23,8 @@
 /tests/Aspire.Hosting.Maui.Tests      @jfversluis
 /tests/Aspire.Hosting.Testing.Tests   @sebastienros
 /tests/Aspire.Hosting.Tests           @mitchdenny
-/tests/Aspire.Templates.Tests         @radical @eerhardt
 /tests/Shared                         @radical @eerhardt
 /tests/helix                          @radical @eerhardt
-/tests/templates.proj                 @radical @eerhardt
 
 # playground apps
 /playground/deployers                @mitchdenny

.github/actions/enumerate-tests/action.yml

@@ -5,7 +5,7 @@ inputs:
     required: false
     type: string
     default: ''
-    description: 'Additional MSBuild arguments passed to the test matrix generation step (e.g., /p:IncludeTemplateTests=true /p:OnlyDeploymentTests=true)'
+    description: 'Additional MSBuild arguments passed to the test matrix generation step (e.g., /p:IncludeCliE2ETests=true /p:OnlyDeploymentTests=true)'
 
 # Output format: JSON with structure {"include": [{...}, ...]}
 # Each entry contains:
@@ -19,7 +19,6 @@ inputs:
 #   - testSessionTimeout: Timeout for the test session (e.g., '20m')
 #   - testHangTimeout: Timeout for hung tests (e.g., '10m')
 #   - requiresNugets: Boolean indicating if NuGet packages are needed
-#   - requiresTestSdk: Boolean indicating if test SDK is needed
 #   - extraTestArgs: Additional test arguments (e.g., '--filter-trait "Partition=P1"')
 #   - collection: (collection type only) Collection/partition name
 #   - classname: (class type only) Fully qualified test class name

.github/workflows/run-tests.yml

@@ -216,13 +216,6 @@ jobs:
             Write-Host "NuGet package source '$source' does not exist — skipping copy"
           }
 
-      - name: Install sdk for nuget based testing
-        if: ${{ fromJson(inputs.properties).requiresTestSdk == true }}
-        run: >
-          ${{ env.DOTNET_SCRIPT }} build ${{ github.workspace }}/tests/workloads.proj
-          /p:SkipPackageCheckForTemplatesTesting=true
-          /bl:${{ github.workspace }}/artifacts/log/Debug/InstallSdkForTesting.binlog
-
       - name: Install Azure Functions Core Tools
         if: runner.os == 'Linux' && (inputs.testShortName == 'Playground' || inputs.testShortName == 'Azure')
         run: |
@@ -616,7 +609,6 @@ jobs:
           path: |
             **/*.binlog
             testresults/**
-            artifacts/bin/Aspire.Templates.Tests/Debug/net8.0/logs/**
             artifacts/log/test-logs/**
 
       - name: Copy CLI E2E recordings for upload

.github/workflows/tests.yml

@@ -28,7 +28,7 @@ jobs:
       - uses: ./.github/actions/enumerate-tests
         id: generate_tests_matrix
         with:
-          buildArgs: '/p:IncludeTemplateTests=true /p:IncludeCliE2ETests=${{ github.event_name == ''pull_request'' }}'
+          buildArgs: '/p:IncludeCliE2ETests=true'
 
       - name: Split matrix by dependency type
         id: split_matrix
@@ -387,14 +387,10 @@ jobs:
         # 'skipped' can be when a transitive dependency fails and the dependent job gets 'skipped'.
         # For example, one of setup_* jobs failing and the dependent test jobs getting 'skipped'.
         # Some jobs are optional and can have an empty matrix. In those cases we allow a 'skipped'
-        # result. Specifically:
-        # - tests_no_nugets_overflow: this overflow bucket is expected to be empty unless the
-        #   primary no-nugets matrix exceeds the overflow threshold.
-        # - tests_requires_nugets_macos: some runs intentionally produce no macOS requires-nugets
-        #   tests, so an empty matrix and 'skipped' result are expected.
-        # - cli_starter_validation_windows: this job only runs for pull requests and is expected to
-        #   be skipped for other workflow events.
-        # All other jobs in this gate are required, and a 'skipped' result is treated as a failure.
+        # result. Any matrix-backed test job is only required when setup_for_tests produced at
+        # least one entry for that bucket. The CLI starter validation job only runs for pull
+        # requests and is expected to be skipped for other workflow events. All other jobs in
+        # this gate are required, and a 'skipped' result is treated as a failure.
         if: >-
           ${{ always() &&
               (contains(needs.*.result, 'failure') ||
@@ -403,21 +399,34 @@ jobs:
                  (needs.extension_tests_win.result == 'skipped' ||
                     needs.cli_starter_validation_windows.result == 'skipped' ||
                     needs.typescript_sdk_tests.result == 'skipped' ||
-                    needs.tests_no_nugets.result == 'skipped' ||
-                    needs.tests_requires_nugets_linux.result == 'skipped' ||
-                   needs.tests_requires_nugets_windows.result == 'skipped' ||
-                   (fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets_macos).include[0] != null &&
-                    needs.tests_requires_nugets_macos.result == 'skipped') ||
-                   needs.tests_requires_cli_archive.result == 'skipped' ||
+                    (fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets).include[0] != null &&
+                     needs.tests_no_nugets.result == 'skipped') ||
+                    (fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets_overflow).include[0] != null &&
+                     needs.tests_no_nugets_overflow.result == 'skipped') ||
+                    (fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets_linux).include[0] != null &&
+                     needs.tests_requires_nugets_linux.result == 'skipped') ||
+                    (fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets_windows).include[0] != null &&
+                     needs.tests_requires_nugets_windows.result == 'skipped') ||
+                    (fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets_macos).include[0] != null &&
+                     needs.tests_requires_nugets_macos.result == 'skipped') ||
+                    (fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_cli_archive).include[0] != null &&
+                     needs.tests_requires_cli_archive.result == 'skipped') ||
                   needs.polyglot_validation.result == 'skipped')) ||
                 (github.event_name != 'pull_request' &&
                  (needs.extension_tests_win.result == 'skipped' ||
                     needs.typescript_sdk_tests.result == 'skipped' ||
-                    needs.tests_no_nugets.result == 'skipped' ||
-                   needs.tests_requires_nugets_linux.result == 'skipped' ||
-                   needs.tests_requires_nugets_windows.result == 'skipped' ||
-                   (fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets_macos).include[0] != null &&
-                    needs.tests_requires_nugets_macos.result == 'skipped') ||
+                    (fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets).include[0] != null &&
+                     needs.tests_no_nugets.result == 'skipped') ||
+                    (fromJson(needs.setup_for_tests.outputs.tests_matrix_no_nugets_overflow).include[0] != null &&
+                     needs.tests_no_nugets_overflow.result == 'skipped') ||
+                    (fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets_linux).include[0] != null &&
+                     needs.tests_requires_nugets_linux.result == 'skipped') ||
+                    (fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets_windows).include[0] != null &&
+                     needs.tests_requires_nugets_windows.result == 'skipped') ||
+                    (fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_nugets_macos).include[0] != null &&
+                     needs.tests_requires_nugets_macos.result == 'skipped') ||
+                    (fromJson(needs.setup_for_tests.outputs.tests_matrix_requires_cli_archive).include[0] != null &&
+                     needs.tests_requires_cli_archive.result == 'skipped') ||
                   needs.polyglot_validation.result == 'skipped'))) }}
         run: |
           echo "One or more dependent jobs failed."

Aspire.slnx

@@ -416,7 +416,6 @@
     <Project Path="tests/Aspire.Hosting.Sdk.Tests/Aspire.Hosting.Sdk.Tests.csproj" />
     <Project Path="tests/Aspire.Managed.Tests/Aspire.Managed.Tests.csproj" />
     <Project Path="tests/Aspire.Playground.Tests/Aspire.Playground.Tests.csproj" />
-    <Project Path="tests/Aspire.Templates.Tests/Aspire.Templates.Tests.csproj" />
     <Project Path="tests/Aspire.TestUtilities/Aspire.TestUtilities.csproj" />
     <Project Path="tests/ConfigurationSchemaGenerator.Tests/ConfigurationSchemaGenerator.Tests.csproj" />
     <Project Path="tests/QuarantineTools.Tests/QuarantineTools.Tests.csproj" />

docs/ci/TestingOnCI.md

@@ -58,7 +58,7 @@ This invokes `eng/TestEnumerationRunsheetBuilder/TestEnumerationRunsheetBuilder.
 - Writes a `.tests-metadata.json` file to `artifacts/helix/` containing:
   - `projectName`, `shortName`, `testProjectPath`
   - `supportedOSes` array (e.g., `["windows", "linux", "macos"]`)
-  - `properties` object with boolean flags (defined in `eng/testing/CITestsProperties.props`): `requiresNugets`, `requiresTestSdk`, `requiresCliArchive`, `requiresGitHubToken`, `enablePlaywrightInstall`
+  - `properties` object with boolean flags (defined in `eng/testing/CITestsProperties.props`): `requiresNugets`, `requiresCliArchive`, `requiresGitHubToken`, `enablePlaywrightInstall`
   - `testSessionTimeout`, `testHangTimeout` values
   - `uncollectedTestsSessionTimeout`, `uncollectedTestsHangTimeout` values
   - `splitTests` flag
@@ -93,19 +93,18 @@ After all projects build, `eng/AfterSolutionBuild.targets` runs `eng/scripts/bui
 {
   "tests": [
     {
-      "name": "Templates-StarterTests",
-      "shortname": "Templates-StarterTests",
-      "testProjectPath": "tests/Aspire.Templates.Tests/...",
-      "supportedOSes": ["windows", "linux", "macos"],
+      "name": "Cli E2E / SmokeTests",
+      "shortname": "SmokeTests",
+      "testProjectPath": "tests/Aspire.Cli.EndToEnd.Tests/Aspire.Cli.EndToEnd.Tests.csproj",
+      "supportedOSes": ["linux"],
       "properties": {
         "requiresNugets": true,
-        "requiresTestSdk": true,
-        "requiresCliArchive": false,
+        "requiresCliArchive": true,
         "enablePlaywrightInstall": false
       },
-      "testSessionTimeout": "20m",
-      "testHangTimeout": "10m",
-      "extraTestArgs": "--filter-class \"...\""
+      "testSessionTimeout": "30m",
+      "testHangTimeout": "15m",
+      "extraTestArgs": "--filter-class \"Aspire.Cli.EndToEnd.Tests.SmokeTests\""
     },
     {
       "name": "Hosting-Docker",
@@ -114,7 +113,6 @@ After all projects build, `eng/AfterSolutionBuild.targets` runs `eng/scripts/bui
       "supportedOSes": ["linux"],
       "properties": {
         "requiresNugets": false,
-        "requiresTestSdk": false,
         "requiresCliArchive": false,
         "enablePlaywrightInstall": false
       },
@@ -253,8 +251,6 @@ For tests that need the built Aspire packages (e.g., template tests, end-to-end
 ```xml
 <PropertyGroup>
   <RequiresNugets>true</RequiresNugets>
-  <!-- Also common for template tests -->
-  <RequiresTestSdk>true</RequiresTestSdk>
 </PropertyGroup>
 ```
 
@@ -299,14 +295,13 @@ This flag is tracked in the test metadata and controls whether Playwright browse
 
 ## CI Test Property Registry
 
-All boolean test properties (such as `RequiresNugets`, `RequiresTestSdk`, `RequiresCliArchive`, `EnablePlaywrightInstall`) are defined in a single source of truth:
+All boolean test properties (such as `RequiresNugets`, `RequiresCliArchive`, `EnablePlaywrightInstall`) are defined in a single source of truth:
 
 **`eng/testing/CITestsProperties.props`**
 
 ```xml
 <ItemGroup>
   <CITestsProperty Include="requiresNugets" MSBuildProp="RequiresNugets" Default="false" />
-  <CITestsProperty Include="requiresTestSdk" MSBuildProp="RequiresTestSdk" Default="false" />
   <CITestsProperty Include="requiresCliArchive" MSBuildProp="RequiresCliArchive" Default="false" />
   <CITestsProperty Include="enablePlaywrightInstall" MSBuildProp="EnablePlaywrightInstall" Default="false" />
 </ItemGroup>
@@ -395,7 +390,7 @@ To run enumeration locally and inspect the generated matrix:
 ./build.sh -test \
   /p:TestRunnerName=TestEnumerationRunsheetBuilder \
   /p:TestMatrixOutputPath=artifacts/canonical-test-matrix.json \
-  /p:IncludeTemplateTests=true \
+  /p:IncludeCliE2ETests=true \
   /p:GenerateCIPartitions=true
 ```
 

docs/ci/azdo-public-pipeline.md

@@ -69,24 +69,22 @@ Which tests run here is controlled by the `RunOnAzdoCI` property (see [Test Rout
 
 #### Helix Tests (`runHelixTests: true`)
 
-1. Installs SDKs for testing (`tests/workloads.proj`)
-2. Sends test work items to Helix via `send-to-helix.yml` → `send-to-helix-ci.proj`
-3. Downloads `.trx` result files from Helix after completion
+1. Sends test work items to Helix via `send-to-helix.yml` → `send-to-helix-ci.proj`
+2. Downloads `.trx` result files from Helix after completion
 
 ## Helix Test Infrastructure
 
 ### How Tests Are Sent to Helix
 
-The entry point is `tests/helix/send-to-helix-ci.proj`, which defines **four test categories**:
+The entry point is `tests/helix/send-to-helix-ci.proj`, which defines **three test categories**:
 
 | Category            | Targets file                              | Runs on Windows | Runs on Linux | Description                                                       |
 |---------------------|-------------------------------------------|-----------------|---------------|-------------------------------------------------------------------|
 | `basictests`        | `send-to-helix-basictests.targets`        | ✅              | ✅            | Standard unit/integration tests                                   |
 | `endtoendtests`     | `send-to-helix-endtoendtests.targets`     | ❌              | ✅            | End-to-end scenario tests (needs Docker)                          |
-| `templatestests`    | `send-to-helix-templatestests.targets`    | ✅              | ✅            | Template creation/run tests                                       |
 | `buildonhelixtests` | `send-to-helix-buildonhelixtests.targets` | ❌              | ✅            | Tests that `dotnet build` + `dotnet test` on Helix (needs Docker) |
 
-The `send-to-helix-ci.proj` first runs `PrepareDependencies` sequentially, then dispatches all categories in parallel via MSBuild.
+The `send-to-helix-ci.proj` first runs `PrepareDependencies` sequentially, then dispatches all categories in parallel via MSBuild. The removed template coverage is replaced by a directly-run AzDO subset of `Aspire.Cli.EndToEnd.Tests`, not by a Helix category.
 
 Each category is handled by `send-to-helix-inner.proj` (the Helix SDK project), which imports the category-specific `.targets` file.
 
@@ -109,14 +107,6 @@ Each test category has its own strategy for splitting tests into Helix work item
 - Each work item filters tests by `--filter-trait "scenario=<scenario>"`
 - Tests run only on Linux (Docker required)
 
-#### `templatestests`
-
-- **One work item per test class** — test class names are extracted at build time
-- The `ExtractTestClassNames` target runs the test assembly with `--list-tests` to discover classes
-- Class names are written to `<TestProject>.tests.list`
-- Each class becomes a separate Helix work item with `--filter-class <ClassName>`
-- Correlation payloads include multiple SDK versions (`dotnet-8`, `dotnet-9`, `dotnet-10`)
-
 #### `buildonhelixtests`
 
 - **One work item per test project zip** — similar to `basictests`
@@ -135,7 +125,7 @@ Each test category has its own strategy for splitting tests into Helix work item
 
 Each work item follows this lifecycle:
 
-1. **Pre-commands**: Clean up stale processes (dotnet-tests, dcp.exe), start Docker cleanup, set environment variables (DCP paths, SDK paths, dev certs, Docker BuildKit)
+1. **Pre-commands**: Clean up stale `dcp.exe` processes, start Docker cleanup, set environment variables (DCP paths, dev certs, Docker BuildKit)
 2. **Command**: Run the test executable with MTP arguments, blame/crash dump collection, quarantine exclusion
 3. **Post-commands**: List Docker state, rename `.trx` files for collection
 
@@ -146,7 +136,6 @@ These are shared across all work items in a Helix job:
 - **DCP binary** — the orchestrator binary, set via `DcpPublisher__CliPath`
 - **Dev cert scripts** — for HTTPS dev certificate setup on Linux
 - **Docker CLI** — specific version installed on the agent
-- **SDKs for testing** — `dotnet-tests` directory with a configured .NET SDK
 - **Built NuGet packages** — `artifacts/packages/Shipping/` for template tests
 - **Playwright browser dependencies** — for UI tests
 - **Azure Functions CLI** — for Functions integration tests
@@ -187,7 +176,6 @@ This is also the easiest way to inspect locally what payload Helix agents will r
 2. **Archive directories** (defined in `tests/Directory.Build.props`):
    - `artifacts/helix/tests/` — basic tests
    - `artifacts/helix/e2e-tests/` — end-to-end tests
-   - `artifacts/helix/templates-tests/` — template tests
    - `artifacts/helix/build-on-helix-tests/` — build-on-helix tests
    - `artifacts/helix/cli-e2e-tests/` — CLI E2E tests
    - `artifacts/helix/deployment-e2e-tests/` — deployment E2E tests
@@ -198,11 +186,6 @@ This is also the easiest way to inspect locally what payload Helix agents will r
    - `nuget.config` — configured to resolve built packages from artifacts
    - Shared test utilities
 
-4. **Test class extraction** (for `templatestests`):
-   - The `ExtractTestClassNames` target runs the test executable with `--list-tests`
-   - Extracts unique class names matching a prefix regex
-   - Writes them to `<ProjectName>.tests.list` alongside the zip
-
 ## Helix xUnit Configuration
 
 When `PrepareForHelix=true`, a special `xunit.runner.json` is used (`tests/helix/xunit.runner.json`):
@@ -275,7 +258,6 @@ Since AzDO tests don't run on PRs, changes can silently break the pipeline. The
 
 **Real incidents**:
 - `49b1fd3b`: Template tests ran `dotnet test --list-tests` which invoked the system dotnet (6.0) instead of the repo's dotnet (8.0+) → "You must install or update .NET" error. A prior PR removed `DOTNET_ROOT` environment variable override for GitHub Actions, breaking AzDO.
-- `258d2e95`: `dotnet-tests` SDK directory wasn't properly prepared for template and helix test runs
 
 **What to watch for**: Changes to `DOTNET_ROOT`, `PATH`, or SDK version settings in `BuildAndTest.yml`, `tests/Directory.Build.targets`, or helix targets. If a change works by relying on the system dotnet or GitHub Actions' pre-installed SDK, it will likely break AzDO/Helix.
 
@@ -362,7 +344,6 @@ When reviewing PRs, flag these for manual AzDO validation (`/azp run aspire-test
 - [ ] Tests using `AddProject<T>()` in projects that run on Helix
 - [ ] Changes to `AspireProjectOrPackageReference` items
 - [ ] New or modified Verify snapshot tests
-- [ ] Changes to `tests/workloads.proj` or SDK setup
 
 ## How to Manually Trigger AzDO Tests
 
@@ -387,7 +368,6 @@ This comment in a PR will trigger the `aspire-tests` pipeline, which runs both p
 | `tests/helix/send-to-helix-inner.proj` | Helix SDK project (work item builder) |
 | `tests/helix/send-to-helix-basictests.targets` | Basic test work items |
 | `tests/helix/send-to-helix-endtoendtests.targets` | E2E test work items (by scenario) |
-| `tests/helix/send-to-helix-templatestests.targets` | Template test work items (by class) |
 | `tests/helix/send-to-helix-buildonhelixtests.targets` | Build-on-Helix test work items |
 | `eng/Testing.props` | Default test runner properties |
 | `eng/Testing.targets` | Test skip/run logic per runner context |

dogfood.sh

@@ -16,9 +16,10 @@ else
 fi
 
 REPO_ROOT=$(cd "${scriptroot}";pwd)
-SDK_PATH=$REPO_ROOT/artifacts/bin/dotnet-tests
+SDK_PATH=$REPO_ROOT/.dotnet
 if [ ! -x "$SDK_PATH/dotnet" ]; then
     echo "Error: Could not find dotnet at $SDK_PATH/dotnet"
+    echo "Run ./restore.sh first to install the repo-local SDK."
     return
 fi