[Rollout] Production rollout 2024-10-30 (#4113)

Author: oleksandr-didyk

.vault-config/product-construction-prod.yaml

@@ -19,3 +19,19 @@ secrets:
       hasPrivateKey: true
       hasWebhookSecret: false
       hasOAuthSecret: true
+
+  dotnet-bot-arcade-services-content-rw:
+    type: github-access-token
+    parameters:
+      gitHubBotAccountSecret: 
+        location: engkeyvault
+        name: BotAccount-dotnet-bot
+      gitHubBotAccountName: dotnet-bot
+
+  dotnet-bot-maestro-auth-test-content-rw:
+    type: github-access-token
+    parameters:
+      gitHubBotAccountSecret: 
+        location: engkeyvault
+        name: BotAccount-dotnet-bot
+      gitHubBotAccountName: dotnet-bot

azure-pipelines-product-construction-service.yml

@@ -12,6 +12,7 @@ pr:
 
 variables:
 # https://dev.azure.com/dnceng/internal/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=189
+# Required for MaestroAppClientId, MaestroStagingAppClientId
 - group: Publish-Build-Assets
 - name: resourceGroupName
   value: product-construction-service

docs/DevGuide.md

@@ -82,7 +82,7 @@ You can deploy your branch to the staging environment where E2E tests can be run
 
 If you want to run the C# scenario tests (make sure that you followed the getting started steps before), you will need to set some environment variables:
 
-   1. GITHUB_TOKEN : Get a github PAT from https://github.com/settings/tokens
+   1. GITHUB_TOKEN : See [instructions](#generating-github-pat-for-local-scenario-test-runs) below
    1. DARC_PACKAGE_SOURCE : Get the path to the darc nuget package (which would be in `arcade-services\artifacts\packages\Debug\NonShipping\`, see below for getting this built)
    1. MAESTRO_BASEURIS : Run ngrok and get the https url
 
@@ -173,3 +173,14 @@ You can disable the DNS Service by deleting `DnsService` from the add-on feature
     ]
 ```
 If you change any settings in `ClusterManifestTemplate.json` run `Reset Local Cluster` from Service Fabric Local Cluster Manager to recreate the cluster configuration using the new settings
+
+## Generating GitHub PAT for local scenario test runs
+
+The GitHub scenario tests are ran against a dedicated organization - [`maestro-auth-tests`](https://github.com/maestro-auth-test). As such, a PAT with adequate permissions is required to run them locally.
+
+To generate one, navigate to https://github.com/settings/tokens and select the `Fine-grained tokens` sub-menu on the navigation bar. The token should be generated with the following settings:
+  - Resource owner: `maestro-auth-test` (if this option is not available in the resource settings please ask the team to add you to the test organization)
+  - Repository access: `All repositories`
+  - Repository permissions: `Contents` - `Access: Read and Write`
+
+This configuration will allow the tests to read and write to the test repos without any additional access to the org or the account itself.

eng/templates/jobs/e2e-pcs-tests.yml

@@ -17,6 +17,9 @@ jobs:
     # https://dev.azure.com/dnceng/internal/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=20&path=Publish-Build-Assets
     # Required for MaestroAppClientId, MaestroStagingAppClientId
     - group: Publish-Build-Assets
+    # https://dev.azure.com/dnceng/internal/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=202&path=Arcade-Services-Scenario-Tests
+    # Required for dotnet-bot-maestro-auth-test-content-rw
+    - group: Arcade-Services-Scenario-Tests
     - ${{ if not(or(startsWith(variables['Build.SourceBranch'], 'refs/heads/production'), startsWith(variables['Build.SourceBranch'], 'refs/heads/production-'), eq(variables['Build.SourceBranch'], 'refs/heads/production'))) }}:
       - group: MaestroInt KeyVault
       - name: PcsTestEndpoint
@@ -106,7 +109,7 @@ jobs:
     env:
       PCS_BASEURI: ${{ variables.PcsTestEndpoint }}
       PCS_TOKEN: $(GetAuthInfo.Token)
-      GITHUB_TOKEN: $(maestro-scenario-test-github-token)
+      GITHUB_TOKEN: $(dotnet-bot-maestro-auth-test-content-rw)
       AZDO_TOKEN: $(AzdoToken)
       DARC_PACKAGE_SOURCE: $(Pipeline.Workspace)\PackageArtifacts
       DARC_DIR: $(Build.SourcesDirectory)\darc

eng/templates/jobs/e2e-tests.yml

@@ -19,16 +19,17 @@ jobs:
     # https://dev.azure.com/dnceng/internal/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=20&path=Publish-Build-Assets
     # Required for MaestroAppClientId, MaestroStagingAppClientId
     - group: Publish-Build-Assets
+    # https://dev.azure.com/dnceng/internal/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=202&path=Arcade-Services-Scenario-Tests
+    # Required for dotnet-bot-maestro-auth-test-content-rw
+    - group: Arcade-Services-Scenario-Tests
     - ${{ if parameters.isProd }}:
-      - group: MaestroProd KeyVault
       - name: MaestroTestEndpoints
         value: https://maestro-prod.westus2.cloudapp.azure.com,https://maestro.dot.net
       - name: ScenarioTestSubscription
         value: "Darc: Maestro Production"
       - name: MaestroAppId
         value: $(MaestroAppClientId)
     - ${{ else }}:
-      - group: MaestroInt KeyVault
       - name: MaestroTestEndpoints
         value: https://maestro-int.westus2.cloudapp.azure.com,https://maestro.int-dot.net
       - name: ScenarioTestSubscription
@@ -119,7 +120,7 @@ jobs:
     env:
       MAESTRO_BASEURIS: ${{ variables.MaestroTestEndpoints }}
       MAESTRO_TOKEN: $(GetAuthInfo.Token)
-      GITHUB_TOKEN: $(maestro-scenario-test-github-token)
+      GITHUB_TOKEN: $(dotnet-bot-maestro-auth-test-content-rw)
       AZDO_TOKEN: $(AzdoToken)
       DARC_PACKAGE_SOURCE: $(Pipeline.Workspace)\PackageArtifacts
       DARC_DIR: $(Build.SourcesDirectory)\darc

eng/templates/stages/deploy.yaml

@@ -2,9 +2,6 @@ parameters:
 - name: isProd
   type: boolean
 
-# --- Secret Variable group requirements ---
-# maestro-scenario-test-github-token
-
 stages:
 - template: /eng/templates/stages/secret-validation.yml@self
   parameters:
@@ -35,7 +32,9 @@ stages:
 
   variables:
   - ${{ if parameters.isProd }}:
-    - group: MaestroProd KeyVault
+    # https://dev.azure.com/dnceng/internal/_library?itemType=VariableGroups&view=VariableGroupView&variableGroupId=202&path=Arcade-Services-Scenario-Tests
+    # Required for dotnet-bot-arcade-services-content-write
+    - group: Arcade-Services-Release
     - name: PublishProfile
       value: Prod
     - name: Subscription
@@ -45,7 +44,6 @@ stages:
     - name: BarMigrationSubscription
       value: BarMigrationProd
   - ${{ else }}:
-    - group: MaestroInt KeyVault
     - name: PublishProfile
       value: Int
     - name: Subscription
@@ -131,7 +129,7 @@ stages:
             --repo dotnet/arcade-services
         displayName: Create GitHub release
         env:
-          GH_TOKEN: $(BotAccount-dotnet-bot-repo-PAT)
+          GH_TOKEN: $(dotnet-bot-arcade-services-content-write)
         continueOnError: true
 
 - stage: validateDeployment

src/ProductConstructionService/ProductConstructionService.BarViz/Components/DependencyGrid.razor

@@ -17,8 +17,8 @@
     <TemplateColumn Title="Commit" Align="Align.Start" Width="6rem">
         <a href="@(context.CommitLink ?? "javascript:void(0)")" target="_blank">@context.CommitShort</a>
     </TemplateColumn>
-    <PropertyColumn Title="Age" Property="@(r => r.AgeDays)" Sortable="true" Align="Align.End" Width="5rem" />
-    <PropertyColumn Title="Staleness" Property="@(r => r.BuildStaleness)" Align="Align.Start" Width="8rem" />
+    <PropertyColumn Title="Age (days)" Property="@(r => r.AgeDays)" Sortable="true" Align="Align.End" Width="5rem" />
+    <PropertyColumn Title="Newer builds?" Property="@(r => r.BuildStaleness)" Align="Align.Start" Width="8rem" />
     <TemplateColumn Sortable="false" Align="Align.Start" Title="Build Number" Width="8rem">
         <a href="@context.BuildUrl" target="_blank">
             <FluentBadge Appearance="Appearance.Accent">@context.BuildNumber</FluentBadge>

src/ProductConstructionService/ProductConstructionService.BarViz/Layout/MainLayout.razor

@@ -5,7 +5,7 @@
 
 <FluentLayout>
     <FluentHeader Class="siteheader">
-        Maestro++
+        <FluentAnchor Href="/" Appearance="Appearance.Outline" Style="font-weight: bold">Maestro++</FluentAnchor>
 
         <FluentSpacer />
 

src/ProductConstructionService/ProductConstructionService.BarViz/Pages/Build.razor

@@ -6,13 +6,18 @@
 @using ProductConstructionService.BarViz.Components;
 @inject IProductConstructionServiceApi PcsApi
 
+<PageTitle>@Channel</PageTitle>
+
 @if (_build == null)
 {
     <PageLoadingStatus StatusText="Loading build info ..." />
 }
 else
 {
     <FluentStack VerticalGap="20" Orientation="Orientation.Vertical">
+        <FluentLabel Typo="Typography.H2">
+            @Channel
+        </FluentLabel>
 
         <ErrorBoundary>
             <BuildInfo BuildId="@BuildId" Repository="@Repo" ChannelId="ChannelId" />
@@ -44,13 +49,16 @@ else
 
     private string? Repo { get; set; }
 
+    private string? Channel { get; set; }
+
     private ProductConstructionService.Client.Models.Build? _build;
 
     protected override async Task OnParametersSetAsync()
     {
         _build = null;
 
         Repo = RepoUrlConverter.SlugToRepoUrl(RepoSlug);
+        Channel = (await PcsApi.Channels.GetChannelAsync(ChannelId)).Name;
 
         if (BuildId == "latest")
         {
@@ -61,5 +69,4 @@ else
             _build = await PcsApi.Builds.GetBuildAsync(int.Parse(BuildId!));
         }
     }
-
 }

src/ProductConstructionService/ProductConstructionService.Cli/Operations/DeploymentOperation.cs

@@ -2,8 +2,6 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 
 using Azure;
-using Azure.Core;
-using Azure.ResourceManager;
 using Azure.ResourceManager.AppContainers;
 using Azure.ResourceManager.AppContainers.Models;
 using Azure.ResourceManager.Resources;
@@ -95,10 +93,17 @@ public async Task<int> RunAsync()
             // Wait for the new app revision to become active
             var newRevisionActive = await WaitForRevisionToBecomeActive(newRevisionName);
 
-            // If the new revision is active, the rollout succeeded, assign a label, and transfer all traffic to it
+            // If the new revision is active, the rollout succeeded, assign a label, transfer all traffic to it,
+            // and deactivate the previously running revision
             if (newRevisionActive)
             {
                 await AssignLabelAndTransferTraffic(newRevisionName, inactiveRevisionLabel);
+                
+                if (!string.IsNullOrEmpty(activeRevisionTrafficWeight.RevisionName))
+                {
+                    await RemoveRevisionLabel(activeRevisionTrafficWeight.RevisionName, activeRevisionTrafficWeight.Label);
+                    await DeactivateRevision(activeRevisionTrafficWeight.RevisionName);
+                }
             }
             // If the new revision is not active, deactivate it and get print log link
             else
@@ -121,30 +126,33 @@ public async Task<int> RunAsync()
         }
     }
 
+    private async Task RemoveRevisionLabel(string revisionName, string label)
+    {
+        var result = await InvokeAzCLI(
+            ["containerapp", "revision", "label", "remove"],
+            ["--label", label]);
+        result.ThrowIfFailed($"Failed to remove label {label} from revision {revisionName}.");
+    }
+
     private async Task CleanupRevisionsAsync(IEnumerable<ContainerAppRevisionTrafficWeight> revisionsTrafficWeight)
     {
-        // Cleanup all revision labels
-        foreach (var revisionTrafficWeight in revisionsTrafficWeight)
+        IEnumerable<ContainerAppRevisionResource> activeRevisions = _containerApp.GetContainerAppRevisions()
+            .ToEnumerable()
+            .Where(revision => revision.Data.IsActive ?? false)
+            .Where(revision => revision.Data.TrafficWeight != 100);
+
+        var revisionsToDeactivate = activeRevisions
+            .Select(revision => (
+                revision.Data.Name,
+                revisionsTrafficWeight.FirstOrDefault(trafficWeight => trafficWeight.RevisionName == revision.Data.Name)?.Label));
+
+        foreach (var revision in revisionsToDeactivate)
         {
-            if (!string.IsNullOrEmpty(revisionTrafficWeight.Label))
+            if (!string.IsNullOrEmpty(revision.Label))
             {
-                var result = await InvokeAzCLI([
-                        "containerapp", "revision", "label", "remove",
-                    ],
-                    [
-                        "--label", revisionTrafficWeight.Label
-                    ]);
-                result.ThrowIfFailed($"Failed to remove label {revisionTrafficWeight.Label} from revision {revisionTrafficWeight.RevisionName}. Stderr: {result.StandardError}");
+                await RemoveRevisionLabel(revision.Name, revision.Label);
             }
-        }
-
-        // Now deactivate all revisions in the list
-        foreach (var revisionTrafficWeight in revisionsTrafficWeight)
-        {
-            _containerApp = await _containerApp.GetAsync();
-            ContainerAppRevisionResource revision = (await _containerApp.GetContainerAppRevisionAsync(revisionTrafficWeight.RevisionName)).Value;
-
-            await revision.DeactivateRevisionAsync();
+            await DeactivateRevision(revision.Name);
         }
     }
 
@@ -219,11 +227,16 @@ private async Task AssignLabelAndTransferTraffic(string revisionName, string lab
             label);
     }
 
-    private async Task DeactivateFailedRevisionAndGetLogs(string revisionName)
+    private async Task DeactivateRevision(string revisionName)
     {
         var revision = (await _containerApp.GetContainerAppRevisionAsync(revisionName)).Value;
         await revision.DeactivateRevisionAsync();
         _logger.LogInformation("Deactivated revision {revisionName}", revisionName);
+    }
+
+    private async Task DeactivateFailedRevisionAndGetLogs(string revisionName)
+    {
+        await DeactivateRevision(revisionName);
 
         _logger.LogInformation("Check revision logs too see failure reason: {logsUri}", GetLogsUri(revisionName));
     }