[Rollout] Production rollout 2025-05-29 (#4897)

#4896

Author: premun

src/Microsoft.DotNet.Darc/DarcLib/Helpers/ProcessExecutionResult.cs

@@ -18,7 +18,7 @@ public void ThrowIfFailed(string failureMessage)
     {
         if (!Succeeded)
         {
-            throw new Exception(failureMessage + Environment.NewLine + this);
+            throw new ProcessFailedException(this, failureMessage);
         }
     }
 
@@ -40,3 +40,11 @@ public override string ToString()
         return output.ToString();
     }
 }
+
+public class ProcessFailedException(ProcessExecutionResult executionResult, string failureMessage)
+    : Exception
+{
+    public ProcessExecutionResult ExecutionResult { get; } = executionResult;
+
+    public override string Message => failureMessage + Environment.NewLine + ExecutionResult.ToString();
+}

src/Microsoft.DotNet.Darc/DarcLib/Models/VirtualMonoRepo/CodeFlowResult.cs

@@ -12,5 +12,4 @@ public record CodeFlowResult(
     bool HadUpdates,
     IReadOnlyCollection<UnixPath> ConflictedFiles,
     NativePath RepoPath,
-    string? PreviouslyFlownSha,
     List<DependencyUpdate> DependencyUpdates);

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/Exceptions.cs

@@ -31,28 +31,44 @@ private static string GetExceptionMessage(VmrIngestionPatch patch, ProcessExecut
 ///     Exception thrown when the service can't apply an update to the PR branch due to a conflict
 ///     between the source repo and a change that was made in the PR after it was opened.
 /// </summary>
-public class ConflictInPrBranchException(ProcessExecutionResult gitMergeResult, string targetBranch, string targetRepo, bool isForwardFlow)
-    : Exception($"Failed to flow changes due to conflicts in the target branch ({targetBranch})")
+public class ConflictInPrBranchException : Exception
 {
-    public List<string> FilesInConflict { get; } = ParseResult(gitMergeResult, targetRepo, isForwardFlow);
-
     private static readonly Regex AlreadyExistsRegex = new("patch failed: (.+): already exist in index");
     private static readonly Regex PatchFailedRegex = new("error: patch failed: (.*):");
     private static readonly Regex PatchDoesNotApplyRegex = new("error: (.+): patch does not apply");
     private static readonly Regex FileDoesNotExistRegex = new("error: (.+): does not exist in index");
+    private static readonly Regex FailedMergeRegex = new("CONFLICT (content): Merge conflict in (.+)");
 
     private static readonly Regex[] ConflictRegex =
-        [
-            AlreadyExistsRegex,
-            PatchFailedRegex,
-            PatchDoesNotApplyRegex,
-            FileDoesNotExistRegex
-        ];
+    [
+        AlreadyExistsRegex,
+        PatchFailedRegex,
+        PatchDoesNotApplyRegex,
+        FileDoesNotExistRegex,
+        FailedMergeRegex,
+    ];
+
+    public List<string> ConflictedFiles { get; }
+
+    public ConflictInPrBranchException(
+            string failedMergeMessage,
+            string targetBranch,
+            string mappingName,
+            bool isForwardFlow)
+        : this(ParseResult(failedMergeMessage, mappingName, isForwardFlow), targetBranch)
+    {
+    }
 
-    private static List<string> ParseResult(ProcessExecutionResult gitMergeResult, string targetRepo, bool isForwardFlow)
+    private ConflictInPrBranchException(List<string> conflictedFiles, string targetBranch)
+        : base($"Failed to flow changes due to conflicts in the target branch ({targetBranch})")
     {
-        List<string> filesInConflict = new();
-        var errors = gitMergeResult.StandardError.Split(Environment.NewLine);
+        ConflictedFiles = conflictedFiles;
+    }
+
+    private static List<string> ParseResult(string failureException, string mappingName, bool isForwardFlow)
+    {
+        List<string> filesInConflict = [];
+        var errors = failureException.Split(Environment.NewLine);
         foreach (var error in errors)
         {
             foreach (var regex in ConflictRegex)
@@ -65,15 +81,16 @@ private static List<string> ParseResult(ProcessExecutionResult gitMergeResult, s
                 }
             }
         }
+
         if (isForwardFlow)
         {
             // Convert VMR paths to normal repo paths, for example src/repo/file.cs -> file.cs
-            return filesInConflict.Select(file => file.Split('/', 3)[2]).Distinct().ToList();
+            return [..filesInConflict.Select(file => file.Split('/', 3)[2]).Distinct()];
         }
-        // If we're backflowing, the file paths are already normal
         else
         {
-            return filesInConflict.Distinct().Select(file => $"src/{targetRepo}/{file}").ToList();
+            // If we're backflowing, the file paths are already normalized
+            return [..filesInConflict.Distinct().Select(file => $"src/{mappingName}/{file}")];
         }
     }
 }

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrBackflower.cs

@@ -121,10 +121,7 @@ public async Task<CodeFlowResult> FlowBackAsync(
             headBranch,
             discardPatches,
             headBranchExisted,
-            cancellationToken) with
-        {
-            PreviouslyFlownSha = lastBackFlow?.SourceSha,
-        };
+            cancellationToken);
     }
 
     protected async Task<CodeFlowResult> FlowBackAsync(
@@ -170,7 +167,6 @@ protected async Task<CodeFlowResult> FlowBackAsync(
             hasChanges || mergeResult.DependencyUpdates.Count > 0,
             mergeResult.ConflictedFiles,
             targetRepo.Path,
-            PreviouslyFlownSha: null,
             mergeResult.DependencyUpdates);
     }
 
@@ -240,7 +236,7 @@ protected override async Task<bool> SameDirectionFlowAsync(
             if (headBranchExisted)
             {
                 _logger.LogInformation("Failed to update a PR branch because of a conflict. Stopping the flow..");
-                throw new ConflictInPrBranchException(e.Result, targetBranch, mapping.Name, isForwardFlow: false);
+                throw new ConflictInPrBranchException(e.Result.StandardError, targetBranch, mapping.Name, isForwardFlow: false);
             }
 
             // Otherwise, we have a conflicting change in the last backflow PR (before merging)
@@ -266,7 +262,19 @@ await RecreatePreviousFlowAndApplyBuild(
 
         await targetRepo.CommitAsync(commitMessage, allowEmpty: false, cancellationToken: cancellationToken);
         await targetRepo.ResetWorkingTree();
-        await workBranch.MergeBackAsync(commitMessage);
+
+        try
+        {
+            await workBranch.MergeBackAsync(commitMessage);
+        }
+        catch (ProcessFailedException e) when (headBranchExisted && e.ExecutionResult.StandardError.Contains("CONFLICT (content): Merge conflict"))
+        {
+            _logger.LogWarning("Failed to merge back the work branch {branchName} into {mainBranch}: {error}",
+                newBranchName,
+                headBranch,
+                e.Message);
+            throw new ConflictInPrBranchException(e.ExecutionResult.StandardError, targetBranch, mapping.Name, isForwardFlow: false);
+        }
 
         _logger.LogInformation("Branch {branch} with code changes is ready in {repoDir}", headBranch, targetRepo);
 

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrCodeflower.cs

@@ -228,7 +228,7 @@ protected abstract Task<bool> OppositeDirectionFlowAsync(
 
         if (objectType1 != GitObjectType.Commit || objectType2 != GitObjectType.Commit)
         {
-            throw new InvalidSynchronizationException($"Failed to find one or both commits {lastBackflow.VmrSha}, {lastForwardFlow.VmrSha} in {sourceRepo}");
+            throw new InvalidSynchronizationException($"Failed to find one or both commits {backwardSha}, {forwardSha} in {sourceRepo}");
         }
 
         // If the SHA's are the same, it's a commit created by inflow which was then flown out
@@ -253,6 +253,34 @@ protected abstract Task<bool> OppositeDirectionFlowAsync(
             throw new InvalidSynchronizationException($"Failed to determine which commit of {sourceRepo} is older ({backwardSha}, {forwardSha})");
         }
 
+        // When the last backflow to our repo came from a different branch, we ignore it and return the last forward flow.
+        // Some repositories do not snap branches for preview (e.g. razor, roslyn, msbuild), and forward-flow their main
+        // branch into both the main branch and preview branch of the VMR.
+        // Forward-flows into preview branches should not accept backflows as the previous flow on which to compute 
+        // the deltas, because those commits come from the main branch VMR. In those cases, we should always return
+        // the previous forward-flow into the same preview branch of the VMR.
+        if (!currentIsBackflow && isForwardOlder)
+        {
+            var vmr = _localGitRepoFactory.Create(_vmrInfo.VmrPath);
+            var currentVmrSha = await vmr.GetShaForRefAsync();
+
+            // We can tell the above by checking if the current target VMR commit is a child of the last backflow commit.
+            // For normal flows it should be, but for the case described above it will be on a different branch.
+            if (!await vmr.IsAncestorCommit(lastBackflow.VmrSha, currentVmrSha))
+            {
+                _logger.LogWarning("Last detected backflow ({sha1}) from VMR is from a different branch than target VMR sha ({sha2}). " +
+                    "Ignoring backflow and considering the last forward flow to be the last flow.",
+                    lastBackflow.VmrSha,
+                    currentVmrSha);
+                
+                return (
+                    lastForwardFlow,
+                    lastBackflow,
+                    lastForwardFlow
+                );
+            }
+        }
+
         return
         (
             isBackwardOlder ? lastForwardFlow : lastBackflow,

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrForwardFlower.cs

@@ -143,7 +143,6 @@ public async Task<CodeFlowResult> FlowForwardAsync(
             hasChanges,
             conflictedFiles ?? [], 
             sourceRepo.Path,
-            lastForwardFlow.SourceSha,
             DependencyUpdates: []);
     }
 
@@ -222,7 +221,7 @@ protected override async Task<bool> SameDirectionFlowAsync(
             if (headBranchExisted)
             {
                 _logger.LogInformation("Failed to update a PR branch because of a conflict. Stopping the flow..");
-                throw new ConflictInPrBranchException(e.Result, targetBranch, mapping.Name, isForwardFlow: true);
+                throw new ConflictInPrBranchException(e.Result.StandardError, targetBranch, mapping.Name, isForwardFlow: true);
             }
 
             // This happens when a conflicting change was made in the last backflow PR (before merging)

src/ProductConstructionService/ProductConstructionService.BarViz/Pages/Subscriptions.razor

@@ -125,7 +125,7 @@
                         @(context.LastAppliedBuild == null ? "N/A" : (DateTime.UtcNow - context.LastAppliedBuild.DateProduced).ToTimeAgo())
                     </FluentLabel>
                 </TemplateColumn>
-                <TemplateColumn Style="width:60px">
+                <TemplateColumn Width="60px">
                     <SubscriptionContextMenu Subscription="@context" Refresh="@OnParametersSetAsync" ShowDetails="@ShowDetails" />
                 </TemplateColumn>
             </ChildContent>

src/ProductConstructionService/ProductConstructionService.DependencyFlow/PcsVmrBackFlower.cs

@@ -95,7 +95,6 @@ public async Task<CodeFlowResult> FlowBackAsync(
         {
             // For already existing PRs, we want to always push the changes (even if only the <Source> tag changed)
             HadUpdates = result.HadUpdates || headBranchExisted,
-            PreviouslyFlownSha = lastFlows.LastBackFlow?.SourceSha,
         };
     }
 

src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestConflictNotifier.cs

@@ -69,7 +69,7 @@ public async Task NotifyAboutConflictingUpdateAsync(
         {
             var (fileUrlInVmr, fileUrlInRepo) = GetFileUrls(update, subscription, filePath, prHeadBranch);
             string vmrString = $"[🔍 View in VMR]({fileUrlInVmr})";
-            string repoString = $"[🔍 View in {GitRepoUrlUtils.GetRepoNameWithOrg(subscription.TargetRepository)}]({fileUrlInRepo})";
+            string repoString = $"[🔍 View in {GitRepoUrlUtils.GetRepoNameWithOrg(subscription.IsBackflow() ? subscription.TargetRepository : subscription.SourceRepository)}]({fileUrlInRepo})";
             sb.AppendLine($" - `{filePath}` - {repoString} / {vmrString}");
         }
         sb.AppendLine();
@@ -87,7 +87,7 @@ public async Task NotifyAboutConflictingUpdateAsync(
         }
     }
 
-    private (string, string) GetFileUrls(SubscriptionUpdateWorkItem update,
+    private static (string, string) GetFileUrls(SubscriptionUpdateWorkItem update,
         Subscription subscription,
         string filePath,
         string prHeadBranch)
@@ -122,12 +122,6 @@ public async Task NotifyAboutMergeConflictAsync(
 
         if (subscription.IsBackflow())
         {
-            if (!conflictedFiles.Any(f => f.Path.Equals(VersionFiles.VersionDetailsXml, StringComparison.InvariantCultureIgnoreCase)))
-            {
-                // No version files in conflict, so we don't need to post a comment
-                return;
-            }
-
             metadataFile = VersionFiles.VersionDetailsXml;
             contentType = "xml";
             var sourceMetadata = new SourceDependency(
@@ -139,13 +133,6 @@ public async Task NotifyAboutMergeConflictAsync(
         }
         else
         {
-            if (!conflictedFiles.Any(f => f.Path.Equals(VmrInfo.DefaultRelativeSourceManifestPath.Path, StringComparison.InvariantCultureIgnoreCase)
-                || f.Path.StartsWith(VmrInfo.GitInfoSourcesDir)))
-            {
-                // No version files in conflict, so we don't need to post a comment
-                return;
-            }
-
             metadataFile = VmrInfo.DefaultRelativeSourceManifestPath;
             contentType = "json";
             correctContent = JsonSerializer.Serialize(

src/ProductConstructionService/ProductConstructionService.DependencyFlow/PullRequestUpdater.cs

@@ -1015,30 +1015,43 @@ private async Task ProcessCodeFlowUpdateAsync(
             subscription.TargetBranch,
             prHeadBranch);
 
+        IRemote remote = await _remoteFactory.CreateRemoteAsync(subscription.TargetRepository);
+
         NativePath localRepoPath;
         CodeFlowResult codeFlowRes;
-        string? previousSourceSha;
+        string? previousSourceSha; // is null in some edge cases like onboarding a new repository
 
         try
         {
             if (isForwardFlow)
             {
                 codeFlowRes = await _vmrForwardFlower.FlowForwardAsync(subscription, build, prHeadBranch, cancellationToken: default);
                 localRepoPath = _vmrInfo.VmrPath;
-                previousSourceSha = codeFlowRes.PreviouslyFlownSha;
+
+                SourceManifest? sourceManifest = await remote.GetSourceManifestAsync(
+                    subscription.TargetRepository,
+                    subscription.TargetBranch);
+
+                previousSourceSha = sourceManifest?
+                    .GetRepoVersion(subscription.TargetDirectory)?.CommitSha;
             }
             else
             {
                 codeFlowRes = await _vmrBackFlower.FlowBackAsync(subscription, build, prHeadBranch, cancellationToken: default);
                 localRepoPath = codeFlowRes.RepoPath;
-                previousSourceSha = codeFlowRes.PreviouslyFlownSha;
+
+                SourceDependency? sourceDependency = await remote.GetSourceDependencyAsync(
+                    subscription.TargetRepository,
+                    subscription.TargetBranch);
+
+                previousSourceSha = sourceDependency?.Sha;
             }
         }
         catch (ConflictInPrBranchException conflictException)
         {
             if (pr != null)
             {
-                await HandlePrUpdateConflictAsync(conflictException.FilesInConflict, update, subscription, pr, prHeadBranch);
+                await HandlePrUpdateConflictAsync(conflictException.ConflictedFiles, update, subscription, pr, prHeadBranch);
             }
             return;
         }