[Rollout] Production rollout 2025-05-08 (#4818)

#4817

Author: dkurepa

arcade-services.sln

@@ -94,8 +94,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tools", "tools", "{02EA681E
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ProductConstructionService.ReproTool", "tools\ProductConstructionService.ReproTool\ProductConstructionService.ReproTool.csproj", "{261CB211-6023-8025-48E1-D11953F4C61C}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FlatFlowMigrationCli", "tools\FlatFlowMigrationCli\FlatFlowMigrationCli.csproj", "{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}"
-EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tools.Common", "tools\Tools.Common\Tools.Common.csproj", "{977060F4-EEEF-420D-02E8-5E58D3E51942}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "VersionPropsFormatter", "tools\VersionPropsFormatter\VersionPropsFormatter.csproj", "{40B3EEE3-2B38-4264-A32F-173D129090F8}"
@@ -508,18 +506,6 @@ Global
 		{261CB211-6023-8025-48E1-D11953F4C61C}.Release|x64.Build.0 = Release|Any CPU
 		{261CB211-6023-8025-48E1-D11953F4C61C}.Release|x86.ActiveCfg = Release|Any CPU
 		{261CB211-6023-8025-48E1-D11953F4C61C}.Release|x86.Build.0 = Release|Any CPU
-		{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Debug|x64.ActiveCfg = Debug|Any CPU
-		{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Debug|x64.Build.0 = Debug|Any CPU
-		{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Debug|x86.ActiveCfg = Debug|Any CPU
-		{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Debug|x86.Build.0 = Debug|Any CPU
-		{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|Any CPU.Build.0 = Release|Any CPU
-		{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|x64.ActiveCfg = Release|Any CPU
-		{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|x64.Build.0 = Release|Any CPU
-		{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|x86.ActiveCfg = Release|Any CPU
-		{89DF188B-B1FC-D3C4-A76E-019144ABA9CB}.Release|x86.Build.0 = Release|Any CPU
 		{977060F4-EEEF-420D-02E8-5E58D3E51942}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{977060F4-EEEF-420D-02E8-5E58D3E51942}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{977060F4-EEEF-420D-02E8-5E58D3E51942}.Debug|x64.ActiveCfg = Debug|Any CPU
@@ -597,7 +583,6 @@ Global
 		{0F27CF0C-4C95-DB17-AF84-78197F32FEBE} = {243A4561-BF35-405A-AF12-AC57BB27796D}
 		{B95B12A8-FCE1-618A-CA77-134B59A5C050} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
 		{261CB211-6023-8025-48E1-D11953F4C61C} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
-		{89DF188B-B1FC-D3C4-A76E-019144ABA9CB} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
 		{977060F4-EEEF-420D-02E8-5E58D3E51942} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
 		{40B3EEE3-2B38-4264-A32F-173D129090F8} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8}
 		{9B43D741-AED1-42DA-88EA-19CB1D71C41C} = {1A456CF0-C09A-4DE6-89CE-1110EED31180}

src/Microsoft.DotNet.Darc/DarcLib/GitHubClient.cs

@@ -130,9 +130,9 @@ private async Task<string> GetFileContentsAsync(string owner, string repo, strin
 
             return this.GetDecodedContent(content);
         }
-        catch (HttpRequestException reqEx) when (reqEx.Message.Contains(((int)HttpStatusCode.NotFound).ToString()))
+        catch (HttpRequestException e) when (e.StatusCode == HttpStatusCode.NotFound)
         {
-            throw new DependencyFileNotFoundException(filePath, $"{owner}/{repo}", branch, reqEx);
+            throw new DependencyFileNotFoundException(filePath, $"{owner}/{repo}", branch, e);
         }
     }
 
@@ -176,7 +176,7 @@ public async Task CreateBranchAsync(string repoUri, string newBranch, string bas
 
             _logger.LogInformation("Branch '{branch}' exists, updated", newBranch);
         }
-        catch (HttpRequestException exc) when (exc.Message.Contains(((int)HttpStatusCode.NotFound).ToString()))
+        catch (HttpRequestException e) when (e.StatusCode == HttpStatusCode.NotFound)
         {
             _logger.LogInformation("'{branch}' branch doesn't exist. Creating it...", newBranch);
 
@@ -903,8 +903,8 @@ private HttpClient CreateHttpClient(string repoUri)
 
             return content["sha"]!.ToString();
         }
-        catch (HttpRequestException exc) when (exc.Message.Contains(((int)HttpStatusCode.NotFound).ToString())
-                                               || exc.Message.Contains(((int)HttpStatusCode.UnprocessableEntity).ToString()))
+        catch (HttpRequestException e) when (e.StatusCode == HttpStatusCode.NotFound
+                                          || e.StatusCode == HttpStatusCode.UnprocessableEntity)
         {
             return null;
         }
@@ -1239,7 +1239,7 @@ public async Task<GitDiff> GitDiffAsync(string repoUri, string baseVersion, stri
                 Valid = true
             };
         }
-        catch (HttpRequestException reqEx) when (reqEx.Message.Contains(((int)HttpStatusCode.NotFound).ToString()))
+        catch (HttpRequestException e) when (e.StatusCode == HttpStatusCode.NotFound)
         {
             return GitDiff.UnknownDiff();
         }

src/Microsoft.DotNet.Darc/DarcLib/Helpers/DependencyFileManager.cs

@@ -158,9 +158,22 @@ public async Task<SemanticVersion> ReadToolsDotnetVersionAsync(string repoUri, s
         return dotnetVersion;
     }
 
-    public async Task<XmlDocument> ReadNugetConfigAsync(string repoUri, string branch)
+    public async Task<(string Name, XmlDocument Content)> ReadNugetConfigAsync(string repoUri, string branch)
     {
-        return await ReadXmlFileAsync(VersionFiles.NugetConfig, repoUri, branch);
+        foreach (var name in VersionFiles.NugetConfigNames)
+        {
+            try
+            {
+                return (name, await ReadXmlFileAsync(name, repoUri, branch));
+            }
+            catch (DependencyFileNotFoundException)
+            {
+                continue;
+            }
+        }
+
+        throw new DependencyFileNotFoundException(
+            $"None of the {VersionFiles.NugetConfigNames.First()} variations were found in the repo '{repoUri}' and branch '{branch}'");
     }
 
     public async Task<VersionDetails> ParseVersionDetailsXmlAsync(string repoUri, string branch, bool includePinned = true)
@@ -391,7 +404,7 @@ public async Task<GitFileContentContainer> UpdateDependencyFiles(
         XmlDocument versionProps = await ReadVersionPropsAsync(repoUri, branch);
         JObject globalJson = await ReadGlobalJsonAsync(repoUri, branch, repoIsVmr);
         JObject toolsConfigurationJson = await ReadDotNetToolsConfigJsonAsync(repoUri, branch, repoIsVmr);
-        XmlDocument nugetConfig = await ReadNugetConfigAsync(repoUri, branch);
+        (string nugetConfigName, XmlDocument nugetConfig) = await ReadNugetConfigAsync(repoUri, branch);
 
         foreach (DependencyDetail itemToUpdate in itemsToUpdate)
         {
@@ -427,7 +440,7 @@ public async Task<GitFileContentContainer> UpdateDependencyFiles(
         // At this point we only care about the Maestro managed locations for the assets.
         // Flatten the dictionary into a set that has all the managed feeds
         Dictionary<string, HashSet<string>> managedFeeds = FlattenLocationsAndSplitIntoGroups(itemsToUpdateLocations);
-        var updatedNugetConfig = UpdatePackageSources(nugetConfig, managedFeeds);
+        nugetConfig = UpdatePackageSources(nugetConfig, managedFeeds);
 
         // Update the dotnet sdk if necessary
         Dictionary<GitFileMetadataName, string> globalJsonMetadata = null;
@@ -441,7 +454,7 @@ public async Task<GitFileContentContainer> UpdateDependencyFiles(
             GlobalJson = new GitFile(VersionFiles.GlobalJson, globalJson, globalJsonMetadata),
             VersionDetailsXml = new GitFile(VersionFiles.VersionDetailsXml, versionDetails),
             VersionProps = new GitFile(VersionFiles.VersionProps, versionProps),
-            NugetConfig = new GitFile(VersionFiles.NugetConfig, updatedNugetConfig)
+            NugetConfig = new GitFile(nugetConfigName, nugetConfig),
         };
 
         // dotnet-tools.json is optional, so only include it if it was found.

src/Microsoft.DotNet.Darc/DarcLib/Helpers/IDependencyFileManager.cs

@@ -40,7 +40,7 @@ public interface IDependencyFileManager
 
     Task<JObject> ReadGlobalJsonAsync(string repoUri, string branch, bool repoIsVmr);
 
-    Task<XmlDocument> ReadNugetConfigAsync(string repoUri, string branch);
+    Task<(string Name, XmlDocument Content)> ReadNugetConfigAsync(string repoUri, string branch);
 
     Task<XmlDocument> ReadVersionDetailsXmlAsync(string repoUri, string branch);
 

src/Microsoft.DotNet.Darc/DarcLib/Helpers/VersionFiles.cs

@@ -3,6 +3,7 @@
 
 using NuGet.Versioning;
 using System;
+using System.Collections.Generic;
 
 namespace Microsoft.DotNet.DarcLib.Helpers;
 
@@ -16,7 +17,7 @@ public static class VersionFiles
     public const string VersionProps = "eng/Versions.props";
     public const string GlobalJson = "global.json";
     public const string DotnetToolsConfigJson = ".config/dotnet-tools.json";
-    public const string NugetConfig = "NuGet.config";
+    public static readonly IReadOnlyCollection<string> NugetConfigNames = ["NuGet.config", "nuget.config", "NuGet.Config"];
 
     private static string GetVersionPropsElementBaseName(string dependencyName)
     {

src/Microsoft.DotNet.Darc/DarcLib/Remote.cs

@@ -154,7 +154,7 @@ await _remoteGitClient.MergeDependencyPullRequestAsync(pullRequestUrl,
     public async Task<IEnumerable<string>> GetPackageSourcesAsync(string repoUri, string commit)
     {
         CheckForValidGitClient();
-        XmlDocument nugetConfig = await _fileManager.ReadNugetConfigAsync(repoUri, commit);
+        (_, XmlDocument nugetConfig) = await _fileManager.ReadNugetConfigAsync(repoUri, commit);
         return _fileManager.GetPackageSources(nugetConfig).Select(nameAndFeed => nameAndFeed.feed);
     }
 
@@ -222,10 +222,11 @@ public async Task<List<GitFile>> CommitUpdatesAsync(
             {
                 engCommonFiles = engCommonFiles
                     .Select(f => new GitFile(
-                        f.FilePath.Replace(VmrInfo.ArcadeRepoDir, null).TrimStart('/'),
+                        f.FilePath.Replace(VmrInfo.ArcadeRepoDir, null, StringComparison.InvariantCultureIgnoreCase).TrimStart('/'),
                         f.Content,
                         f.ContentEncoding,
-                        f.Mode))
+                        f.Mode,
+                        f.Operation))
                     .ToList();
             }
 
@@ -239,7 +240,7 @@ public async Task<List<GitFile>> CommitUpdatesAsync(
 
             foreach (GitFile file in targetEngCommonFiles)
             {
-                if (!engCommonFiles.Where(f => f.FilePath == file.FilePath).Any())
+                if (!engCommonFiles.Any(f => f.FilePath.Equals(file.FilePath, StringComparison.InvariantCultureIgnoreCase)))
                 {
                     deletedFiles.Add(file.FilePath);
                     // This is a file in the repo's eng/common folder that isn't present in Arcade at the
@@ -258,10 +259,16 @@ public async Task<List<GitFile>> CommitUpdatesAsync(
 
             if (deletedFiles.Count > 0)
             {
-                _logger.LogInformation($"Dependency update from Arcade commit {arcadeItem.Commit} to {repoUri} " +
-                                       $"on branch {branch}@{latestCommit} will delete files in eng/common." +
-                                       $" Source file count: {engCommonFiles.Count}, Target file count: {targetEngCommonFiles.Count}." +
-                                       $" Deleted files: {string.Join(Environment.NewLine, deletedFiles)}");
+                _logger.LogInformation(
+                    "Dependency update from Arcade commit {commit} to {repoUri} on branch {branch}@{latestCommit} will delete files in eng/common. " +
+                    "Source file count: {sourceFileCount}, Target file count: {targetFileCount}. Deleted files: {deletedFiles}",
+                    arcadeItem.Commit,
+                    repoUri,
+                    branch,
+                    latestCommit,
+                    engCommonFiles.Count,
+                    targetEngCommonFiles.Count,
+                    string.Join(Environment.NewLine, deletedFiles));
             }
         }
 

src/Microsoft.DotNet.Darc/DarcLib/RemoteRepoBase.cs

@@ -172,7 +172,7 @@ private async Task<string> SparseAndShallowCheckoutAsync(
         await ExecuteGitCommand(["config", "user.name", user], workingDirectory);
         await ExecuteGitCommand(["config", "user.email", email], workingDirectory);
 
-        File.WriteAllLines(Path.Combine(workingDirectory, ".git/info/sparse-checkout"), ["eng/", ".config/", $"/{VersionFiles.NugetConfig}", $"/{VersionFiles.GlobalJson}"]);
+        File.WriteAllLines(Path.Combine(workingDirectory, ".git/info/sparse-checkout"), ["eng/", ".config/", $"/{VersionFiles.NugetConfigNames.First()}", $"/{VersionFiles.GlobalJson}"]);
 
         await ExecuteGitCommand([$"-c", "core.askpass=", "-c", "credential.helper=", "pull", "--depth=1", remote, branch], workingDirectory, secretToMask: pat);
         await ExecuteGitCommand([$"checkout", branch], workingDirectory);

src/Microsoft.DotNet.Darc/DarcLib/VirtualMonoRepo/VmrBackflower.cs

@@ -9,7 +9,6 @@
 using LibGit2Sharp;
 using Microsoft.DotNet.DarcLib.Helpers;
 using Microsoft.DotNet.DarcLib.Models;
-using Microsoft.DotNet.DarcLib.Models.Darc;
 using Microsoft.DotNet.DarcLib.Models.VirtualMonoRepo;
 using Microsoft.DotNet.ProductConstructionService.Client.Models;
 using Microsoft.Extensions.Logging;
@@ -410,18 +409,26 @@ await _repositoryCloneManager.PrepareCloneAsync(
         }
     }
 
-    private IReadOnlyCollection<string> GetPatchExclusions(SourceMapping mapping) =>
+    private IReadOnlyCollection<string> GetPatchExclusions(SourceMapping mapping)
+    {
         // Exclude all submodules that belong to the mapping
-        [.._sourceManifest.Submodules
+        var exclusions = _sourceManifest.Submodules
             .Where(s => s.Path.StartsWith(mapping.Name + '/'))
-            .Select(s => s.Path.Substring(mapping.Name.Length + 1))
+            .Select(s => s.Path.Substring(mapping.Name.Length + 1));
 
         // Exclude version files as those will be handled manually
-        .Concat(DependencyFileManager.DependencyFiles)
+        exclusions = exclusions
+            .Concat(DependencyFileManager.DependencyFiles);
 
-        // Exclude eng/common as that will be copied based on the arcade version
-        .Append(Constants.CommonScriptFilesPath)
-        .Select(VmrPatchHandler.GetExclusionRule)];
+        // Exclude eng/common for non-arcade mappings (it will be copied separately based on the Arcade.Sdk package version)
+        if (mapping.Name != "arcade")
+        {
+            exclusions = exclusions
+                .Append(Constants.CommonScriptFilesPath);
+        }
+
+        return [.. exclusions.Select(VmrPatchHandler.GetExclusionRule)];
+    }
 
     private async Task RecreatePreviousFlowAndApplyBuild(
         SourceMapping mapping,

src/ProductConstructionService/ProductConstructionService.DependencyFlow/PcsVmrBackFlower.cs

@@ -115,7 +115,12 @@ await _vmrCloneManager.PrepareVmrAsync(
 
         // Prepare repo
         SourceMapping mapping = _dependencyTracker.GetMapping(subscription.SourceDirectory);
-        var remotes = new[] { mapping.DefaultRemote, _sourceManifest.GetRepoVersion(mapping.Name).RemoteUri }
+        var remotes = new[]
+            {
+                mapping.DefaultRemote,
+                _sourceManifest.GetRepoVersion(mapping.Name).RemoteUri,
+                subscription.TargetRepository
+            }
             .Distinct()
             .OrderRemotesByLocalPublicOther()
             .ToList();

test/Microsoft.DotNet.Darc.Tests/DependencyCoherencyTests.cs

@@ -1245,7 +1245,7 @@ private void RepositoryHasFeeds(string repo, string commit, string[] feeds)
             string.Join(Environment.NewLine, feeds.Select(feed => $@"<add key = ""{GetRandomId()}"" value = ""{feed}"" />")));
 
         _gitRepoMock
-            .Setup(m => m.GetFileContentsAsync(VersionFiles.NugetConfig, repo, commit))
+            .Setup(m => m.GetFileContentsAsync(VersionFiles.NugetConfigNames.First(), repo, commit))
             .ReturnsAsync(nugetConfig);
     }