diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index bef19c7b..f7466ce7 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -228,19 +228,77 @@ jobs: /p:Version=$env:BUILD_VERSION ` /p:PublishSingleFile=false ` -o .\artifacts\standalone + - name: Fetch existing update catalog from B2 + shell: pwsh + env: + B2_BUCKET: ${{ secrets.B2_BUCKET }} + B2_APPKEY_ID: ${{ secrets.B2_APPKEY_ID }} + B2_APPKEY: ${{ secrets.B2_APPKEY }} + run: | + if (-not (Get-Command python -ErrorAction SilentlyContinue)) { + Write-Host "Python not available, skipping catalog fetch." + exit 0 + } + pip install --quiet --cache-dir C:\pip-cache "b2==$env:B2_CLI_VERSION" + b2 account authorize $env:B2_APPKEY_ID $env:B2_APPKEY --quiet + if ($LASTEXITCODE -ne 0) { + Write-Host "B2 auth failed, will create fresh catalog." + exit 0 + } + try { + b2 download-file-by-name $env:B2_BUCKET TelegramSearchBot/catalog.json .\artifacts\existing-catalog.json --quiet + Write-Host "Downloaded existing catalog.json for merge." + } catch { + Write-Host "No existing catalog found on B2, will create fresh catalog." + } + b2 account clear + - name: Fetch previous release standalone for step package comparison + shell: pwsh + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + $prevRelease = gh release list --repo ${{ github.repository }} --limit 1 --exclude-drafts --json tagName,name | ConvertFrom-Json + if ($prevRelease.Count -gt 0) { + $prevTag = $prevRelease[0].tagName + $prevVersion = $prevTag -replace '^v', '' + Write-Host "Previous release: $prevTag (version: $prevVersion)" + $prevArchive = ".\artifacts\prev-standalone.zip" + gh release download $prevTag --repo ${{ github.repository }} --pattern "TelegramSearchBot-win-x64-full-*.zip" --dir artifacts\prev-release + $zipFile = Get-ChildItem artifacts\prev-release -Filter "TelegramSearchBot-win-x64-full-*.zip" | Select-Object -First 1 + if ($zipFile) { + Expand-Archive -Path $zipFile.FullName -DestinationPath .\artifacts\prev-standalone -Force + Write-Host "PREV_VERSION=$prevVersion" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + Write-Host "PREV_STANDALONE_DIR=artifacts\prev-standalone" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + Write-Host "Extracted previous standalone from $($zipFile.Name)." + } else { + Write-Host "No previous full zip found; step package will not be generated." + } + } else { + Write-Host "No previous release found; this is the first release." + } - name: Build Moder.Update feed shell: pwsh env: BUILD_VERSION: ${{ needs.prepare.outputs.build-version }} run: | + $builderArgs = @( + '--source-dir', '.\artifacts\standalone', + '--output-dir', '.\artifacts\update-feed', + '--target-version', $env:BUILD_VERSION, + '--min-source-version', $env:LEGACY_BRIDGE_VERSION + ) + if (Test-Path '.\artifacts\existing-catalog.json') { + $builderArgs += @('--existing-catalog', '.\artifacts\existing-catalog.json') + Write-Host "Passing existing catalog for merge." + } + if ($env:PREV_VERSION -and (Test-Path $env:PREV_STANDALONE_DIR)) { + $builderArgs += @('--prev-source-dir', $env:PREV_STANDALONE_DIR, '--prev-version', $env:PREV_VERSION) + Write-Host "Passing previous version $env:PREV_VERSION for step package generation." + } dotnet run --project .\TelegramSearchBot.UpdateBuilder\TelegramSearchBot.UpdateBuilder.csproj ` -c Release ` --no-build ` - -- ` - --source-dir .\artifacts\standalone ` - --output-dir .\artifacts\update-feed ` - --target-version $env:BUILD_VERSION ` - --min-source-version $env:LEGACY_BRIDGE_VERSION + -- @builderArgs Copy-Item .\moder-update-bin\moder_update_updater.exe .\artifacts\update-feed\moder_update_updater.exe -Force $catalogPath = '.\artifacts\update-feed\catalog.json' $catalog = Get-Content $catalogPath -Raw | ConvertFrom-Json @@ -415,29 +473,35 @@ jobs: } } - function Prune-B2PackageVersions { + function Prune-B2PackagesByCatalog { param( [Parameter(Mandatory = $true)] - [string[]]$KeepRelativePaths + [string]$CatalogPath ) - $items = @(Get-B2Versions -B2Uri "b2://$env:B2_BUCKET/TelegramSearchBot/packages/" -Recursive) - if ($items.Count -eq 0) { + if (!(Test-Path $CatalogPath)) { + Write-Host "Catalog not found, skipping reachability prune." return } - $keepSet = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) - foreach ($relativePath in $KeepRelativePaths) { - $normalizedPath = $relativePath.Replace('\', '/') - [void]$keepSet.Add("TelegramSearchBot/packages/$normalizedPath") + $catalog = Get-Content $CatalogPath -Raw | ConvertFrom-Json + $reachablePaths = [System.Collections.Generic.HashSet[string]]::new([System.StringComparer]::OrdinalIgnoreCase) + foreach ($entry in $catalog.Entries) { + if ($entry.PackagePath) { + $normalized = $entry.PackagePath -replace '^packages/', '' + [void]$reachablePaths.Add("TelegramSearchBot/packages/$normalized") + } } + Write-Host "Catalog has $($catalog.Entries.Count) entries, $($reachablePaths.Count) reachable package paths." + + $items = @(Get-B2Versions -B2Uri "b2://$env:B2_BUCKET/TelegramSearchBot/packages/" -Recursive) + if ($items.Count -eq 0) { return } foreach ($group in ($items | Group-Object fileName)) { $sortedItems = $group.Group | Sort-Object uploadTimestamp -Descending - if ($keepSet.Contains($group.Name)) { + if ($reachablePaths.Contains($group.Name)) { $sortedItems = $sortedItems | Select-Object -Skip 1 } - foreach ($item in $sortedItems) { Remove-B2VersionById -FileId $item.fileId } @@ -463,11 +527,7 @@ jobs: b2 upload-file $env:B2_BUCKET .\artifacts\update-feed\moder_update_updater.exe TelegramSearchBot/moder_update_updater.exe b2 upload-file $env:B2_BUCKET .\artifacts\update-feed\catalog.json TelegramSearchBot/catalog.json - $packageRoot = (Resolve-Path .\artifacts\update-feed\packages).Path - $packageFiles = Get-ChildItem $packageRoot -File -Recurse | ForEach-Object { - [System.IO.Path]::GetRelativePath($packageRoot, $_.FullName) - } - Prune-B2PackageVersions -KeepRelativePaths $packageFiles + Prune-B2PackagesByCatalog -CatalogPath '.\artifacts\update-feed\catalog.json' Prune-B2FileVersions -RelativePath 'TelegramSearchBot/moder_update_updater.exe' Prune-B2FileVersions -RelativePath 'TelegramSearchBot/catalog.json' b2 account clear diff --git a/TelegramSearchBot.Common/Model/Update/UpdateCatalog.cs b/TelegramSearchBot.Common/Model/Update/UpdateCatalog.cs new file mode 100644 index 00000000..4dcd630c --- /dev/null +++ b/TelegramSearchBot.Common/Model/Update/UpdateCatalog.cs @@ -0,0 +1,10 @@ +namespace TelegramSearchBot.Common.Model.Update; + +public sealed class UpdateCatalog +{ + public required string LatestVersion { get; init; } + public required List Entries { get; init; } + public DateTime LastUpdated { get; init; } + public string? MinRequiredVersion { get; init; } + public string? UpdaterChecksum { get; init; } +} diff --git a/TelegramSearchBot.Common/Model/Update/UpdateCatalogEntry.cs b/TelegramSearchBot.Common/Model/Update/UpdateCatalogEntry.cs new file mode 100644 index 00000000..fdd7128f --- /dev/null +++ b/TelegramSearchBot.Common/Model/Update/UpdateCatalogEntry.cs @@ -0,0 +1,16 @@ +namespace TelegramSearchBot.Common.Model.Update; + +public sealed class UpdateCatalogEntry +{ + public required string PackagePath { get; init; } + public required string TargetVersion { get; init; } + public required string MinSourceVersion { get; init; } + public string? MaxSourceVersion { get; init; } + public bool IsCumulative { get; init; } + public bool IsAnchor { get; init; } + public int ChainDepth { get; init; } + public required string PackageChecksum { get; init; } + public int FileCount { get; init; } + public long CompressedSize { get; init; } + public long UncompressedSize { get; init; } +} diff --git a/TelegramSearchBot.Common/Model/Update/UpdateFile.cs b/TelegramSearchBot.Common/Model/Update/UpdateFile.cs new file mode 100644 index 00000000..11169311 --- /dev/null +++ b/TelegramSearchBot.Common/Model/Update/UpdateFile.cs @@ -0,0 +1,7 @@ +namespace TelegramSearchBot.Common.Model.Update; + +public sealed class UpdateFile +{ + public required string RelativePath { get; init; } + public required string NewChecksum { get; init; } +} diff --git a/TelegramSearchBot.Common/Model/Update/UpdateManifest.cs b/TelegramSearchBot.Common/Model/Update/UpdateManifest.cs new file mode 100644 index 00000000..3d4114c5 --- /dev/null +++ b/TelegramSearchBot.Common/Model/Update/UpdateManifest.cs @@ -0,0 +1,14 @@ +namespace TelegramSearchBot.Common.Model.Update; + +public sealed class UpdateManifest +{ + public required string TargetVersion { get; init; } + public required string MinSourceVersion { get; init; } + public string? MaxSourceVersion { get; init; } + public bool IsAnchor { get; init; } + public bool IsCumulative { get; init; } + public int ChainDepth { get; init; } + public required List Files { get; init; } + public required string Checksum { get; init; } + public DateTime CreatedAt { get; init; } +} diff --git a/TelegramSearchBot.UpdateBuilder/Program.cs b/TelegramSearchBot.UpdateBuilder/Program.cs index 15083057..dc1e067a 100644 --- a/TelegramSearchBot.UpdateBuilder/Program.cs +++ b/TelegramSearchBot.UpdateBuilder/Program.cs @@ -2,64 +2,188 @@ using System.Security.Cryptography; using System.Text; using System.Text.Json; +using TelegramSearchBot.Common.Model.Update; using ZstdSharp; var arguments = BuilderArguments.Parse(args); -if (!arguments.IsValid) { +if (!arguments.IsValid) +{ BuilderArguments.PrintUsage(); return 1; } var sourceDirectory = Path.GetFullPath(arguments.SourceDirectory!); var outputDirectory = Path.GetFullPath(arguments.OutputDirectory!); -if (!Directory.Exists(sourceDirectory)) { + +if (!Directory.Exists(sourceDirectory)) +{ Console.Error.WriteLine($"Source directory not found: {sourceDirectory}"); return 1; } if (Path.GetFullPath(sourceDirectory).TrimEnd(Path.DirectorySeparatorChar) - .Equals(Path.GetFullPath(outputDirectory).TrimEnd(Path.DirectorySeparatorChar), StringComparison.OrdinalIgnoreCase)) { + .Equals(Path.GetFullPath(outputDirectory).TrimEnd(Path.DirectorySeparatorChar), + StringComparison.OrdinalIgnoreCase)) +{ Console.Error.WriteLine("Source and output directories must be different."); return 1; } -var packageRelativePath = $"packages/update-{SanitizeVersion(arguments.TargetVersion!)}.zst"; -var packagePath = Path.Combine(outputDirectory, packageRelativePath.Replace('/', Path.DirectorySeparatorChar)); -Directory.CreateDirectory(Path.GetDirectoryName(packagePath)!); Directory.CreateDirectory(outputDirectory); +var packagesDir = Path.Combine(outputDirectory, "packages"); +Directory.CreateDirectory(packagesDir); -var files = LoadFiles(sourceDirectory); -var manifestFiles = files - .Select(file => new UpdateFile { - RelativePath = file.RelativePath, - NewChecksum = ComputeSha512(file.Content) - }) - .ToList(); +var currentFiles = LoadFiles(sourceDirectory); +var currentVersion = arguments.TargetVersion!; +var catalogEntries = new List(); -var tempManifest = CreateManifest(arguments, manifestFiles, checksum: string.Empty); -var tempPackage = CreatePackage(tempManifest, files); -var finalManifest = CreateManifest(arguments, manifestFiles, ComputeSha512(tempPackage)); -var finalPackage = CreatePackage(finalManifest, files); +UpdateCatalog? existingCatalog = null; +if (!string.IsNullOrWhiteSpace(arguments.ExistingCatalog) && File.Exists(arguments.ExistingCatalog)) +{ + try + { + var catalogJson = await File.ReadAllTextAsync(arguments.ExistingCatalog); + existingCatalog = JsonSerializer.Deserialize(catalogJson, JsonOptions); + if (existingCatalog?.Entries is { Count: > 0 }) + { + catalogEntries.AddRange(existingCatalog.Entries); + } + } + catch (Exception ex) + { + Console.Error.WriteLine($"Warning: Failed to parse existing catalog, will create fresh. {ex.Message}"); + } +} -await File.WriteAllBytesAsync(packagePath, finalPackage); +catalogEntries.RemoveAll(e => string.Equals(e.TargetVersion, currentVersion, StringComparison.OrdinalIgnoreCase)); -var catalog = new UpdateCatalog { - LatestVersion = arguments.TargetVersion!, - MinRequiredVersion = arguments.MinSourceVersion, - LastUpdated = DateTime.UtcNow, - Entries = [ - new UpdateCatalogEntry { - PackagePath = packageRelativePath.Replace('\\', '/'), - TargetVersion = arguments.TargetVersion!, - MinSourceVersion = arguments.MinSourceVersion!, - MaxSourceVersion = arguments.MaxSourceVersion, - IsCumulative = true, - PackageChecksum = ComputeSha512(finalPackage), - FileCount = files.Count, - CompressedSize = finalPackage.LongLength, - UncompressedSize = files.Sum(file => (long)file.Content.Length) +if (!string.IsNullOrWhiteSpace(arguments.PrevSourceDir) && Directory.Exists(arguments.PrevSourceDir)) +{ + var prevFiles = LoadFiles(arguments.PrevSourceDir); + var prevVersion = arguments.PrevVersion!; + + var changedFiles = ComputeChangedFiles(prevFiles, currentFiles); + if (changedFiles.Count > 0) + { + var stepPackagePath = $"packages/update-{SanitizeVersion(prevVersion)}-to-{SanitizeVersion(currentVersion)}.zst"; + var (stepBytes, stepChecksum) = BuildPackageFile(changedFiles, prevVersion, currentVersion, + arguments.MinSourceVersion, isCumulative: false, isAnchor: false, chainDepth: 1); + + var stepFilePath = Path.Combine(outputDirectory, stepPackagePath.Replace('/', Path.DirectorySeparatorChar)); + Directory.CreateDirectory(Path.GetDirectoryName(stepFilePath)!); + await File.WriteAllBytesAsync(stepFilePath, stepBytes); + + catalogEntries.Add(new UpdateCatalogEntry + { + PackagePath = stepPackagePath.Replace('\\', '/'), + TargetVersion = currentVersion, + MinSourceVersion = prevVersion, + MaxSourceVersion = prevVersion, + IsCumulative = false, + IsAnchor = false, + ChainDepth = 1, + PackageChecksum = stepChecksum, + FileCount = changedFiles.Count, + CompressedSize = stepBytes.LongLength, + UncompressedSize = changedFiles.Sum(f => (long)f.Content.Length) + }); + + Console.WriteLine( + $"Step package written: {stepPackagePath} ({changedFiles.Count} changed files, {stepBytes.LongLength} bytes compressed)"); + } + else + { + Console.WriteLine("No changed files detected; skipping step package generation."); + } +} +else +{ + Console.WriteLine("No --prev-source-dir provided; step package will not be generated this run."); +} + +if (!string.IsNullOrWhiteSpace(arguments.AnchorVersion) && !string.IsNullOrWhiteSpace(arguments.AnchorSourceDir)) +{ + var anchorDir = Path.GetFullPath(arguments.AnchorSourceDir); + if (Directory.Exists(anchorDir)) + { + var anchorFiles = LoadFiles(anchorDir); + var changedFromAnchor = ComputeChangedFiles(anchorFiles, currentFiles); + if (changedFromAnchor.Count > 0) + { + var cumulativePath = + $"packages/update-{SanitizeVersion(arguments.AnchorVersion)}-to-{SanitizeVersion(currentVersion)}-cumulative.zst"; + var (cumBytes, cumChecksum) = BuildPackageFile(changedFromAnchor, arguments.AnchorVersion, + currentVersion, arguments.AnchorVersion, isCumulative: true, isAnchor: true, chainDepth: 0); + + var cumFilePath = Path.Combine(outputDirectory, cumulativePath.Replace('/', Path.DirectorySeparatorChar)); + Directory.CreateDirectory(Path.GetDirectoryName(cumFilePath)!); + await File.WriteAllBytesAsync(cumFilePath, cumBytes); + + catalogEntries.Add(new UpdateCatalogEntry + { + PackagePath = cumulativePath.Replace('\\', '/'), + TargetVersion = currentVersion, + MinSourceVersion = arguments.AnchorVersion, + MaxSourceVersion = null, + IsCumulative = true, + IsAnchor = true, + ChainDepth = 0, + PackageChecksum = cumChecksum, + FileCount = changedFromAnchor.Count, + CompressedSize = cumBytes.LongLength, + UncompressedSize = changedFromAnchor.Sum(f => (long)f.Content.Length) + }); + + Console.WriteLine( + $"Cumulative package written: {cumulativePath} ({changedFromAnchor.Count} files from anchor {arguments.AnchorVersion}, {cumBytes.LongLength} bytes compressed)"); } - ] + } + else + { + Console.WriteLine( + $"Warning: Anchor source dir not found: {anchorDir}, skipping cumulative package generation."); + } +} + +var fallbackStepFrom = + arguments.MinSourceVersion ?? arguments.PrevVersion ?? "unknown"; +var fallbackChangedFiles = currentFiles; +var fallbackPackagePath = + $"packages/update-{SanitizeVersion(fallbackStepFrom)}-to-{SanitizeVersion(currentVersion)}-full.zst"; +var (fallbackBytes, fallbackChecksum) = BuildPackageFile(fallbackChangedFiles, fallbackStepFrom, + currentVersion, arguments.MinSourceVersion, isCumulative: true, isAnchor: true, chainDepth: 0); + +var fallbackFilePath = Path.Combine(outputDirectory, fallbackPackagePath.Replace('/', Path.DirectorySeparatorChar)); +Directory.CreateDirectory(Path.GetDirectoryName(fallbackFilePath)!); +await File.WriteAllBytesAsync(fallbackFilePath, fallbackBytes); + +catalogEntries.Add(new UpdateCatalogEntry +{ + PackagePath = fallbackPackagePath.Replace('\\', '/'), + TargetVersion = currentVersion, + MinSourceVersion = arguments.MinSourceVersion ?? fallbackStepFrom, + MaxSourceVersion = null, + IsCumulative = true, + IsAnchor = true, + ChainDepth = 0, + PackageChecksum = fallbackChecksum, + FileCount = fallbackChangedFiles.Count, + CompressedSize = fallbackBytes.LongLength, + UncompressedSize = fallbackChangedFiles.Sum(f => (long)f.Content.Length) +}); + +Console.WriteLine( + $"Fallback full package written: {fallbackPackagePath} ({fallbackChangedFiles.Count} files, {fallbackBytes.LongLength} bytes compressed)"); + +catalogEntries = PruneCatalogEntries(catalogEntries, currentVersion, arguments.MinSourceVersion, + arguments.PrevVersion, arguments.AnchorVersion); + +var catalog = new UpdateCatalog +{ + LatestVersion = currentVersion, + Entries = catalogEntries, + LastUpdated = DateTime.UtcNow, + MinRequiredVersion = arguments.MinSourceVersion }; var catalogPath = Path.Combine(outputDirectory, "catalog.json"); @@ -67,34 +191,99 @@ await File.WriteAllTextAsync( catalogPath, JsonSerializer.Serialize(catalog, new JsonSerializerOptions { WriteIndented = true })); -Console.WriteLine($"Catalog written: {catalogPath}"); -Console.WriteLine($"Package written: {packagePath}"); -Console.WriteLine($"Target version: {arguments.TargetVersion}"); -Console.WriteLine($"Min source version: {arguments.MinSourceVersion}"); +Console.WriteLine($"Catalog written: {catalogPath} with {catalog.Entries.Count} entries."); +Console.WriteLine($"Target version: {currentVersion}"); +Console.WriteLine($"Min required version: {arguments.MinSourceVersion}"); return 0; -static UpdateManifest CreateManifest(BuilderArguments arguments, List files, string checksum) -{ - return new UpdateManifest { - TargetVersion = arguments.TargetVersion!, - MinSourceVersion = arguments.MinSourceVersion!, - MaxSourceVersion = arguments.MaxSourceVersion, - IsAnchor = false, - IsCumulative = true, - ChainDepth = 0, - Files = files, - Checksum = checksum, +// ================================================================ +// Helper Methods +// ================================================================ + +static List PruneCatalogEntries( + List entries, + string latestVersion, + string? minRequiredVersion, + string? prevVersion, + string? anchorVersion) +{ + const int MaxEntries = 20; + + var latest = entries.Where(e => + string.Equals(e.TargetVersion, latestVersion, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + var rest = entries.Where(e => + !string.Equals(e.TargetVersion, latestVersion, StringComparison.OrdinalIgnoreCase)) + .OrderByDescending(e => e.IsCumulative) + .ThenByDescending(e => Version.TryParse(e.TargetVersion, out var v) ? v : new Version(0, 0)) + .ToList(); + + var result = latest; + result.AddRange(rest.Take(MaxEntries - latest.Count)); + + return result; +} + +static (byte[] PackageBytes, string Checksum) BuildPackageFile( + IReadOnlyList files, + string fromVersion, + string toVersion, + string? minSourceVersion, + bool isCumulative, + bool isAnchor, + int chainDepth) +{ + var manifestFiles = files + .Select(file => new UpdateFile + { + RelativePath = file.RelativePath, + NewChecksum = ComputeSha512(file.Content) + }) + .ToList(); + + var tempManifest = new UpdateManifest + { + TargetVersion = toVersion, + MinSourceVersion = fromVersion, + MaxSourceVersion = isCumulative ? null : fromVersion, + IsAnchor = isAnchor, + IsCumulative = isCumulative, + ChainDepth = chainDepth, + Files = manifestFiles, + Checksum = string.Empty, + CreatedAt = DateTime.UtcNow + }; + + var tempBytes = CreatePackage(tempManifest, files); + var finalChecksum = ComputeSha512(tempBytes); + + var finalManifest = new UpdateManifest + { + TargetVersion = toVersion, + MinSourceVersion = fromVersion, + MaxSourceVersion = isCumulative ? null : fromVersion, + IsAnchor = isAnchor, + IsCumulative = isCumulative, + ChainDepth = chainDepth, + Files = manifestFiles, + Checksum = finalChecksum, CreatedAt = DateTime.UtcNow }; + + var finalBytes = CreatePackage(finalManifest, files); + return (finalBytes, finalChecksum); } static byte[] CreatePackage(UpdateManifest manifest, IReadOnlyList files) { using var tarStream = new MemoryStream(); - using (var tarWriter = new TarWriter(tarStream, leaveOpen: true)) { + using (var tarWriter = new TarWriter(tarStream, leaveOpen: true)) + { WriteTarEntry(tarWriter, "manifest.json", JsonSerializer.SerializeToUtf8Bytes(manifest)); - foreach (var file in files) { + foreach (var file in files) + { WriteTarEntry(tarWriter, file.RelativePath, file.Content); } } @@ -102,7 +291,8 @@ static byte[] CreatePackage(UpdateManifest manifest, IReadOnlyList LoadFiles(string sourceDirectory) .ToList(); } +static List ComputeChangedFiles( + List prevFiles, + List currentFiles) +{ + var prevMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var f in prevFiles) + { + prevMap[f.RelativePath] = ComputeSha512(f.Content); + } + + var changed = new List(); + foreach (var f in currentFiles) + { + var currentHash = ComputeSha512(f.Content); + if (!prevMap.TryGetValue(f.RelativePath, out var prevHash) || prevHash != currentHash) + { + changed.Add(f); + } + } + + return changed; +} + static string ComputeSha512(byte[] data) { var hash = SHA512.HashData(data); @@ -140,13 +354,18 @@ static string ComputeSha512(byte[] data) static string SanitizeVersion(string version) { var builder = new StringBuilder(version.Length); - foreach (var ch in version) { + foreach (var ch in version) + { builder.Append(char.IsLetterOrDigit(ch) ? ch : '-'); } return builder.ToString(); } +// ================================================================ +// Types +// ================================================================ + internal sealed class BuilderArguments { public string? SourceDirectory { get; private init; } @@ -154,6 +373,11 @@ internal sealed class BuilderArguments public string? TargetVersion { get; private init; } public string? MinSourceVersion { get; private init; } public string? MaxSourceVersion { get; private init; } + public string? PrevSourceDir { get; private init; } + public string? PrevVersion { get; private init; } + public string? ExistingCatalog { get; private init; } + public string? AnchorVersion { get; private init; } + public string? AnchorSourceDir { get; private init; } public bool IsValid => !string.IsNullOrWhiteSpace(SourceDirectory) @@ -164,8 +388,10 @@ internal sealed class BuilderArguments public static BuilderArguments Parse(string[] args) { var values = new Dictionary(StringComparer.OrdinalIgnoreCase); - for (var index = 0; index < args.Length; index += 2) { - if (!args[index].StartsWith("--", StringComparison.Ordinal) || index + 1 >= args.Length) { + for (var index = 0; index < args.Length; index += 2) + { + if (!args[index].StartsWith("--", StringComparison.Ordinal) || index + 1 >= args.Length) + { return new BuilderArguments(); } @@ -177,61 +403,41 @@ public static BuilderArguments Parse(string[] args) values.TryGetValue("--target-version", out var targetVersion); values.TryGetValue("--min-source-version", out var minSourceVersion); values.TryGetValue("--max-source-version", out var maxSourceVersion); - - return new BuilderArguments { + values.TryGetValue("--prev-source-dir", out var prevSourceDir); + values.TryGetValue("--prev-version", out var prevVersion); + values.TryGetValue("--existing-catalog", out var existingCatalog); + values.TryGetValue("--anchor-version", out var anchorVersion); + values.TryGetValue("--anchor-source-dir", out var anchorSourceDir); + + return new BuilderArguments + { SourceDirectory = sourceDirectory, OutputDirectory = outputDirectory, TargetVersion = targetVersion, MinSourceVersion = minSourceVersion, - MaxSourceVersion = maxSourceVersion + MaxSourceVersion = maxSourceVersion, + PrevSourceDir = prevSourceDir, + PrevVersion = prevVersion, + ExistingCatalog = existingCatalog, + AnchorVersion = anchorVersion, + AnchorSourceDir = anchorSourceDir }; } public static void PrintUsage() { Console.WriteLine("Usage:"); - Console.WriteLine(" TelegramSearchBot.UpdateBuilder --source-dir --output-dir --target-version --min-source-version [--max-source-version ]"); + Console.WriteLine( + " TelegramSearchBot.UpdateBuilder --source-dir --output-dir --target-version --min-source-version [--prev-source-dir --prev-version ] [--existing-catalog ] [--anchor-version --anchor-source-dir ]"); } } internal sealed record UpdateFileContent(string RelativePath, byte[] Content); -internal sealed class UpdateFile -{ - public required string RelativePath { get; init; } - public required string NewChecksum { get; init; } -} - -internal sealed class UpdateManifest -{ - public required string TargetVersion { get; init; } - public required string MinSourceVersion { get; init; } - public string? MaxSourceVersion { get; init; } - public bool IsAnchor { get; init; } - public bool IsCumulative { get; init; } - public int ChainDepth { get; init; } - public required List Files { get; init; } - public required string Checksum { get; init; } - public DateTime CreatedAt { get; init; } -} - -internal sealed class UpdateCatalog +internal static partial class Program { - public required string LatestVersion { get; init; } - public required List Entries { get; init; } - public DateTime LastUpdated { get; init; } - public string? MinRequiredVersion { get; init; } -} - -internal sealed class UpdateCatalogEntry -{ - public required string PackagePath { get; init; } - public required string TargetVersion { get; init; } - public required string MinSourceVersion { get; init; } - public string? MaxSourceVersion { get; init; } - public bool IsCumulative { get; init; } - public required string PackageChecksum { get; init; } - public int FileCount { get; init; } - public long CompressedSize { get; init; } - public long UncompressedSize { get; init; } + private static readonly JsonSerializerOptions JsonOptions = new() + { + PropertyNameCaseInsensitive = true + }; } diff --git a/TelegramSearchBot.UpdateBuilder/TelegramSearchBot.UpdateBuilder.csproj b/TelegramSearchBot.UpdateBuilder/TelegramSearchBot.UpdateBuilder.csproj index 54e9538f..86fadb30 100644 --- a/TelegramSearchBot.UpdateBuilder/TelegramSearchBot.UpdateBuilder.csproj +++ b/TelegramSearchBot.UpdateBuilder/TelegramSearchBot.UpdateBuilder.csproj @@ -1,4 +1,4 @@ - + Exe @@ -11,4 +11,8 @@ + + + + diff --git a/TelegramSearchBot/Service/Update/SelfUpdateBootstrap.Windows.cs b/TelegramSearchBot/Service/Update/SelfUpdateBootstrap.Windows.cs index f2b1c42b..2d242df3 100644 --- a/TelegramSearchBot/Service/Update/SelfUpdateBootstrap.Windows.cs +++ b/TelegramSearchBot/Service/Update/SelfUpdateBootstrap.Windows.cs @@ -13,6 +13,7 @@ using System.Threading.Tasks; using Serilog; using TelegramSearchBot.Common; +using TelegramSearchBot.Common.Model.Update; using TelegramSearchBot.Helper; using ZstdSharp; @@ -69,19 +70,20 @@ private static partial async Task StartUpdateOnWindowsAsync var currentVersion = ResolveCurrentVersion(); using var httpClient = HttpClientHelper.CreateProxyHttpClient(); var result = await CheckForUpdatesAsync(httpClient, currentVersion, CancellationToken.None); - if (result.Status != UpdateCheckStatus.UpdateAvailable || result.UpdateEntry is null) { + if (result.Status != UpdateCheckStatus.UpdateAvailable || result.UpdatePath is not { Count: > 0 }) { return ToManagedUpdateResult(result, currentVersion); } - await PrepareUpdateAsync(httpClient, currentVersion, result.UpdateEntry, CancellationToken.None); + await PrepareUpdateChainAsync(httpClient, currentVersion, result.UpdatePath, CancellationToken.None); + var finalTarget = result.UpdatePath[^1].TargetVersion; return new ManagedUpdateResult { State = ManagedUpdateState.UpdateScheduled, CurrentVersion = currentVersion, LatestVersion = result.LatestVersion, - TargetVersion = result.UpdateEntry.TargetVersion, + TargetVersion = finalTarget, ManagedInstallExists = File.Exists(ManagedExecutablePath), RunningManagedInstall = PathsEqual(Environment.ProcessPath, ManagedExecutablePath), - Message = $"已计划更新到 {result.UpdateEntry.TargetVersion}。" + Message = $"已计划更新到 {finalTarget}（共 {result.UpdatePath.Count} 步）。" }; } @@ -109,14 +111,64 @@ private static async Task CheckForUpdatesAsync(HttpClient htt $"Version {currentVersion} is below the minimum required version {catalog.MinRequiredVersion}. Reinstallation required."); } - var updateEntry = catalog.Entries - .Where(entry => CanApplyEntry(current, entry)) - .OrderByDescending(entry => Version.Parse(entry.TargetVersion)) - .FirstOrDefault(); + var updatePath = PlanUpdatePath(catalog.Entries, current, latest); + if (updatePath is not { Count: > 0 }) { + return UpdateCheckResult.NoPathFound($"No update path found from {currentVersion} to {catalog.LatestVersion}."); + } - return updateEntry is null - ? UpdateCheckResult.NoPathFound($"No update path found from {currentVersion} to {catalog.LatestVersion}.") - : UpdateCheckResult.UpdateAvailable(catalog.LatestVersion, updateEntry); + return UpdateCheckResult.UpdateAvailable(catalog.LatestVersion, updatePath); + } + + private static List PlanUpdatePath( + List entries, + Version currentVersion, + Version targetVersion) + { + var remaining = new HashSet(entries); + var path = new List(); + var versionCursor = currentVersion; + + while (versionCursor < targetVersion) { + var candidates = remaining + .Where(e => CanApplyEntry(versionCursor, e)) + .ToList(); + + if (candidates.Count == 0) { + return []; + } + + UpdateCatalogEntry bestPick; + var directToTarget = candidates.FirstOrDefault(e => + Version.TryParse(e.TargetVersion, out var tv) && tv >= targetVersion); + + if (directToTarget is not null) { + bestPick = directToTarget; + } else { + var cumulative = candidates + .Where(e => e.IsCumulative) + .OrderByDescending(e => Version.TryParse(e.TargetVersion, out var v) ? v : new Version(0, 0)) + .FirstOrDefault(); + + bestPick = cumulative ?? candidates + .OrderByDescending(e => Version.TryParse(e.TargetVersion, out var v) ? v : new Version(0, 0)) + .First(); + } + + path.Add(bestPick); + remaining.Remove(bestPick); + + if (!Version.TryParse(bestPick.TargetVersion, out var newCursor)) { + return []; + } + + if (newCursor <= versionCursor) { + return []; + } + + versionCursor = newCursor; + } + + return path; } private static ManagedUpdateResult ToManagedUpdateResult(UpdateCheckResult result, string currentVersion) @@ -130,37 +182,45 @@ private static ManagedUpdateResult ToManagedUpdateResult(UpdateCheckResult resul }, CurrentVersion = currentVersion, LatestVersion = result.LatestVersion, - TargetVersion = result.UpdateEntry?.TargetVersion, + TargetVersion = result.UpdatePath is { Count: > 0 } ? result.UpdatePath[^1].TargetVersion : null, Message = result.Message, ManagedInstallExists = File.Exists(ManagedExecutablePath), RunningManagedInstall = PathsEqual(Environment.ProcessPath, ManagedExecutablePath) }; } - private static async Task PrepareUpdateAsync( + private static async Task PrepareUpdateChainAsync( HttpClient httpClient, string currentVersion, - UpdateCatalogEntry updateEntry, + List updatePath, CancellationToken cancellationToken) { + var finalTarget = updatePath[^1].TargetVersion; var updateRoot = Path.Combine( Env.WorkDir, "updates", - $"{SanitizeVersion(currentVersion)}-to-{SanitizeVersion(updateEntry.TargetVersion)}"); + $"{SanitizeVersion(currentVersion)}-to-{SanitizeVersion(finalTarget)}"); var stagingDir = Path.Combine(updateRoot, "staging"); var backupDir = Path.Combine(updateRoot, "backup"); ResetDirectory(stagingDir); ResetDirectory(backupDir); - await using var packageStream = await DownloadStreamAsync(httpClient, updateEntry.PackagePath, cancellationToken); - await using var packageBuffer = new MemoryStream(); - await packageStream.CopyToAsync(packageBuffer, cancellationToken); - packageBuffer.Position = 0; + foreach (var entry in updatePath) + { + Log.Information("下载更新包: {PackagePath} (from {MinVersion})", + entry.PackagePath, entry.MinSourceVersion); + + await using var packageStream = await DownloadStreamAsync( + httpClient, entry.PackagePath, cancellationToken); + await using var packageBuffer = new MemoryStream(); + await packageStream.CopyToAsync(packageBuffer, cancellationToken); + packageBuffer.Position = 0; - VerifyPackageChecksum(packageBuffer, updateEntry); - packageBuffer.Position = 0; - ExtractPackageToDirectory(packageBuffer, stagingDir); + VerifyPackageChecksum(packageBuffer, entry); + packageBuffer.Position = 0; + ExtractPackageToDirectory(packageBuffer, stagingDir); + } var updaterPath = await DownloadUpdaterAsync(httpClient, cancellationToken); SpawnUpdater(new UpdaterLaunchArgs { @@ -424,33 +484,11 @@ private sealed class UpdaterLaunchArgs public required string BackupDir { get; init; } } - private sealed class UpdateCatalog - { - public required string LatestVersion { get; init; } - public required List Entries { get; init; } - public DateTime LastUpdated { get; init; } - public string? MinRequiredVersion { get; init; } - public string? UpdaterChecksum { get; init; } - } - - private sealed class UpdateCatalogEntry - { - public required string PackagePath { get; init; } - public required string TargetVersion { get; init; } - public required string MinSourceVersion { get; init; } - public string? MaxSourceVersion { get; init; } - public bool IsCumulative { get; init; } - public required string PackageChecksum { get; init; } - public long CompressedSize { get; init; } - public long UncompressedSize { get; init; } - public int FileCount { get; init; } - } - private sealed class UpdateCheckResult { public required UpdateCheckStatus Status { get; init; } public string? LatestVersion { get; init; } - public UpdateCatalogEntry? UpdateEntry { get; init; } + public List? UpdatePath { get; init; } public string? Message { get; init; } public static UpdateCheckResult UpToDate(string latestVersion) => new() { @@ -458,10 +496,10 @@ private sealed class UpdateCheckResult LatestVersion = latestVersion }; - public static UpdateCheckResult UpdateAvailable(string latestVersion, UpdateCatalogEntry updateEntry) => new() { + public static UpdateCheckResult UpdateAvailable(string latestVersion, List updatePath) => new() { Status = UpdateCheckStatus.UpdateAvailable, LatestVersion = latestVersion, - UpdateEntry = updateEntry + UpdatePath = updatePath }; public static UpdateCheckResult UpdateUnavailable(string latestVersion, string message) => new() {