Refactor hash DB formats and protect against "full game backup" (#3868)

* Refactor GOG client to include build details, manifest handling, and hash indexing; enhance GOG depot and file stream operations. Add new GOG models, exceptions, and utilities for manifest and hash management.

* Add GOG language support, manifest models, and depot enhancements; implement default language filter, refine manifest-handling logic, and update hash generation processes.

* Integrate file hash queries for Steam and GOG; enhance `FileHashesService` with locator ID resolution, add database query dependencies, and implement SQL macros for depot and build handling.

* Enforce 2GB maximum backup size in `ALoadoutSynchronizer` and add validation for total archived file size.

* Fix steam indexing and db generation

* Add unimplemented `GetLocatorIdsForGame` method in `StubbedFileHasherService`

* Update GOG schema with additional attributes for depots and manifests; adjust statistics and fingerprint

* Adjust `TestsFor_0004_RemoveGameFiles` to handle database-specific conditions for `LocatorIds`.

Author: halgari

src/NexusMods.Abstractions.GOG/DTOs/DepotInfo.cs

@@ -39,6 +39,13 @@ public class DepotItem
     [JsonPropertyName("path")]
     public required RelativePath Path { get; init; }
 
+    
+    /// <summary>
+    /// The MD5 hash of the file.
+    /// </summary>
+    [JsonPropertyName("md5")]
+    public Md5Value? Md5 { get; init; }
+    
     /// <summary>
     /// The chunks in the file.
     /// </summary>

src/NexusMods.Abstractions.GOG/IClient.cs

@@ -36,14 +36,20 @@ public interface IClient
     /// </summary>
     public Task<Build[]> GetBuilds(ProductId productId, OS os, CancellationToken token);
 
+
+    /// <summary>
+    /// Gets the details for a build, can include multiple depots, some of which may come from other productIds
+    /// </summary>
+    public Task<BuildDetails> GetBuildDetails(Build build, CancellationToken token);
+
     /// <summary>
     /// Get the depot information for a build.
     /// </summary>
-    public Task<DepotInfo> GetDepot(Build build, CancellationToken token);
+    public Task<DepotInfo> GetDepot(BuildDetailsDepot depot, CancellationToken token);
 
     /// <summary>
     /// Given a depot, a build, and a path, return a stream to the file. This file is seekable, and will cache and
     /// stream in data as required from the CDN.
     /// </summary>
-    public Task<Stream> GetFileStream(Build build, DepotInfo depotInfo, RelativePath path, CancellationToken token);
+    public Task<Stream> GetFileStream(ProductId productId, DepotInfo depotInfo, RelativePath path, CancellationToken token);
 }

src/NexusMods.Abstractions.Games.FileHashes/IFileHashesService.cs

@@ -58,6 +58,8 @@ public interface IFileHashesService
     /// Suggest version data for a given game installation and files.
     /// </summary>
     public Optional<VersionData> SuggestVersionData(GameInstallation gameInstallation, IEnumerable<(GamePath Path, Hash Hash)> files);
+
+    LocatorId[] GetLocatorIdsForGame(GameInstallation loadoutInstallationInstance);
 }
 
 /// <summary>

src/NexusMods.Abstractions.Games.FileHashes/Models/GogBuild.cs

@@ -15,13 +15,18 @@ public partial class GogBuild : IModelDefinition
     /// <summary>
     /// The GOG build ID.
     /// </summary>
-    public static readonly BuildIdAttribute BuildId = new(Namespace, nameof(BuildId)) { IsIndexed = true };
+    public static readonly BuildIdAttribute BuildId = new(Namespace, nameof(BuildId)) { IsIndexed = true, IsOptional = true };
 
     /// <summary>
     /// The GOG product ID.
     /// </summary>
     public static readonly ProductIdAttribute ProductId = new(Namespace, nameof(ProductId)) { IsIndexed = true };
 
+    /// <summary>
+    /// The unique manifest ID for this build.
+    /// </summary>
+    public static readonly StringAttribute ManifestId = new(Namespace, nameof(ManifestId)) { IsIndexed = true };
+    
     /// <summary>
     /// The Operating System the build is for.
     /// </summary>
@@ -43,7 +48,12 @@ public partial class GogBuild : IModelDefinition
     public static readonly BooleanAttribute Public = new(Namespace, nameof(Public));
 
     /// <summary>
-    /// The files in the GOG build.
+    /// (Deprecated) The files in the GOG build, use Depots instead.
     /// </summary>
     public static readonly ReferencesAttribute<PathHashRelation> Files = new(Namespace, nameof(Files));
+    
+    /// <summary>
+    /// All the depots in this build
+    /// </summary>
+    public static readonly ReferencesAttribute<GogDepot> Depots = new(Namespace, nameof(Depots));
 }

src/NexusMods.Abstractions.Games.FileHashes/Models/GogDepot.cs

@@ -0,0 +1,27 @@
+using NexusMods.Abstractions.Games.FileHashes.Attributes.Gog;
+using NexusMods.Abstractions.GOG.Values;
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.Models;
+
+namespace NexusMods.Abstractions.Games.FileHashes.Models;
+
+public partial class GogDepot : IModelDefinition
+{
+    private const string Namespace = "NexusMods.Abstractions.Games.FileHashes.GogDepot";
+    
+    public static readonly ProductIdAttribute ProductId = new(Namespace, nameof(ProductId)) { IsIndexed = true };
+
+    public static readonly SizeAttribute Size = new(Namespace, nameof(Size));
+    
+    public static readonly SizeAttribute CompressedSize = new(Namespace, nameof(CompressedSize));
+    
+    /// <summary>
+    /// The manifest pointed to by this depot
+    /// </summary>
+    public static readonly ReferenceAttribute<GogManifest> Manifest = new(Namespace, nameof(Manifest));
+    
+    /// <summary>
+    /// The languages in this depot.
+    /// </summary>
+    public static readonly StringsAttribute Languages = new(Namespace, nameof(Languages)) { IsIndexed = false };
+}

src/NexusMods.Abstractions.Games.FileHashes/Models/GogManifest.cs

@@ -0,0 +1,19 @@
+using NexusMods.MnemonicDB.Abstractions.Attributes;
+using NexusMods.MnemonicDB.Abstractions.Models;
+
+namespace NexusMods.Abstractions.Games.FileHashes.Models;
+
+public partial class GogManifest : IModelDefinition
+{
+    private const string Namespace = "NexusMods.Abstractions.Games.FileHashes.GogManifest";
+    
+    /// <summary>
+    /// The (unique) primary key of the manifest.
+    /// </summary>
+    public static readonly StringAttribute ManifestId = new(Namespace, nameof(ManifestId)) { IsIndexed = true };
+    
+    /// <summary>
+    /// The files in the manifest
+    /// </summary>
+    public static readonly ReferencesAttribute<PathHashRelation> Files = new(Namespace, nameof(Files));
+}

src/NexusMods.Abstractions.Loadouts.Synchronizers/ALoadoutSynchronizer.cs

@@ -31,6 +31,7 @@
 using NexusMods.Sdk;
 using NexusMods.Sdk.FileStore;
 using NexusMods.Sdk.IO;
+using ReactiveUI;
 using OneOf;
 using Reloaded.Memory.Extensions;
 
@@ -45,6 +46,12 @@ namespace NexusMods.Abstractions.Loadouts.Synchronizers;
 [PublicAPI]
 public class ALoadoutSynchronizer : ILoadoutSynchronizer
 {
+    /// <summary>
+    /// We'll limit backups to 2GB, for now we should never see much more than this
+    /// of modified game files. s
+    /// </summary>
+    private static Size MaximumBackupSize => Size.GB * 2;
+    
     private readonly ScopedAsyncLock _lock = new();
     private readonly IFileStore _fileStore;
 
@@ -571,6 +578,10 @@ public void ProcessSyncTree(Dictionary<GamePath, SyncNode> tree)
 
         loadout = await ReprocessOverrides(loadout);
 
+        job?.SetStatus("Archive Cleanup");
+        await _garbageCollectorRunner.RunAsync();
+        
+
         return loadout;
     }
 
@@ -1314,6 +1325,10 @@ await Parallel.ForEachAsync(files, async (item, _) =>
             }
         );
 
+        var totalSize = archivedFiles.Sum(static x => x.Size);
+        if (totalSize > MaximumBackupSize)
+            throw new Exception($"Cannot backup files, total size is {totalSize}, which is larger than the maximum of {MaximumBackupSize}");
+        
         // PERFORMANCE: We deduplicate above with the HaveFile call.
         await _fileStore.BackupFiles(archivedFiles, deduplicate: false);
 

src/NexusMods.DataModel/Services.cs

@@ -78,7 +78,7 @@ public static IServiceCollection AddDataModel(this IServiceCollection coll)
         // Game Registry
         coll.AddSingleton<IGameRegistry, GameRegistry.GameRegistry>();
         coll.AddHostedService(s => (GameRegistry.GameRegistry)s.GetRequiredService<IGameRegistry>());
-        coll.AddAttributeCollection(typeof(GameInstallMetadata));
+        coll.AddGameInstallMetadataModel();
 
         // File Store
         coll.AddAllSingleton<IFileStore, NxFileStore>();

src/NexusMods.Games.FileHashes/FileHashesQueries.sql

@@ -0,0 +1,35 @@
+-- namespace: NexusMods.Games.FileHashes
+
+CREATE SCHEMA IF NOT EXISTS file_hashes; 
+
+-- Find all the gog builds that match the given game's files, and rank them by the number of files that match
+CREATE MACRO file_hashes.resolve_gog_build(GameMetadataId, DefaultLanguage := 'en-US') AS TABLE
+SELECT build.BuildId, ANY_VALUE(build.ProductId) AS BuildProductId, COUNT(*) matching_files, ANY_VALUE(build."version"), list_distinct(LIST(depot.ProductId)) ProductIds
+FROM MDB_DISKSTATEENTRY() entry
+         LEFT JOIN HASHES_HASHRELATION() hashrel on entry.Hash = hashRel.xxHash3
+         LEFT JOIN HASHES_PATHHASHRELATION() pathrel on pathrel.Path = entry.Path.Item3 AND pathrel.Hash = hashrel.Id
+         LEFT JOIN (SELECT Id, unnest(Files) FileId FROM HASHES_GOGMANIFEST()) manifest on pathRel.Id = manifest.FileId
+         LEFT JOIN HASHES_GOGDEPOT() depot on depot.Manifest = Manifest.Id
+         LEFT JOIN (SELECT Id, unnest(depots) depot, ProductId, buildId, "version" FROM HASHES_GOGBUILD()) build on depot.Id = build.Depot
+WHERE entry.Game = GameMetadataId
+AND DefaultLanguage in depot.Languages
+GROUP BY build.BuildId
+ORDER BY COUNT(*) DESC;
+
+-- Find all the steam manifests that match the given game's files, and rank them by the number of files that match
+CREATE MACRO file_hashes.resolve_steam_manifests(GameMetadataId) AS TABLE
+SELECT ANY_VALUE(steam.depotId) DepotId, COUNT(*) matching_count, ANY_VALUE(steam.AppId) AppId, ANY_VALUE(steam.ManifestId)
+FROM MDB_DISKSTATEENTRY() entry
+         LEFT JOIN HASHES_HASHRELATION() hashrel on entry.Hash = hashRel.xxHash3
+         LEFT JOIN HASHES_PATHHASHRELATION() pathrel on pathrel.Path = entry.Path.Item3 AND pathrel.Hash = hashrel.Id
+         LEFT JOIN (SELECT AppId, ManifestId, DepotId, unnest(Files) File FROM HASHES_STEAMMANIFEST()) steam on steam.File = pathrel.Id
+WHERE entry.Game = GameMetadataId
+GROUP BY steam.ManifestId
+ORDER BY COUNT(*) DESC;
+
+-- Find all the depots (LocatorIds) for a given game. This will be the most matching depot for every AppId found in a given game folder
+CREATE MACRO file_hashes.resolve_steam_depots(GameMetadataId) AS TABLE 
+SELECT arg_max(ManifestId, matching_count) DepotId 
+FROM file_hashes.resolve_steam_manifests(GameMetadataId) manifests
+GROUP BY manifests.AppId
+Having DepotId is not null;

src/NexusMods.Games.FileHashes/FileHashesService.cs

@@ -8,25 +8,29 @@
 using NexusMods.Abstractions.GameLocators;
 using NexusMods.Abstractions.Games.FileHashes;
 using NexusMods.Abstractions.Games.FileHashes.Models;
+using NexusMods.Abstractions.GOG.Values;
 using NexusMods.Abstractions.Jobs;
 using NexusMods.Abstractions.NexusWebApi.Types.V2;
 using NexusMods.Abstractions.Settings;
 using NexusMods.Abstractions.Steam.Values;
 using NexusMods.Games.FileHashes.DTOs;
 using NexusMods.Hashing.xxHash3;
-using NexusMods.MnemonicDB;
+using NexusMods.HyperDuck;
 using NexusMods.MnemonicDB.Abstractions;
 using NexusMods.MnemonicDB.Storage;
 using NexusMods.MnemonicDB.Storage.RocksDbBackend;
 using NexusMods.Paths;
 using NexusMods.Sdk;
 using NexusMods.Sdk.IO;
 using BuildId = NexusMods.Abstractions.GOG.Values.BuildId;
+using Connection = NexusMods.MnemonicDB.Connection;
 
 namespace NexusMods.Games.FileHashes;
 
 internal sealed class FileHashesService : IFileHashesService, IDisposable
 {
+    private const string DefaultLanguage = "en-US";
+    
     private readonly ScopedAsyncLock _lock = new();
     private readonly FileHashesServiceSettings _settings;
     private readonly IFileSystem _fileSystem;
@@ -42,6 +46,8 @@ internal sealed class FileHashesService : IFileHashesService, IDisposable
     private ConnectedDb? _currentDb;
 
     private readonly ILogger<FileHashesService> _logger;
+    private readonly IQueryEngine _queryEngine;
+    private IQueryMixin _queryMixin;
 
     private record ConnectedDb(IDb Db, DatomStore Store, Backend Backend, DatabaseInfo DatabaseInfo);
 
@@ -54,6 +60,8 @@ public FileHashesService(ILogger<FileHashesService> logger, ISettingsManager set
         _settings = settingsManager.Get<FileHashesServiceSettings>();
         _databases = new Dictionary<AbsolutePath, ConnectedDb>();
         _provider = provider;
+        _queryEngine = provider.GetRequiredService<IQueryEngine>();
+        _queryMixin = _queryEngine.DuckDb;
 
         _hashDatabaseLocation = _settings.HashDatabaseLocation.ToPath(_fileSystem);
         _hashDatabaseLocation.CreateDirectory();
@@ -76,7 +84,7 @@ private ConnectedDb OpenDb(DatabaseInfo databaseInfo)
             };
 
             var store = new DatomStore(_provider.GetRequiredService<ILogger<DatomStore>>(), settings, backend);
-            var connection = new Connection(_provider.GetRequiredService<ILogger<Connection>>(), store, _provider, [], readOnlyMode: true);
+            var connection = new Connection(_provider.GetRequiredService<ILogger<Connection>>(), store, _provider, [], readOnlyMode: true, prefix: "hashes", queryEngine: _queryEngine);
             var connectedDb = new ConnectedDb(connection.Db, store, backend, databaseInfo);
 
             _databases[databaseInfo.Path] = connectedDb;
@@ -333,17 +341,48 @@ public IEnumerable<GameFileRecord> GetGameFiles(LocatorIdsWithGameStore locatorI
 
         if (gameStore == GameStore.GOG)
         {
+            HashSet<GogBuild.ReadOnly> gogBuilds = [];
+            HashSet<ProductId> gogProducts = [];
+            Dictionary<EntityId, GogManifest.ReadOnly> gogManifests = [];
+            
+            // So first we find all the valid build Ids, and then assume that everything else is a product Id
             foreach (var id in locatorIds)
             {
                 if (!ulong.TryParse(id.Value, out var parsedId))
                     continue;
 
                 var gogId = BuildId.From(parsedId);
 
-                if (!GogBuild.FindByBuildId(Current, gogId).TryGetFirst(out var firstBuild))
+                if (GogBuild.FindByBuildId(Current, gogId).TryGetFirst(out var firstBuild))
+                {
+                    gogBuilds.Add(firstBuild);
                     continue;
+                }
+                
+                var productId = ProductId.From(parsedId);
+                gogProducts.Add(productId);
+            }
+            
+            // Now we emit all the files from the build products, and then also from any secondary products
+            foreach (var build in gogBuilds)
+            {
+                foreach (var depot in build.Depots)
+                {
+                    // We only care about the productId of the build, and the productIds of the secondary products
+                    if (!(depot.ProductId == build.ProductId || gogProducts.Contains(depot.ProductId)))
+                        continue;
+                    
+                    // If there is a language setting for the files, they have to be the same as the default language
+                    if (!(depot.Languages.Count == 0 || depot.Languages.Contains(DefaultLanguage)))
+                        continue;
+
+                    gogManifests[depot.Manifest.Id] = depot.Manifest;
+                }
+            }
 
-                foreach (var file in firstBuild.Files)
+            foreach (var (_ , manifest) in gogManifests)
+            {
+                foreach (var file in manifest.Files)
                 {
                     yield return new GameFileRecord
                     {
@@ -354,6 +393,7 @@ public IEnumerable<GameFileRecord> GetGameFiles(LocatorIdsWithGameStore locatorI
                     };
                 }
             }
+            
         }
         else if (gameStore == GameStore.Steam)
         {
@@ -560,7 +600,7 @@ public LocatorId[] GetLocatorIdsForVersionDefinition(GameStore gameStore, Versio
     {
         if (gameStore == GameStore.GOG)
         {
-            return versionDefinition.GogBuilds.Select(build => LocatorId.From(build.BuildId.ToString())).ToArray();
+            return versionDefinition.GogBuilds.Select(build => LocatorId.From(build.BuildId!.Value.ToString())).ToArray();
         }
 
         if (gameStore == GameStore.Steam)
@@ -598,6 +638,42 @@ public Optional<VersionData> SuggestVersionData(GameInstallation gameInstallatio
             .FirstOrOptional(_ => true);
     }
 
+    public LocatorId[] GetLocatorIdsForGame(GameInstallation gameInstallation)
+    {
+        if (gameInstallation.Store == GameStore.Steam)
+        {
+            var ids = _queryMixin.Query<DepotId>("SELECT * FROM file_hashes.resolve_steam_depots({gameInstallation.GameMetadataId});")
+                .Select(id => LocatorId.From(id.Value.ToString()))
+                .ToArray();
+            return ids;
+        }
+        else if (gameInstallation.Store == GameStore.GOG)
+        {
+            if (!_queryMixin.Query<(BuildId, ProductId, List<ProductId>)>("SELECT BuildId, BuildProductId, ProductIds FROM file_hashes.resolve_gog_build({gameInstallation.GameMetadataId})")
+                .TryGetFirst(out var found))
+                return [];
+            
+            var ids = new List<LocatorId>();
+            
+            ids.Add(LocatorId.From(found.Item1.Value.ToString()));
+            
+            // We want to add the Build Id and then all the product Ids that are not the same as the Build's product
+            foreach (var productId in found.Item3)
+            {
+                if (productId == found.Item2)
+                    continue;
+                
+                ids.Add(LocatorId.From(productId.Value.ToString()));
+            }
+            
+            return ids.ToArray();
+        }
+        else
+        {
+            throw new NotSupportedException("No way to get locator IDs for: " + gameInstallation.Store);
+        }
+    }
+
     /// <inheritdoc/>
     public void Dispose()
     {