From 7a83e109342d3df90d1cccdd453e0bc8efa90bb2 Mon Sep 17 00:00:00 2001 From: jonas Date: Mon, 1 Dec 2025 21:57:10 +0100 Subject: [PATCH] Fix (de)serialization of an index containing unicode surrogate characters --- src/Infidex.Tests/PersistenceTests.cs | 100 +++++++++++++----- src/Infidex/Indexing/Fst/FstSerializer.cs | 48 ++++----- .../ShortQuery/PositionalPrefixIndex.cs | 86 +++++++-------- 3 files changed, 142 insertions(+), 92 deletions(-) diff --git a/src/Infidex.Tests/PersistenceTests.cs b/src/Infidex.Tests/PersistenceTests.cs index 32e4942..7f3daf2 100644 --- a/src/Infidex.Tests/PersistenceTests.cs +++ b/src/Infidex.Tests/PersistenceTests.cs @@ -1,10 +1,8 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Infidex.Core; +using CsvHelper; +using CsvHelper.Configuration; using Infidex.Api; -using System.IO; +using Infidex.Core; using System.Globalization; -using CsvHelper; -using CsvHelper.Configuration; namespace Infidex.Tests; @@ -25,15 +23,15 @@ public void SaveAndLoadIndex_PreservesData() new Document(2L, "jumps over the lazy dog") }; engine.IndexDocuments(documents); - + // Verify search before save var resultsBefore = engine.Search(new Query("fox", 10)); Assert.AreEqual(1, resultsBefore.Records.Length); Assert.AreEqual(1L, resultsBefore.Records[0].DocumentId); - + // 2. Save engine.Save(filePath); - + // 3. Load // Use default config for loading (must match what was used for indexing usually) var config = ConfigurationParameters.GetConfig(400); @@ -49,17 +47,17 @@ public void SaveAndLoadIndex_PreservesData() config.StopTermLimit, config.WordMatcherSetup ); - + // 4. Verify search after load var resultsAfter = loadedEngine.Search(new Query("fox", 10)); Assert.AreEqual(1, resultsAfter.Records.Length); Assert.AreEqual(1L, resultsAfter.Records[0].DocumentId); - + // Verify another term var resultsDog = loadedEngine.Search(new Query("dog", 10)); Assert.AreEqual(1, resultsDog.Records.Length); Assert.AreEqual(2L, resultsDog.Records[0].DocumentId); - + // Verify statistics var statsBefore = engine.GetStatistics(); var statsAfter = loadedEngine.GetStatistics(); @@ -72,7 +70,7 @@ public void SaveAndLoadIndex_PreservesData() File.Delete(filePath); } } - + [TestMethod] public void SaveAndLoad40kMovies_MeasureIndexSize() { @@ -82,32 +80,32 @@ public void SaveAndLoad40kMovies_MeasureIndexSize() // Load movies from CSV var movies = LoadMovies(); Console.WriteLine($"Loaded {movies.Count} movies from CSV"); - + // Create and index var engine = SearchEngine.CreateDefault(); var documents = movies.Select((m, i) => new Document((long)i, m.Title)).ToList(); - + Console.WriteLine($"Indexing {documents.Count} movie titles..."); engine.IndexDocuments(documents); - + var stats = engine.GetStatistics(); Console.WriteLine($"Index stats: {stats.DocumentCount} documents, {stats.VocabularySize} unique terms"); - + // Test search before save var testResults = engine.Search(new Query("redemption", 5)); Console.WriteLine($"Test search found {testResults.Records.Length} results"); - + // Save index Console.WriteLine("Saving index to disk..."); engine.Save(filePath); - + // Measure file size var fileInfo = new FileInfo(filePath); long fileSizeBytes = fileInfo.Length; double fileSizeKB = fileSizeBytes / 1024.0; double fileSizeMB = fileSizeKB / 1024.0; - + Console.WriteLine($"\n=== INDEX FILE SIZE METRICS ==="); Console.WriteLine($"Documents indexed: {documents.Count:N0}"); Console.WriteLine($"Unique terms: {stats.VocabularySize:N0}"); @@ -116,7 +114,7 @@ public void SaveAndLoad40kMovies_MeasureIndexSize() Console.WriteLine($"File size: {fileSizeMB:N2} MB"); Console.WriteLine($"Bytes per document: {fileSizeBytes / (double)documents.Count:N2}"); Console.WriteLine($"================================\n"); - + // Load back and verify Console.WriteLine("Loading index from disk..."); var config = ConfigurationParameters.GetConfig(400); @@ -132,17 +130,17 @@ public void SaveAndLoad40kMovies_MeasureIndexSize() config.StopTermLimit, config.WordMatcherSetup ); - + // Verify loaded index works var loadedStats = loadedEngine.GetStatistics(); Assert.AreEqual(stats.DocumentCount, loadedStats.DocumentCount); Assert.AreEqual(stats.VocabularySize, loadedStats.VocabularySize); - + // Verify search results match var loadedResults = loadedEngine.Search(new Query("redemption", 5)); Assert.AreEqual(testResults.Records.Length, loadedResults.Records.Length); Console.WriteLine($"Loaded index verified: search returned {loadedResults.Records.Length} results"); - + // Additional searches to verify quality var searchTerms = new[] { "batman", "matrix", "star wars", "love", "action" }; foreach (var term in searchTerms) @@ -157,7 +155,61 @@ public void SaveAndLoad40kMovies_MeasureIndexSize() File.Delete(filePath); } } - + + [TestMethod] + public void SaveAndLoadIndex_UnicodeSurrogateCharacters() + { + string filePath = "surrogates_index.bin"; + try + { + // 1. Create and index + var engine = SearchEngine.CreateDefault(); + var documents = new[] + { + new Document(1L, "\uD83D\uDD0D") + }; + engine.IndexDocuments(documents); + + var resultsBefore = engine.Search(new Query("\uD83D\uDD0D", 10)); + Assert.HasCount(1, resultsBefore.Records); + Assert.AreEqual(1L, resultsBefore.Records[0].DocumentId); + + // 2. Save + engine.Save(filePath); + + // 3. Load + var config = ConfigurationParameters.GetConfig(400); + var loadedEngine = SearchEngine.Load( + filePath, + config.IndexSizes, + config.StartPadSize, + config.StopPadSize, + true, + config.TextNormalizer, + config.TokenizerSetup, + null, + config.StopTermLimit, + config.WordMatcherSetup + ); + + // 4. Verify search after load + var resultsAfter = loadedEngine.Search(new Query("\uD83D\uDD0D", 10)); + Assert.HasCount(1, resultsAfter.Records); + Assert.AreEqual(1L, resultsAfter.Records[0].DocumentId); + + // Verify statistics + var statsBefore = engine.GetStatistics(); + var statsAfter = loadedEngine.GetStatistics(); + Assert.AreEqual(statsBefore.DocumentCount, statsAfter.DocumentCount); + Assert.AreEqual(statsBefore.VocabularySize, statsAfter.VocabularySize); + } + finally + { + if (File.Exists(filePath)) + File.Delete(filePath); + } + } + private static List LoadMovies() { var config = new CsvConfiguration(CultureInfo.InvariantCulture) diff --git a/src/Infidex/Indexing/Fst/FstSerializer.cs b/src/Infidex/Indexing/Fst/FstSerializer.cs index 5b96187..4d38c41 100644 --- a/src/Infidex/Indexing/Fst/FstSerializer.cs +++ b/src/Infidex/Indexing/Fst/FstSerializer.cs @@ -1,5 +1,3 @@ -using System.Runtime.InteropServices; - namespace Infidex.Indexing.Fst; /// @@ -10,7 +8,7 @@ internal static class FstSerializer { private const uint FST_MAGIC = 0x46535432; // "FST2" in little-endian private const ushort FST_VERSION = 1; - + /// /// Serializes an FST index to a binary writer. /// @@ -20,21 +18,21 @@ public static void Write(BinaryWriter writer, FstIndex index) writer.Write(FST_MAGIC); writer.Write(FST_VERSION); writer.Write(index.TermCount); - + (FstNode[] forwardNodes, FstArc[] forwardArcs, int forwardRoot) = index.GetForwardFst(); (FstNode[] reverseNodes, FstArc[] reverseArcs, int reverseRoot) = index.GetReverseFst(); - + // Write forward FST WriteNodeArray(writer, forwardNodes); WriteArcArray(writer, forwardArcs); writer.Write(forwardRoot); - + // Write reverse FST WriteNodeArray(writer, reverseNodes); WriteArcArray(writer, reverseArcs); writer.Write(reverseRoot); } - + /// /// Deserializes an FST index from a binary reader. /// @@ -44,33 +42,33 @@ public static FstIndex Read(BinaryReader reader) uint magic = reader.ReadUInt32(); if (magic != FST_MAGIC) throw new InvalidDataException($"Invalid FST magic number: 0x{magic:X8}"); - + ushort version = reader.ReadUInt16(); if (version != FST_VERSION) throw new InvalidDataException($"Unsupported FST version: {version}"); - + int termCount = reader.ReadInt32(); - + // Read forward FST FstNode[] forwardNodes = ReadNodeArray(reader); FstArc[] forwardArcs = ReadArcArray(reader); int forwardRoot = reader.ReadInt32(); - + // Read reverse FST FstNode[] reverseNodes = ReadNodeArray(reader); FstArc[] reverseArcs = ReadArcArray(reader); int reverseRoot = reader.ReadInt32(); - + return new FstIndex( forwardNodes, forwardArcs, forwardRoot, reverseNodes, reverseArcs, reverseRoot, termCount); } - + private static void WriteNodeArray(BinaryWriter writer, FstNode[] nodes) { writer.Write(nodes.Length); - + foreach (ref readonly FstNode node in nodes.AsSpan()) { writer.Write(node.ArcStartIndex); @@ -79,12 +77,12 @@ private static void WriteNodeArray(BinaryWriter writer, FstNode[] nodes) writer.Write(node.Output); } } - + private static FstNode[] ReadNodeArray(BinaryReader reader) { int length = reader.ReadInt32(); FstNode[] nodes = new FstNode[length]; - + for (int i = 0; i < length; i++) { nodes[i] = new FstNode @@ -95,41 +93,41 @@ private static FstNode[] ReadNodeArray(BinaryReader reader) Output = reader.ReadInt32() }; } - + return nodes; } - + private static void WriteArcArray(BinaryWriter writer, FstArc[] arcs) { writer.Write(arcs.Length); - + foreach (ref readonly FstArc arc in arcs.AsSpan()) { - writer.Write(arc.Label); + writer.Write((ushort)arc.Label); writer.Write(arc.TargetNodeIndex); writer.Write(arc.Output); writer.Write(arc.IsFinal); } } - + private static FstArc[] ReadArcArray(BinaryReader reader) { int length = reader.ReadInt32(); FstArc[] arcs = new FstArc[length]; - + for (int i = 0; i < length; i++) { arcs[i] = new FstArc { - Label = reader.ReadChar(), + Label = (char)reader.ReadUInt16(), TargetNodeIndex = reader.ReadInt32(), Output = reader.ReadInt32(), IsFinal = reader.ReadBoolean() }; } - + return arcs; } - + } diff --git a/src/Infidex/Indexing/ShortQuery/PositionalPrefixIndex.cs b/src/Infidex/Indexing/ShortQuery/PositionalPrefixIndex.cs index b23af05..e260faf 100644 --- a/src/Infidex/Indexing/ShortQuery/PositionalPrefixIndex.cs +++ b/src/Infidex/Indexing/ShortQuery/PositionalPrefixIndex.cs @@ -11,21 +11,21 @@ internal sealed class PositionalPrefixIndex { // Index for 1-char prefixes: direct array indexing by char value private readonly PrefixPostingList?[] _singleCharIndex; - + // Index for 2-char and 3-char prefixes: dictionary-based private readonly Dictionary _multiCharIndex; - + // Maximum prefix length to index private const int MAX_PREFIX_LENGTH = 3; - + // Configuration private readonly int _minPrefixLength; private readonly int _maxPrefixLength; private readonly char[] _delimiters; - + public int MinPrefixLength => _minPrefixLength; public int MaxPrefixLength => _maxPrefixLength; - + /// /// Creates a new positional prefix index. /// @@ -37,16 +37,16 @@ public PositionalPrefixIndex(int minPrefixLength = 1, int maxPrefixLength = 3, c _minPrefixLength = Math.Max(1, minPrefixLength); _maxPrefixLength = Math.Min(MAX_PREFIX_LENGTH, maxPrefixLength); _delimiters = delimiters ?? [' ']; - + // Direct array for single-char lookups (covers most common case) _singleCharIndex = new PrefixPostingList?[char.MaxValue + 1]; - + // Dictionary for multi-char prefixes _multiCharIndex = new Dictionary(StringComparer.Ordinal); } - + #region Indexing - + /// /// Indexes a document's text, extracting all short prefixes with positions. /// @@ -56,44 +56,44 @@ public void IndexDocument(string text, int documentId) { if (string.IsNullOrEmpty(text)) return; - + ReadOnlySpan textSpan = text.AsSpan(); HashSet delimiterSet = new HashSet(_delimiters); - + int tokenIndex = 0; int i = 0; - + // Skip leading delimiters while (i < textSpan.Length && delimiterSet.Contains(textSpan[i])) i++; - + while (i < textSpan.Length) { // Find end of current token int tokenStart = i; while (i < textSpan.Length && !delimiterSet.Contains(textSpan[i])) i++; - + int tokenLength = i - tokenStart; - + if (tokenLength > 0) { // Index prefixes of this token IndexTokenPrefixes(textSpan.Slice(tokenStart, tokenLength), documentId, (ushort)tokenIndex); tokenIndex++; } - + // Skip delimiters to next token while (i < textSpan.Length && delimiterSet.Contains(textSpan[i])) i++; } } - + [MethodImpl(MethodImplOptions.AggressiveInlining)] private void IndexTokenPrefixes(ReadOnlySpan token, int documentId, ushort tokenPosition) { int maxLen = Math.Min(token.Length, _maxPrefixLength); - + for (int prefixLen = _minPrefixLength; prefixLen <= maxLen; prefixLen++) { if (prefixLen == 1) @@ -117,7 +117,7 @@ private void IndexTokenPrefixes(ReadOnlySpan token, int documentId, ushort } } } - + /// /// Freezes the index after all documents have been added. /// Sorts all posting lists and compacts memory. @@ -135,7 +135,7 @@ public void Freeze() list.BuildDocSet(); } } - + // Finalize multi-char lists foreach (PrefixPostingList list in _multiCharIndex.Values) { @@ -144,7 +144,7 @@ public void Freeze() list.BuildDocSet(); } } - + /// /// Clears all indexed data. /// @@ -153,11 +153,11 @@ public void Clear() Array.Clear(_singleCharIndex); _multiCharIndex.Clear(); } - + #endregion - + #region Querying - + /// /// Gets the posting list for a prefix, or null if not found. /// Complexity: O(1) for single char, O(1) average for multi-char. @@ -167,15 +167,15 @@ public void Clear() { if (prefix.IsEmpty || prefix.Length > _maxPrefixLength) return null; - + if (prefix.Length == 1) return _singleCharIndex[prefix[0]]; - + // For multi-char, we need to create a string key // This is unavoidable with Dictionary return _multiCharIndex.GetValueOrDefault(prefix.ToString()); } - + /// /// Gets all unique document IDs matching the prefix. /// @@ -184,7 +184,7 @@ public void GetDocumentIds(ReadOnlySpan prefix, HashSet result) PrefixPostingList? list = GetPostingList(prefix); list?.GetUniqueDocumentIds(result); } - + /// /// Gets document IDs where the prefix appears at word start. /// @@ -193,7 +193,7 @@ public void GetWordStartDocumentIds(ReadOnlySpan prefix, HashSet resu PrefixPostingList? list = GetPostingList(prefix); list?.GetWordStartDocumentIds(result); } - + /// /// Checks if any document contains the prefix. /// @@ -203,7 +203,7 @@ public bool HasPrefix(ReadOnlySpan prefix) PrefixPostingList? list = GetPostingList(prefix); return list != null && list.Count > 0; } - + /// /// Returns the number of documents matching the prefix. /// @@ -211,12 +211,12 @@ public int CountDocuments(ReadOnlySpan prefix) { PrefixPostingList? list = GetPostingList(prefix); if (list == null) return 0; - + HashSet docs = new HashSet(); list.GetUniqueDocumentIds(docs); return docs.Count; } - + /// /// Enumerates all prefixes and their posting lists. /// Intended for building secondary structures like champion lists. @@ -241,29 +241,29 @@ public int CountDocuments(ReadOnlySpan prefix) yield return (prefix, list); } } - + #endregion - + #region Serialization - + public void Write(BinaryWriter writer) { // Write single-char index int singleCharCount = 0; for (int i = 0; i < _singleCharIndex.Length; i++) if (_singleCharIndex[i] != null) singleCharCount++; - + writer.Write(singleCharCount); for (int i = 0; i < _singleCharIndex.Length; i++) { PrefixPostingList? list = _singleCharIndex[i]; if (list != null) { - writer.Write((char)i); + writer.Write((ushort)i); list.Write(writer); } } - + // Write multi-char index writer.Write(_multiCharIndex.Count); foreach ((string prefix, PrefixPostingList list) in _multiCharIndex) @@ -272,19 +272,19 @@ public void Write(BinaryWriter writer) list.Write(writer); } } - + public void Read(BinaryReader reader) { Clear(); - + // Read single-char index int singleCharCount = reader.ReadInt32(); for (int i = 0; i < singleCharCount; i++) { - char c = reader.ReadChar(); + char c = (char)reader.ReadUInt16(); _singleCharIndex[c] = PrefixPostingList.Read(reader); } - + // Read multi-char index int multiCharCount = reader.ReadInt32(); for (int i = 0; i < multiCharCount; i++) @@ -293,6 +293,6 @@ public void Read(BinaryReader reader) _multiCharIndex[prefix] = PrefixPostingList.Read(reader); } } - + #endregion }