Add SIMD window to key digest search

sandbox/DryDB.Benchmark/ReadBenchmark.cs

@@ -57,6 +57,25 @@ public void DryDB_FindByKey_RandomKeys()
         }
     }
 
+    uint noRepeatSeed = 123456789u;
+
+    // The variant above re-seeds every op, so the same 1000-key sequence repeats and a
+    // large branch predictor gradually memorizes its branch history across invocations.
+    // Carrying the seed across ops makes the sequence genuinely non-repeating — the
+    // closest model of real random access.
+    [Benchmark]
+    public void DryDB_FindByKey_RandomKeys_NoRepeat()
+    {
+        var seed = noRepeatSeed;
+        for (var i = 0; i < Iterations; i++)
+        {
+            seed = seed * 1664525u + 1013904223u;
+            var table = database.GetTable("items");
+            using var _ = table.Get((long)(seed % N));
+        }
+        noRepeatSeed = seed;
+    }
+
     const int ThreadCount = 8;
 
     [Benchmark]

src/DryDB/BTree/DigestSearch.cs

@@ -0,0 +1,109 @@
+using System.Runtime.CompilerServices;
+#if NET8_0_OR_GREATER
+using System.Runtime.Intrinsics;
+#endif
+
+namespace DryDB.BTree;
+
+/// <summary>
+/// Lower-bound search over the contiguous key digest array of a node.
+/// </summary>
+/// <remarks>
+/// The tail levels of a binary search carry the unpredictable branches (a ~50%
+/// mispredict per level on non-repeating random keys). This kernel runs the branchy
+/// binary search only down to a 32-element window and finishes with a branch-free SIMD
+/// count, which removes those mispredicts. A full SIMD scan would touch every cache
+/// line of the digest array and loses when the node is out of cache; the hybrid wins
+/// in both regimes on unpredictable keys.
+/// </remarks>
+static class DigestSearch
+{
+    /// <summary>
+    /// Whether the SIMD window path is available. The lower-bound restructure only
+    /// pays off together with the SIMD window (losing the classic search's early
+    /// digest-equality exit costs more than the restructure alone gains), so callers
+    /// must keep using the classic mixed digest binary search when this is false —
+    /// notably on netstandard targets, which have no <c>Vector128</c>.
+    /// </summary>
+#if NET8_0_OR_GREATER
+    public static bool IsAccelerated
+    {
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        get => Vector128.IsHardwareAccelerated;
+    }
+#else
+    public const bool IsAccelerated = false;
+#endif
+
+    /// <summary>
+    /// Returns the number of digests strictly less than <paramref name="keyDigest"/>,
+    /// i.e. the index of the first digest &gt;= <paramref name="keyDigest"/> (or
+    /// <paramref name="count"/> if there is none).
+    /// </summary>
+    [MethodImpl(MethodImplOptions.AggressiveInlining)]
+    public static int LowerBound(ref byte pageReference, int digestBase, int count, ulong keyDigest)
+    {
+        var min = 0;
+        var max = count;
+        while (max - min > 32)
+        {
+            var mid = min + ((max - min) >> 1);
+            var digest = Unsafe.ReadUnaligned<ulong>(
+                ref Unsafe.Add(ref pageReference, digestBase + mid * sizeof(ulong)));
+            if (digest < keyDigest)
+            {
+                min = mid + 1;
+            }
+            else
+            {
+                max = mid;
+            }
+        }
+
+#if NET8_0_OR_GREATER
+        if (Vector128.IsHardwareAccelerated && max >= 32)
+        {
+            // Anchor the 32-wide window at [max-32, max): it stays inside the array,
+            // and every element below `min` it may cover is < keyDigest by the binary
+            // search invariant, so counting them keeps the result exact — no sentinels
+            // or masking needed.
+            var start = max - 32;
+            var keyVec = Vector128.Create(keyDigest);
+            // Four independent accumulators: a single accumulator would serialize the
+            // sixteen subtracts into one long dependency chain, whose latency exceeds
+            // the mispredict cost it replaces.
+            var acc0 = Vector128<ulong>.Zero;
+            var acc1 = Vector128<ulong>.Zero;
+            var acc2 = Vector128<ulong>.Zero;
+            var acc3 = Vector128<ulong>.Zero;
+            ref var window = ref Unsafe.Add(ref pageReference, digestBase + start * sizeof(ulong));
+            for (var i = 0; i < 32 * sizeof(ulong); i += 8 * sizeof(ulong))
+            {
+                // a true lane is all-ones (= -1): accumulate counts by subtraction
+                acc0 -= Vector128.LessThan(Unsafe.ReadUnaligned<Vector128<ulong>>(ref Unsafe.Add(ref window, i)), keyVec);
+                acc1 -= Vector128.LessThan(Unsafe.ReadUnaligned<Vector128<ulong>>(ref Unsafe.Add(ref window, i + 2 * sizeof(ulong))), keyVec);
+                acc2 -= Vector128.LessThan(Unsafe.ReadUnaligned<Vector128<ulong>>(ref Unsafe.Add(ref window, i + 4 * sizeof(ulong))), keyVec);
+                acc3 -= Vector128.LessThan(Unsafe.ReadUnaligned<Vector128<ulong>>(ref Unsafe.Add(ref window, i + 6 * sizeof(ulong))), keyVec);
+            }
+            var acc = (acc0 + acc1) + (acc2 + acc3);
+            return start + (int)(acc.GetElement(0) + acc.GetElement(1));
+        }
+#endif
+
+        while (min < max)
+        {
+            var mid = min + ((max - min) >> 1);
+            var digest = Unsafe.ReadUnaligned<ulong>(
+                ref Unsafe.Add(ref pageReference, digestBase + mid * sizeof(ulong)));
+            if (digest < keyDigest)
+            {
+                min = mid + 1;
+            }
+            else
+            {
+                max = mid;
+            }
+        }
+        return min;
+    }
+}

src/DryDB/BTree/InternalNodeReader.cs

@@ -61,38 +61,55 @@ public bool TrySearch<TComparer>(
         ref var pageReference = ref MemoryMarshal.GetReference(page);
 #endif
         var useDigest = hasKeyDigests && hasKeyDigest;
-
-        var min = 0;
-        var max = entryCount;
-
+        int min;
         NodeEntryMeta meta;
-        while (min < max)
+        if (useDigest && DigestSearch.IsAccelerated)
         {
-            var mid = min + ((max - min) >> 1);
-
-            int cmp;
-            if (useDigest)
+            // Branch-free lower bound over the digest array, then advance through the
+            // run of equal digests with full comparisons to reach the upper bound
+            // (first entry > key). Entries with a greater digest are already > key.
+            min = DigestSearch.LowerBound(ref pageReference, DigestBase, entryCount, keyDigest);
+            while (min < entryCount)
             {
-                // One contiguous load instead of dereferencing the variable-length key;
-                // only digest ties fall back to the full comparison.
                 var digest = Unsafe.ReadUnaligned<ulong>(
-                    ref Unsafe.Add(ref pageReference, DigestBase + mid * sizeof(ulong)));
-                cmp = digest != keyDigest
-                    ? (digest < keyDigest ? -1 : 1)
-                    : CompareFull(ref pageReference, mid, key, comparer);
-            }
-            else
-            {
-                cmp = CompareFull(ref pageReference, mid, key, comparer);
+                    ref Unsafe.Add(ref pageReference, DigestBase + min * sizeof(ulong)));
+                if (digest != keyDigest) break;
+                if (CompareFull(ref pageReference, min, key, comparer) > 0) break;
+                min++;
             }
-
-            if (cmp <= 0) // upper bounds
-            {
-                min = mid + 1;
-            }
-            else
+        }
+        else
+        {
+            min = 0;
+            var max = entryCount;
+            while (min < max)
             {
-                max = mid;
+                var mid = min + ((max - min) >> 1);
+
+                int cmp;
+                if (useDigest)
+                {
+                    // One contiguous load instead of dereferencing the variable-length
+                    // key; only digest ties fall back to the full comparison.
+                    var digest = Unsafe.ReadUnaligned<ulong>(
+                        ref Unsafe.Add(ref pageReference, DigestBase + mid * sizeof(ulong)));
+                    cmp = digest != keyDigest
+                        ? (digest < keyDigest ? -1 : 1)
+                        : CompareFull(ref pageReference, mid, key, comparer);
+                }
+                else
+                {
+                    cmp = CompareFull(ref pageReference, mid, key, comparer);
+                }
+
+                if (cmp <= 0) // upper bounds
+                {
+                    min = mid + 1;
+                }
+                else
+                {
+                    max = mid;
+                }
             }
         }
 

src/DryDB/BTree/LeafNodeReader.cs

@@ -83,6 +83,36 @@ public bool TryFindValue<TComparer>(
         ref var pageReference = ref MemoryMarshal.GetReference(page);
 #endif
         var useDigest = hasKeyDigests && hasKeyDigest;
+        if (useDigest && DigestSearch.IsAccelerated)
+        {
+            // Branch-free lower bound over the digest array; a match can only sit in
+            // the run of digests equal to keyDigest, which starts at the bound.
+            // (Order preservation: digest < keyDigest implies entry < key, and
+            // digest > keyDigest implies entry > key.)
+            var i = DigestSearch.LowerBound(ref pageReference, DigestBase, entryCount, keyDigest);
+            for (; i < entryCount; i++)
+            {
+                var digest = Unsafe.ReadUnaligned<ulong>(
+                    ref Unsafe.Add(ref pageReference, DigestBase + i * sizeof(ulong)));
+                if (digest != keyDigest) break;
+
+                var compared = CompareFull(ref pageReference, i, key, comparer);
+                if (compared == 0)
+                {
+                    var meta = GetMeta(i);
+                    index = i;
+                    valueOffset = meta.PageOffset + meta.KeyLength;
+                    valueLength = meta.ValueLength;
+                    return true;
+                }
+                if (compared > 0) break;
+            }
+
+            index = default;
+            valueOffset = default;
+            valueLength = default;
+            return false;
+        }
 
         var min = 0;
         var max = entryCount;
@@ -144,6 +174,70 @@ public bool TrySearch<TComparer>(
         ref var pageReference = ref MemoryMarshal.GetReference(page);
 #endif
         var useDigest = hasKeyDigests && hasKeyDigest;
+        if (useDigest && DigestSearch.IsAccelerated)
+        {
+            // Branch-free lower bound over the digest array, then resolve the bound
+            // inside the run of equal digests with full comparisons (run length is
+            // almost always 0 or 1; exact digests such as Int64 never exceed 1).
+            // Entries before the bound are < key; the first entry with a greater
+            // digest is > key, which already satisfies both bound operators.
+            var i = DigestSearch.LowerBound(ref pageReference, DigestBase, entryCount, keyDigest);
+            switch (op)
+            {
+                case SearchOperator.Equal:
+                    for (; i < entryCount; i++)
+                    {
+                        var digest = Unsafe.ReadUnaligned<ulong>(
+                            ref Unsafe.Add(ref pageReference, DigestBase + i * sizeof(ulong)));
+                        if (digest != keyDigest) break;
+
+                        var compared = CompareFull(ref pageReference, i, key, comparer);
+                        if (compared == 0)
+                        {
+                            index = i;
+                            return true;
+                        }
+                        if (compared > 0) break;
+                    }
+                    index = default;
+                    return false;
+
+                case SearchOperator.LowerBound:
+                    // first entry >= key
+                    while (i < entryCount)
+                    {
+                        var digest = Unsafe.ReadUnaligned<ulong>(
+                            ref Unsafe.Add(ref pageReference, DigestBase + i * sizeof(ulong)));
+                        if (digest != keyDigest) break;
+                        if (CompareFull(ref pageReference, i, key, comparer) >= 0) break;
+                        i++;
+                    }
+                    break;
+
+                case SearchOperator.UpperBound:
+                    // first entry > key
+                    while (i < entryCount)
+                    {
+                        var digest = Unsafe.ReadUnaligned<ulong>(
+                            ref Unsafe.Add(ref pageReference, DigestBase + i * sizeof(ulong)));
+                        if (digest != keyDigest) break;
+                        if (CompareFull(ref pageReference, i, key, comparer) > 0) break;
+                        i++;
+                    }
+                    break;
+
+                default:
+                    throw new ArgumentOutOfRangeException(nameof(op), op, null);
+            }
+
+            if (i >= entryCount)
+            {
+                index = default;
+                return false;
+            }
+            index = i;
+            return true;
+        }
 
         var min = 0;
         var max = entryCount;