bench(pipeline): add sebpop/upstream as a second reference series

.github/workflows/pipeline-bench.yml

@@ -1,14 +1,15 @@
 name: Pipeline Benchmark
 
-# Comparative benchmark: the experimental `PipelineTokenizer` vs the latest
-# *released* tokenizers crate (the baseline to beat — the in-tree legacy
-# `Tokenizer` is being phased out, so it is only the id-correctness oracle,
-# never a benched series), for every model in
+# Comparative benchmark: the experimental `PipelineTokenizer` vs two references
+# — the latest *released* tokenizers crate (the baseline to beat — the in-tree
+# legacy `Tokenizer` is being phased out, so it is only the id-correctness
+# oracle, never a benched series) and sebpop's performance branch
+# (github.com/sebpop/tokenizers#upstream) — for every model in
 # tk-encode/examples/bench_models.json across every corpus in data/fixtures/.
 # Measures single- and multi-thread throughput (1/2/4/8/device-max), per-
-# implementation memory footprint (RSS), and stripped minimal-binary size. The
-# release dep is behind tk-encode's `bench-baseline` feature, so production
-# builds never pull it.
+# implementation memory footprint (RSS), and stripped minimal-binary size. Both
+# reference deps are behind tk-encode's `bench-baseline` feature, so production
+# builds never pull them.
 #
 # The work is fanned out across a matrix: each `bench` shard benches a slice of
 # the models on its own isolated 8-vCPU runner (so the multi-thread sweep is
@@ -113,9 +114,14 @@ jobs:
           HF_TOKEN: ${{ secrets.HF_TOKEN }}
         run: make fixtures bench-models HF="uvx --from huggingface_hub hf"
 
+      # bench-mimalloc: the whole bench binary (all three series uniformly) runs on
+      # mimalloc — the allocator sebpop's branch ships by default. The references are
+      # benched through `encode_fast` (see fixture_bench.rs), so sebpop's fused
+      # fast path is actually engaged.
       - name: Run comparative benchmark (shard ${{ matrix.shard }} of ${{ env.SHARDS }})
         run: |
-          cargo run --release -p tk-encode --features tk-encode/bench-baseline \
+          cargo run --release -p tk-encode \
+            --features tk-encode/bench-baseline,tk-encode/bench-mimalloc \
             --example fixture_bench -- --shard ${{ matrix.shard }} ${{ env.SHARDS }} \
             > "pipeline_bench_${{ matrix.shard }}.json"
           cat "pipeline_bench_${{ matrix.shard }}.json"
@@ -180,7 +186,9 @@ jobs:
           for f in parts:
               d = json.load(open(f))
               if merged is None:
-                  merged = {"baseline": d["baseline"], "models": []}
+                  # carry every metadata key (baseline, sebpop, …) from the first shard
+                  merged = {k: v for k, v in d.items() if k != "models"}
+                  merged["models"] = []
               merged["models"].extend(d["models"])
           if merged is None:
               raise SystemExit("no shard partials found")
@@ -194,10 +202,10 @@ jobs:
       - name: Measure binary sizes
         run: |
           cargo build --release -p tk-encode --features tk-encode/bench-baseline \
-            --example binsize_baseline --example binsize_pipeline
+            --example binsize_baseline --example binsize_sebpop --example binsize_pipeline
           printf '{' > binary_sizes.json
           sep=''
-          for key in baseline pipeline; do
+          for key in baseline sebpop pipeline; do
             bin="target/release/examples/binsize_$key"
             "$bin" data/gpt2.json "The quick brown fox jumps 123."   # binary actually works
             strip -o "$bin.stripped" "$bin"
@@ -269,7 +277,7 @@ jobs:
           fi
           python3 ${{ github.workspace }}/.github/scripts/render_pipeline_bench.py \
             pipeline_bench.json \
-            --subtitle "~10 kB inputs · single thread + 1/2/4/8/max-thread sweep" \
+            --subtitle "~10 kB inputs · refs via encode_fast · mimalloc (all series) · single thread + 1/2/4/8/max-thread sweep" \
             --revision "${{ github.event.pull_request.head.sha || github.sha }}" \
             --img-base "$base_url" \
             --run-id "${{ github.run_id }}" \

tokenizers/tk-encode/Cargo.toml

@@ -63,13 +63,24 @@ yada = "0.7.0"
 # Latest released tokenizers, used as the comparison baseline by the CI benchmark
 # (examples gated on `bench-baseline`). Optional so production builds never pull it.
 tokenizers-release = { package = "tokenizers", version = "=0.23.1", optional = true }
+# sebpop's performance branch, the second comparison reference. Cargo.lock pins the
+# resolved commit — `cargo update tokenizers@0.22.3-dev.0` moves it to the branch tip.
+# His default features minus `mimalloc`: its `override` feature replaces the global
+# allocator for the whole process, which would skew every series benched in the same
+# binary.
+tokenizers-sebpop = { package = "tokenizers", git = "https://github.com/sebpop/tokenizers", branch = "upstream", default-features = false, features = ["progressbar", "pcre2", "esaxx_fast"], optional = true }
 # Reference regex engines `fixture_bench` times the classify+fsm pre-tokenize against: Oniguruma and
 # PCRE2 (both C) alongside the pure-Rust `fancy-regex` (optional, above). All three are pulled in by
 # `bench-baseline` — optional + behind that feature so the C builds happen ONLY for the CI benchmark,
 # never for `cargo test`/production.
 onig = { version = "6.5.1", optional = true }
 pcre2 = { version = "0.2", optional = true }
 logos = { version = "0.15", optional = true } # compile-time DFA lexer reference (pure Rust)
+# Global allocator for the CI bench binary (`bench-mimalloc`): every series runs on
+# mimalloc — the allocator sebpop's branch ships by default — because one process can't
+# give each series its own allocator. No `override` feature: `#[global_allocator]` in
+# fixture_bench.rs is explicit and scoped to that binary.
+mimalloc = { version = "0.1", optional = true }
 
 [features]
 # `fancy-regex` is the OPTIONAL system-regex backend, needed ONLY for a `Split` pre-tokenizer with an
@@ -81,7 +92,8 @@ progressbar = ["indicatif"]
 http = ["hf-hub"]
 unstable_wasm = ["fancy-regex", "getrandom/wasm_js"]
 rustls-tls = ["hf-hub?/rustls-tls"]
-bench-baseline = ["dep:tokenizers-release", "dep:onig", "dep:pcre2", "dep:logos", "fancy-regex"]
+bench-baseline = ["dep:tokenizers-release", "dep:tokenizers-sebpop", "dep:onig", "dep:pcre2", "dep:logos", "fancy-regex"]
+bench-mimalloc = ["dep:mimalloc"]
 
 [dev-dependencies]
 criterion = "0.6"
@@ -101,3 +113,7 @@ required-features = ["bench-baseline"]
 [[example]]
 name = "binsize_baseline"
 required-features = ["bench-baseline"]
+
+[[example]]
+name = "binsize_sebpop"
+required-features = ["bench-baseline"]

tokenizers/tk-encode/examples/binsize_sebpop.rs

@@ -0,0 +1,17 @@
+//! Minimal encode program measured by CI for binary size: the `tokenizers`
+//! crate from sebpop's performance branch (the second comparison reference).
+//! Structurally identical to `binsize_pipeline.rs`.
+
+use tokenizers_sebpop::Tokenizer;
+
+fn main() {
+    let mut args = std::env::args().skip(1);
+    let path = args
+        .next()
+        .expect("usage: binsize_sebpop <tokenizer.json> <text>");
+    let text = args
+        .next()
+        .expect("usage: binsize_sebpop <tokenizer.json> <text>");
+    let tok = Tokenizer::from_file(path).unwrap();
+    println!("{}", tok.encode(text.as_str(), false).unwrap().len());
+}