implement ultra-basic, erroneous decode

bindings/node/src/decoders.rs

@@ -1,4 +1,5 @@
 use crate::arc_rwlock_serde;
+use ahash::AHashMap;
 use serde::{Deserialize, Serialize};
 extern crate tokenizers as tk;
 use napi::bindgen_prelude::*;
@@ -58,7 +59,7 @@ pub fn bpe_decoder(suffix: Option<String>) -> Decoder {
 pub fn byte_fallback_decoder() -> Decoder {
   Decoder {
     decoder: Some(Arc::new(RwLock::new(
-      tk::decoders::byte_fallback::ByteFallback::new().into(),
+      tk::decoders::byte_fallback::ByteFallback::new(AHashMap::new()).into(),
     ))),
   }
 }

bindings/python/src/decoders.rs

@@ -3,6 +3,7 @@ use std::sync::{Arc, RwLock};
 use crate::pre_tokenizers::from_string;
 use crate::tokenizer::PyTokenizer;
 use crate::utils::PyPattern;
+use ahash::AHashMap;
 use pyo3::exceptions;
 use pyo3::prelude::*;
 use pyo3::types::*;
@@ -299,7 +300,7 @@ impl PyByteFallbackDec {
     #[new]
     #[pyo3(signature = (), text_signature = "(self)")]
     fn new() -> PyClassInitializer<Self> {
-        PyClassInitializer::<PyDecoder>::from(PyDecoder::from(ByteFallback::new()))
+        PyClassInitializer::<PyDecoder>::from(PyDecoder::from(ByteFallback::new(AHashMap::new())))
             .add_subclass(PyByteFallbackDec {})
     }
 }

tokenizers/benches/ci_benchmark.rs

@@ -267,7 +267,7 @@ fn bench_decode(c: &mut Criterion) {
     let mut sp_chain = Tokenizer::from_file("data/albert-base-v1-tokenizer.json").unwrap();
     sp_chain.with_decoder(Some(Sequence::new(vec![
         Replace::new("▁", " ").unwrap().into(),
-        ByteFallback::new().into(),
+        ByteFallback::default().into(),
         Fuse::new().into(),
     ])));
     let lines = encode_lines(&sp_chain, &data);

tokenizers/tk-encode/examples/fixture_bench.rs

@@ -670,7 +670,9 @@ fn bench_threads(
 // release. No released baseline → no decode oracle, so the whole phase is null.
 // `pipeline_ok` is the once-probed "can the pipeline decode yet" flag: while
 // `PipelineTokenizer::decode` is a loud stub it is false, so the pipeline series
-// is `null` (rendered "pending") and only the baseline bar is drawn.
+// is `null` (rendered "pending") and only the baseline bar is drawn. A decoder
+// that passes that probe but `Err`s on real ids fails `text_match` and nulls
+// its series for the affected fixtures/sweep — it never aborts the run.
 
 /// Encode every chunk with the released crate into its id stream (untimed input;
 /// specials included, so decode sees the frame tokens a real stream carries).
@@ -724,22 +726,28 @@ fn bench_decode(
     };
     let mbps = |secs: f64| dec_bytes as f64 / secs / 1e6;
 
+    // The `main` probe only decodes `[0]`, so a partial decoder can still `Err`
+    // on this fixture's real id streams — that fails the `text_match` gate and
+    // skips the pipeline timing (series stays null) instead of aborting the run.
+    let pipe_ok = pipeline_ok && ids.iter().all(|i| pipeline.decode(i, false).is_ok());
     // Correctness gate (first 3 chunks): pipeline decode == released decode.
     let text_match = pipeline_ok.then(|| {
-        ids.iter()
-            .take(3)
-            .all(|i| pipeline.decode(i, false).unwrap() == baseline.decode(i, false).unwrap())
+        pipe_ok
+            && ids
+                .iter()
+                .take(3)
+                .all(|i| pipeline.decode(i, false).unwrap() == baseline.decode(i, false).unwrap())
     });
 
     // Interleaved warm-up + REPS so thermal drift hits both equally.
     one_pass(&|i| baseline.decode(i, false).unwrap().len());
-    if pipeline_ok {
+    if pipe_ok {
         one_pass(&|i| pipeline.decode(i, false).unwrap().len());
     }
     let (mut base_s, mut pipe_s) = (Vec::new(), Vec::new());
     for _ in 0..REPS {
         base_s.push(one_pass(&|i| baseline.decode(i, false).unwrap().len()));
-        if pipeline_ok {
+        if pipe_ok {
             pipe_s.push(one_pass(&|i| pipeline.decode(i, false).unwrap().len()));
         }
     }
@@ -793,11 +801,14 @@ fn bench_decode_threads(
         .iter()
         .map(|i| baseline.decode(i, false).unwrap().len())
         .sum();
+    // Same tolerance as `bench_decode`: any `Err` over the corpus drops the
+    // pipeline series to null instead of panicking mid-sweep.
+    let pipe_ok = pipeline_ok && ids.iter().all(|i| pipeline.decode(i, false).is_ok());
     let counts = thread_counts();
     let (mut pipe, mut base) = (Vec::new(), Vec::new());
     for &n in &counts {
         let b = par_decode_mbps(|i| baseline.decode(i, false).unwrap().len(), &ids, bytes, n);
-        let p = pipeline_ok
+        let p = pipe_ok
             .then(|| par_decode_mbps(|i| pipeline.decode(i, false).unwrap().len(), &ids, bytes, n));
         eprintln!(
             "    decode {n} thread(s): pipeline {}, baseline {b:.1} MB/s",

tokenizers/tk-encode/src/decoders/byte_fallback.rs

@@ -1,4 +1,8 @@
-use crate::tokenizer::{Decoder, Result};
+use crate::{
+    pipeline::{self, DecoderState},
+    tokenizer::{Decoder, Result},
+};
+use ahash::AHashMap;
 use monostate::MustBe;
 
 use serde::{Deserialize, Serialize};
@@ -11,14 +15,32 @@ use serde::{Deserialize, Serialize};
 pub struct ByteFallback {
     #[serde(rename = "type")]
     type_: MustBe!("ByteFallback"),
+    /// Lookup mapping a token id to the raw byte it represents. Built from
+    /// the model when the pipeline is assembled, never part of tokenizer.json.
+    /// todo: closed-addressing, can use ptrhash
+    #[serde(skip)]
+    fallback_lookup: AHashMap<u32, u8>,
 }
 
 impl ByteFallback {
-    pub fn new() -> Self {
+    pub fn new(fallback_lookup: AHashMap<u32, u8>) -> Self {
         Self {
             type_: MustBe!("ByteFallback"),
+            fallback_lookup,
         }
     }
+
+    /// Invert the model's encode-time byte -> id table into the id -> byte
+    /// lookup decoding needs.
+    pub(crate) fn from_byte_to_id(byte_to_id: &[u32; 256]) -> Self {
+        Self::new(
+            byte_to_id
+                .iter()
+                .enumerate()
+                .map(|(byte, &id)| (id, byte as u8))
+                .collect(),
+        )
+    }
 }
 
 impl Decoder for ByteFallback {
@@ -62,13 +84,48 @@ impl Decoder for ByteFallback {
     }
 }
 
+impl pipeline::Decoder for ByteFallback {
+    fn decode_token(
+        &self,
+        state: &mut pipeline::DecoderState,
+        token_id: u32,
+        token_bytes: &[u8],
+        decoded: &mut Vec<u8>,
+    ) -> Result<()> {
+        if let Some(&raw_byte) = self.fallback_lookup.get(&token_id) {
+            state.pending_buffer.push(raw_byte);
+            return Ok(());
+        }
+        self.flush(state, decoded)?;
+        decoded.extend_from_slice(token_bytes);
+        Ok(())
+    }
+
+    fn flush(&self, state: &mut DecoderState, decoded: &mut Vec<u8>) -> Result<()> {
+        if state.pending_buffer.is_empty() {
+            return Ok(());
+        }
+        if std::str::from_utf8(&state.pending_buffer).is_ok() {
+            decoded.append(&mut state.pending_buffer);
+        } else {
+            // one '�' per byte token, like `decode_chain` above — not
+            // from_utf8_lossy, which merges maximal invalid subparts
+            for _ in 0..state.pending_buffer.len() {
+                decoded.extend_from_slice("�".as_bytes());
+            }
+            state.pending_buffer.clear();
+        }
+        Ok(())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
 
     #[test]
     fn decode() {
-        let decoder = ByteFallback::new();
+        let decoder = ByteFallback::new(AHashMap::new());
         let res = decoder
             .decode_chain(vec!["Hey".into(), "friend!".into()])
             .unwrap();

tokenizers/tk-encode/src/decoders/strip.rs

@@ -1,4 +1,7 @@
-use crate::tokenizer::{Decoder, Result};
+use crate::{
+    pipeline,
+    tokenizer::{Decoder, Result},
+};
 
 use serde::{Deserialize, Serialize};
 
@@ -59,6 +62,34 @@ impl Decoder for Strip {
     }
 }
 
+impl pipeline::Decoder for Strip {
+    fn decode_token(
+        &self,
+        _state: &mut pipeline::DecoderState,
+        _token_id: u32,
+        token_bytes: &[u8],
+        decoded: &mut Vec<u8>,
+    ) -> Result<()> {
+        let mut pat_buf = [0u8; 4];
+        let pat = self.content.encode_utf8(&mut pat_buf).as_bytes();
+        let mut token = token_bytes;
+        for _ in 0..self.start {
+            match token.strip_prefix(pat) {
+                Some(rest) => token = rest,
+                None => break,
+            }
+        }
+        for _ in 0..self.stop {
+            match token.strip_suffix(pat) {
+                Some(rest) => token = rest,
+                None => break,
+            }
+        }
+        decoded.extend_from_slice(token);
+        Ok(())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

tokenizers/tk-encode/src/decoders/wordpiece.rs

@@ -1,4 +1,9 @@
-use crate::tokenizer::{Decoder, Result};
+use std::mem::replace;
+
+use crate::{
+    pipeline::{self, DecoderState},
+    tokenizer::{Decoder, Result},
+};
 
 use serde::{Deserialize, Serialize};
 
@@ -28,6 +33,7 @@ impl Default for WordPiece {
         }
     }
 }
+
 pub fn cleanup(dirty_input: &str) -> String {
     dirty_input
         .replace(" .", ".")
@@ -61,6 +67,32 @@ impl Decoder for WordPiece {
     }
 }
 
+const CLEANUP_LIST: [&[u8]; 10] = [
+    b".", b"?", b"!", b",", b"n't", b"'m", b"do not", b"'s", b"'ve", b"'re",
+];
+
+impl pipeline::Decoder for WordPiece {
+    fn decode_token(
+        &self,
+        state: &mut DecoderState,
+        _token_id: u32,
+        token_bytes: &[u8],
+        decoded: &mut Vec<u8>,
+    ) -> Result<()> {
+        if !replace(&mut state.started, true) {
+            decoded.extend(token_bytes)
+        } else if token_bytes.starts_with(self.prefix.as_bytes()) {
+            decoded.extend_from_slice(&token_bytes[self.prefix.len()..]);
+        } else if self.cleanup && CLEANUP_LIST.contains(&token_bytes) {
+            decoded.extend(token_bytes);
+        } else {
+            decoded.push(b' ');
+            decoded.extend(token_bytes);
+        }
+        Ok(())
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

tokenizers/tk-encode/src/models/bpe/model.rs

@@ -763,6 +763,19 @@ impl PipelineBPE {
         })
     }
 
+    /// The `<0xHH>` token ids indexed by byte, when this model encodes with
+    /// byte fallback. `Atoms::Bytes` also holds a byte -> id table, but it
+    /// maps byte-level atoms, not `<0xHH>` tokens, so it is not exposed here.
+    pub(crate) fn byte_fallback_ids(&self) -> Option<&[u32; 256]> {
+        match &self.atoms {
+            Atoms::Chars {
+                byte_fallback: Some(table),
+                ..
+            } => Some(table),
+            _ => None,
+        }
+    }
+
     fn merge_word(
         &self,
         sequence: &str,
@@ -855,6 +868,10 @@ impl pipeline::Model for PipelineBPE {
             skip: Vec::new(),
         }
     }
+
+    fn id_to_token_bytes(&self, id: u32) -> Option<&[u8]> {
+        self.vocab.id_to_token_bytes(id)
+    }
 }
 
 pub struct BpeScratch {

tokenizers/tk-encode/src/models/unigram/model.rs

@@ -549,6 +549,10 @@ impl pipeline::Model for Unigram {
         }
         Ok(())
     }
+
+    fn id_to_token_bytes(&self, id: u32) -> Option<&[u8]> {
+        self.token_to_ids.id_to_token_bytes(id)
+    }
 }
 
 #[cfg(test)]

tokenizers/tk-encode/src/models/wordlevel/mod.rs

@@ -228,6 +228,10 @@ impl pipeline::Model for WordLevel {
         }
         Ok(())
     }
+
+    fn id_to_token_bytes(&self, id: u32) -> Option<&[u8]> {
+        self.vocab_r.get(&id).map(|s| s.as_bytes())
+    }
 }
 
 #[cfg(test)]