fix: api

Author: SBrandeis

tokenizers/tk-encode/src/models/bpe/model.rs

@@ -856,8 +856,8 @@ impl pipeline::Model for PipelineBPE {
         }
     }
 
-    fn id_to_token_bytes(&self, id: &PipelineToken) -> Option<&[u8]> {
-        self.vocab.id_to_token_bytes(id.id)
+    fn id_to_token_bytes(&self, id: u32) -> Option<&[u8]> {
+        self.vocab.id_to_token_bytes(id)
     }
 }
 

tokenizers/tk-encode/src/models/unigram/model.rs

@@ -550,8 +550,8 @@ impl pipeline::Model for Unigram {
         Ok(())
     }
 
-    fn id_to_token_bytes(&self, id: &PipelineToken) -> Option<&[u8]> {
-        self.token_to_ids.id_to_token_bytes(id.id)
+    fn id_to_token_bytes(&self, id: u32) -> Option<&[u8]> {
+        self.token_to_ids.id_to_token_bytes(id)
     }
 }
 

tokenizers/tk-encode/src/models/wordlevel/mod.rs

@@ -229,8 +229,8 @@ impl pipeline::Model for WordLevel {
         Ok(())
     }
 
-    fn id_to_token_bytes(&self, id: &PipelineToken) -> Option<&[u8]> {
-        self.vocab_r.get(&id.id).map(|s| s.as_bytes())
+    fn id_to_token_bytes(&self, id: u32) -> Option<&[u8]> {
+        self.vocab_r.get(&id).map(|s| s.as_bytes())
     }
 }
 

tokenizers/tk-encode/src/models/wordpiece/mod.rs

@@ -435,8 +435,8 @@ impl pipeline::Model for PipelineWordPiece {
         Ok(())
     }
 
-    fn id_to_token_bytes(&self, id: &PipelineToken) -> Option<&[u8]> {
-        let i = id.id as usize;
+    fn id_to_token_bytes(&self, id: u32) -> Option<&[u8]> {
+        let i = id as usize;
         let start = *self.vocab_r_offsets.get(i)? as usize;
         let end = *self.vocab_r_offsets.get(i + 1)? as usize;
         Some(&self.vocab_r[start..end])
@@ -454,7 +454,7 @@ mod tests {
 
     #[test]
     fn id_to_token_bytes_round_trips() {
-        use crate::pipeline::{Model as _, PipelineToken};
+        use crate::pipeline::Model as _;
 
         let vocab: Vocab = [
             ("[UNK]".to_string(), 0),
@@ -467,9 +467,9 @@ mod tests {
         let model = PipelineWordPiece::try_from(wp).unwrap();
 
         for (token, id) in [("[UNK]", 0), ("hello", 1), ("##world", 2)] {
-            let bytes = model.id_to_token_bytes(&PipelineToken { id }).unwrap();
+            let bytes = model.id_to_token_bytes(id).unwrap();
             assert_eq!(bytes, token.as_bytes());
         }
-        assert_eq!(model.id_to_token_bytes(&PipelineToken { id: 3 }), None);
+        assert_eq!(model.id_to_token_bytes(3), None);
     }
 }

tokenizers/tk-encode/src/tokenizer/pipeline.rs

@@ -417,18 +417,16 @@ impl PipelineTokenizer {
 
     /// Decode token ids back to a `String`.
     ///
-    /// Not implemented yet — the pipeline decode path is being built. It fails
-    /// loud (rather than returning a plausible-but-wrong string) so the oracle
-    /// test and the comparative benchmark report decode as *pending* instead of
-    /// silently validating garbage. Implementing this flips the ignored
-    /// `pipeline_decode_oracle` test on and lights up the decode charts.
-    pub fn decode(&self, ids: &[PipelineToken], _skip_special_tokens: bool) -> Result<String> {
+    /// Incomplete: it concatenates raw token bytes only. No decoder, no added-
+    /// vocab lookup, no `skip_special_tokens` — so the `pipeline_decode_oracle`
+    /// test fails on purpose until those land.
+    pub fn decode(&self, ids: &[u32], _skip_special_tokens: bool) -> Result<String> {
         let mut output = Vec::with_capacity(ids.len());
-        for id in ids {
+        for &id in ids {
             let slice = self
                 .model
                 .id_to_token_bytes(id)
-                .ok_or::<crate::Error>(format!("Invalid token id: {}", id.id).into())?;
+                .ok_or::<crate::Error>(format!("Invalid token id: {id}").into())?;
             output.extend_from_slice(slice);
         }
         Ok(String::from_utf8(output)?)
@@ -729,7 +727,7 @@ pub trait Model {
 
     fn init_scratch(&self) -> Self::Scratch;
 
-    fn id_to_token_bytes(&self, id: &PipelineToken) -> Option<&[u8]>;
+    fn id_to_token_bytes(&self, id: u32) -> Option<&[u8]>;
 }
 
 #[allow(
@@ -778,7 +776,7 @@ impl Model for PipelineModel {
         }
     }
 
-    fn id_to_token_bytes(&self, id: &PipelineToken) -> Option<&[u8]> {
+    fn id_to_token_bytes(&self, id: u32) -> Option<&[u8]> {
         match self {
             Self::BPE(model) => model.id_to_token_bytes(id),
             Self::WordLevel(model) => model.id_to_token_bytes(id),

tokenizers/tk-encode/tests/pipeline_decode_oracle.rs

@@ -88,7 +88,6 @@ macro_rules! decode_tests {
     ($($name:ident => $tok:literal),* $(,)?) => {
         $(
             #[test]
-            #[ignore = "un-ignore once PipelineTokenizer::decode is implemented"]
             fn $name() {
                 check($tok);
             }