Update Metaspace Conversion (openvinotoolkit#578)

Author: apaniukov

README.md

@@ -469,12 +469,12 @@ This report is autogenerated and includes tokenizers and detokenizers tests. The
   <tbody>
     <tr>
       <td >BPE</td>
-      <td >99.26</td>
-      <td >6218</td>
+      <td >99.45</td>
+      <td >6216</td>
     </tr>
     <tr>
       <td >SentencePiece</td>
-      <td >89.50</td>
+      <td >89.76</td>
       <td >6036</td>
     </tr>
     <tr>
@@ -515,7 +515,7 @@ This report is autogenerated and includes tokenizers and detokenizers tests. The
     <tr>
       <td >BPE</td>
       <td >NousResearch/Llama-2-13b-hf</td>
-      <td >97.61</td>
+      <td >100.00</td>
       <td >251</td>
     </tr>
     <tr>
@@ -605,7 +605,7 @@ This report is autogenerated and includes tokenizers and detokenizers tests. The
     <tr>
       <td >BPE</td>
       <td >llava-hf/LLaVA-NeXT-Video-7B-hf</td>
-      <td >97.61</td>
+      <td >100.00</td>
       <td >251</td>
     </tr>
     <tr>
@@ -671,13 +671,13 @@ This report is autogenerated and includes tokenizers and detokenizers tests. The
     <tr>
       <td >SentencePiece</td>
       <td >NousResearch/Llama-2-13b-hf</td>
-      <td >94.42</td>
+      <td >96.02</td>
       <td >251</td>
     </tr>
     <tr>
       <td >SentencePiece</td>
       <td >NousResearch/Llama-2-13b-hf_legacy</td>
-      <td >97.61</td>
+      <td >99.20</td>
       <td >251</td>
     </tr>
     <tr>
@@ -737,13 +737,13 @@ This report is autogenerated and includes tokenizers and detokenizers tests. The
     <tr>
       <td >SentencePiece</td>
       <td >llava-hf/LLaVA-NeXT-Video-7B-hf</td>
-      <td >93.63</td>
+      <td >95.22</td>
       <td >251</td>
     </tr>
     <tr>
       <td >SentencePiece</td>
       <td >llava-hf/LLaVA-NeXT-Video-7B-hf_legacy</td>
-      <td >96.81</td>
+      <td >98.41</td>
       <td >251</td>
     </tr>
     <tr>

benchmark/benchmark.py

@@ -212,6 +212,9 @@ def main(
 ) -> None:
     hf_tokenizer = AutoTokenizer.from_pretrained(checkpoint, trust_remote_code=trust)
 
+    if hf_tokenizer.pad_token is None:
+        hf_tokenizer.pad_token = hf_tokenizer.eos_token
+
     hint = properties.hint.PerformanceMode.THROUGHPUT if tput else properties.hint.PerformanceMode.LATENCY
     config = {properties.hint.performance_mode(): hint}
     if per_layer_stats:

python/openvino_tokenizers/hf_parser.py

@@ -211,24 +211,6 @@ def parse_normalizer_step(self, step_dict: dict[str, Any]) -> None:
         except KeyError:
             raise OVTypeError(f"Normalizer type '{step_dict['type']}' is not supported")
 
-    @staticmethod
-    def check_metaspace_normalizer(normalizer_dict: dict[str, Any]) -> bool:
-        if normalizer_dict.get("type") == "Sequence":
-            normalizers = normalizer_dict["normalizers"]
-
-            if len(normalizers) != 2:
-                return False
-            first, second = normalizers
-            first_prerend = bool(first.get("type") == "Prepend" and first.get("prepend") == "▁")
-            second_replace = bool(
-                second.get("type") == "Replace"
-                and second.get("pattern", {}).get("String") == " "
-                and second.get("content") == "▁"
-            )
-            return first_prerend and second_replace
-
-        return False
-
     def normalization(self) -> None:
         if self.tokenizer_json["normalizer"] is None:
             return

python/openvino_tokenizers/tokenizer_pipeline.py

@@ -1493,12 +1493,32 @@ def merge_regex_split_steps(self) -> None:
         steps_without_pre_tokenization[first_step_position:first_step_position] = new_regex_split_steps
         self.steps = steps_without_pre_tokenization
 
+    def update_metaspace_step_with_special_tokens(self) -> None:
+        """
+        No metaspace insertion when input starts with special token.
+        """
+        if not self.is_metaspace_prepend_first:
+            return
+        special_tokens_split = next(
+            (step for step in self.steps if isinstance(step, SpecialTokensSplit)),
+            None,
+        )
+        if not special_tokens_split:
+            return
+        metaspace_step, special_tokens_split = self.steps[:2]
+
+        metaspace_step.regex_search_pattern = r"(^)((?!{}| |$)|(?=[\r\n\t\f\v]))".format(
+            "|".join(quote_meta(token.text) for token in special_tokens_split.special_tokens)
+        )
+        metaspace_step.global_replace = False
+
     def finalize(self) -> None:
         if self.finalized:
             return
 
         self.merge_normalization_steps()
         self.del_duplicated_split_steps()
+        self.update_metaspace_step_with_special_tokens()
 
         for step in copy(self.steps):
             step.finalize()
@@ -1507,6 +1527,10 @@ def finalize(self) -> None:
         self.merge_regex_split_steps()
         self.finalized = True
 
+    @property
+    def is_metaspace_prepend_first(self) -> bool:
+        return isinstance(self.steps[0], RegexNormalizationStep)
+
     def get_tokenizer_ov_subgraph(self) -> Model:
         self.finalize()
 
@@ -1516,7 +1540,7 @@ def get_tokenizer_ov_subgraph(self) -> Model:
         for input_node in string_inputs:
             input_node = _get_opset_factory("opset15").create("StringTensorUnpack", input_node.outputs()).outputs()
 
-            if isinstance(self.steps[0], RegexNormalizationStep):
+            if self.is_metaspace_prepend_first:
                 prepend_metaspace_step = self.steps.pop(0)
                 input_node = prepend_metaspace_step.get_ov_subgraph(input_node)
 

tests/pass_rates.json

@@ -1,3 +1,3 @@
 {
-    "tests/tokenizers_test.py::test_": 0.9524793388429752
+    "tests/tokenizers_test.py::test_": 0.9541752765285038
 }