[GGUF] Create tokenizers factory for GGUF support in OpenVINO GenAI (#494)

* [GGUF] Create tokenizers factory method

Needed for GGUF tokenizers support

Signed-off-by: Kazantsev, Roman <[email protected]>

* Clean-up implementation

Signed-off-by: Kazantsev, Roman <[email protected]>

* Update src/tokenizers_factory.cpp

* Update src/tokenizers_factory.cpp

---------

Signed-off-by: Kazantsev, Roman <[email protected]>

Author: rkazants

src/tokenizers_factory.cpp

@@ -0,0 +1,57 @@
+// Copyright (C) 2018-2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#include "tokenizers_factory.hpp"
+
+#include "openvino/core/except.hpp"
+#include "tokenizer.hpp"
+
+namespace ov {
+namespace tokenizers {
+
+namespace {
+template <typename T>
+T get_attribute_value(const ov::AnyMap& attributes, const std::string& attribute_name, const T& default_value) {
+    return attributes.count(attribute_name) && attributes.at(attribute_name).is<T>()
+               ? attributes.at(attribute_name).as<T>()
+               : default_value;
+}
+
+}  // namespace
+
+ov::OutputVector create_tokenizer_node(const std::string& op_type,
+                                       const ov::OutputVector& inputs,
+                                       const ov::AnyMap& attributes) {
+    if (op_type == "StringTensorUnpack") {
+        return std::make_shared<StringTensorUnpack>(inputs)->outputs();
+    } else if (op_type == "SpecialTokensSplit") {
+        return std::make_shared<SpecialTokensSplit>(inputs)->outputs();
+    } else if (op_type == "RegexSplit") {
+        auto behaviour = get_attribute_value<std::string>(attributes, "behaviour", "remove");
+        auto invert = get_attribute_value<bool>(attributes, "invert", false);
+        return std::make_shared<RegexSplit>(inputs, behaviour, invert)->outputs();
+    } else if (op_type == "RaggedToDense") {
+        auto pad_right = get_attribute_value<bool>(attributes, "pad_right", true);
+        auto pad_max_length = get_attribute_value<bool>(attributes, "pad_max_length", false);
+        return std::make_shared<RaggedToDense>(inputs, pad_right, pad_max_length)->outputs();
+    } else if (op_type == "VocabDecoder") {
+        return std::make_shared<VocabDecoder>(inputs, std::vector<int32_t>{})->outputs();
+    } else if (op_type == "FuzeRagged") {
+        return std::make_shared<FuzeRagged>(inputs)->outputs();
+    } else if (op_type == "StringTensorPack") {
+        return std::make_shared<StringTensorPack>(inputs)->outputs();
+    } else if (op_type == "BPETokenizer") {
+        auto unk_token = get_attribute_value<std::string>(attributes, "unk_token", "");
+        auto fuse_unk = get_attribute_value<bool>(attributes, "fuse_unk", false);
+        auto suffix_indicator = get_attribute_value<std::string>(attributes, "suffix_indicator", "");
+        auto end_suffix = get_attribute_value<std::string>(attributes, "end_suffix", "");
+        auto byte_fallback = get_attribute_value<bool>(attributes, "byte_fallback", false);
+        return std::make_shared<BPETokenizer>(inputs, unk_token, fuse_unk, suffix_indicator, end_suffix, byte_fallback)
+            ->outputs();
+    }
+    OPENVINO_THROW("Unsupported operation type: `", op_type, "`");
+}
+
+}  // namespace tokenizers
+}  // namespace ov

src/tokenizers_factory.hpp

@@ -0,0 +1,36 @@
+// Copyright (C) 2018-2025 Intel Corporation
+// SPDX-License-Identifier: Apache-2.0
+//
+
+#pragma once
+
+#include "openvino/core/core.hpp"
+
+namespace ov {
+namespace tokenizers {
+
+/**
+ * @brief Creates an OpenVINO operation node of the specified type with the given inputs and attributes.
+ *
+ * This function constructs a node in the OpenVINO computational graph based on the provided operation type,
+ * input tensors, and a map of operation-specific attributes. It returns the output(s) produced by the node.
+ *
+ * @note This function is used exclusively by OpenVINO GenAI to create tokenizer and detokenizer operations
+ * from a GGUF file. It is expected to be an external symbol that is located at runtime via `dlopen`.
+ *
+ * @warning The signature of this function must not be changed. It is dynamically loaded at runtime,
+ * and any modifications will break compatibility with OpenVINO GenAI.
+ *
+ * @param op_type A string specifying the type of operation to create (e.g., "BPETokenizer").
+ * @param inputs A vector of OpenVINO outputs (`ov::OutputVector`) representing the input tensors to the operation.
+ * @param attributes A map (`ov::AnyMap`) containing operation-specific attributes.
+ *
+ * @return ov::OutputVector A vector containing the output(s) of the created node. The number of outputs depends on the
+ * operation type.
+ */
+
+OPENVINO_API_C(ov::OutputVector)
+create_tokenizer_node(const std::string& op_type, const ov::OutputVector& inputs, const ov::AnyMap& attributes);
+
+}  // namespace tokenizers
+}  // namespace ov