Merge pull request #87 from IAPark/oss-mods

Support `o200k_harmony`

Author: gjtorikian

Cargo.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    tiktoken_ruby (0.0.12)
+    tiktoken_ruby (0.0.13)
       rb_sys (~> 0.9)
 
 GEM

Gemfile.lock

@@ -12,4 +12,4 @@ crate-type = ["cdylib"]
 [dependencies]
 magnus = { version = "0.8.2" }
 rb-sys = { version = "0.9.117", features = ["stable-api-compiled-fallback"] }
-tiktoken-rs = { version = "0.7.0" }
+tiktoken-rs = { version = "0.9.0" }

ext/tiktoken_ruby/Cargo.toml

@@ -25,12 +25,20 @@ fn o200k_base() -> CoreBPEWrapper {
     CoreBPEWrapper::new(core_bpe)
 }
 
+fn o200k_harmony() -> CoreBPEWrapper {
+    let core_bpe = tiktoken_rs::o200k_harmony().unwrap();
+    CoreBPEWrapper::new(core_bpe)
+}
+
 fn module() -> Result<RModule, magnus::Error> {
     Ruby::get().unwrap().define_module("Tiktoken")
 }
 
 fn uncicode_error() -> Result<ExceptionClass, magnus::Error> {
-    module()?.define_error("UnicodeError", Ruby::get().unwrap().exception_standard_error())
+    module()?.define_error(
+        "UnicodeError",
+        Ruby::get().unwrap().exception_standard_error(),
+    )
 }
 
 #[magnus::init]
@@ -43,6 +51,7 @@ fn init() -> Result<(), Error> {
     factory_module.define_singleton_method("p50k_edit", function!(p50k_edit, 0))?;
     factory_module.define_singleton_method("cl100k_base", function!(cl100k_base, 0))?;
     factory_module.define_singleton_method("o200k_base", function!(o200k_base, 0))?;
+    factory_module.define_singleton_method("o200k_harmony", function!(o200k_harmony, 0))?;
 
     let ext_module = module.define_module("Ext")?;
     let bpe_class = ext_module.define_class("CoreBPE", Ruby::get().unwrap().class_object())?;

ext/tiktoken_ruby/src/lib.rs

@@ -65,7 +65,8 @@ def list_model_names
       :p50k_base,
       :p50k_edit,
       :cl100k_base,
-      :o200k_base
+      :o200k_base,
+      :o200k_harmony
     ]
 
     # taken from the python library here https://github.com/openai/tiktoken/blob/main/tiktoken/model.py
@@ -143,6 +144,7 @@ def list_model_names
       "gpt-4-": "cl100k_base",  # e.g., gpt-4-0314, etc., plus gpt-4-32k
       "gpt-3.5-turbo-": "cl100k_base",  # e.g, gpt-3.5-turbo-0301, -0401, etc.
       "gpt-35-turbo-": "cl100k_base",  # Azure deployment name
+      "gpt-oss-": "o200k_harmony",
       # fine-tuned
       "ft:gpt-4o": "cl100k_base",
       "ft:gpt-4": "cl100k_base",

lib/tiktoken_ruby.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Tiktoken
-  VERSION = "0.0.12"
+  VERSION = "0.0.13"
 end