botpress · slvnperron · Jul 2, 2026 · Jul 1, 2026 · Jul 1, 2026 · Jul 1, 2026
diff --git a/.gitignore b/.gitignore
@@ -81,3 +81,6 @@ gen/
 .ignore.me.*
 
 custom-e2e/
+
+# gstack (local tooling state)
+.gstack/
diff --git a/thicktoken/NOTICE b/thicktoken/NOTICE
@@ -0,0 +1,39 @@
+thicktoken
+==========
+
+This package's experimental WebAssembly tokenizer (see `wasm/`) is built on the
+`tokie` BPE tokenizer crate:
+
+    tokie — https://github.com/feyninc/tokie
+    Copyright (c) 2025 Chonkie, Inc.
+    Licensed under MIT OR Apache-2.0
+
+We depend on the published `tokie` crate as our BPE engine (loaded with a
+custom merges-only cl100k asset and a thin wasm-bindgen wrapper). Shoutout and
+thanks to the Chonkie team for the fast, accurate tokenizer.
+
+The full MIT license text for `tokie` follows.
+
+---------------------------------------------------------------------
+
+MIT License
+
+Copyright (c) 2025 Chonkie, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
diff --git a/thicktoken/browser-test.html b/thicktoken/browser-test.html
@@ -0,0 +1,51 @@
+<!doctype html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>thicktoken wasm browser test</title>
+  </head>
+  <body>
+    <h1>thicktoken browser test — full / lite / micro</h1>
+    <p>Serve this directory over HTTP (module imports don't work from <code>file://</code>).</p>
+    <pre id="out">loading…</pre>
+    <script type="module">
+      const out = document.getElementById('out')
+      const lines = []
+      const log = (s) => {
+        lines.push(s)
+        out.textContent = lines.join('\n')
+      }
+      window.addEventListener('error', (e) => log('WINDOW ERROR: ' + e.message))
+      try {
+        const sample = 'The quick brown fox jumps over the lazy dog. '.repeat(200)
+        const checks = []
+        let fullCount = 0
+        for (const [name, path] of [
+          ['full ', './dist/tokenizer.mjs'],
+          ['lite ', './dist/lite.mjs'],
+          ['micro', './dist/micro.mjs'],
+        ]) {
+          const t0 = performance.now()
+          const { getWasmTokenizer } = await import(path)
+          const tok = await getWasmTokenizer()
+          const init = performance.now() - t0
+          const count = tok.count(sample, { approximate: false })
+          if (name.trim() === 'full') fullCount = count
+          const ok =
+            tok.truncate('one two three four', 2) === 'one two' &&
+            tok.truncate('one two three four', 2, 'tail') === ' three four' &&
+            tok.split('Hello').join('') === 'Hello' &&
+            count >= fullCount // truncated variants only ever overcount
+          checks.push(ok)
+          log(
+            `${ok ? 'PASS' : 'FAIL'} ${name} init=${init.toFixed(0)}ms count=${count} (${((count / fullCount) * 100).toFixed(1)}% of full) truncate/split ok=${ok}`
+          )
+        }
+        log('')
+        log(checks.every(Boolean) ? `ALL ${checks.length} VARIANTS PASSED ✅` : 'SOME CHECKS FAILED ❌')
+      } catch (e) {
+        log('ERROR: ' + (e && e.stack ? e.stack : String(e)))
+      }
+    </script>
+  </body>
+</html>
diff --git a/thicktoken/package.json b/thicktoken/package.json
@@ -1,7 +1,7 @@
 {
   "name": "@bpinternal/thicktoken",
-  "version": "2.0.0",
-  "description": "Tiktoken but thicker; A bundled tokenizer with added helper functions",
+  "version": "3.0.0",
+  "description": "A bundled cl100k tokenizer (Rust→WASM) with count/truncate/split helpers",
   "repository": {
     "url": "https://github.com/botpress/packages"
   },
@@ -17,6 +17,16 @@
       "types": "./dist/tokenizer.d.ts",
       "import": "./dist/tokenizer.mjs",
       "require": "./dist/tokenizer.js"
+    },
+    "./lite": {
+      "types": "./dist/lite.d.ts",
+      "import": "./dist/lite.mjs",
+      "require": "./dist/lite.js"
+    },
+    "./micro": {
+      "types": "./dist/micro.d.ts",
+      "import": "./dist/micro.mjs",
+      "require": "./dist/micro.js"
     }
   },
   "keywords": [],
@@ -25,8 +35,8 @@
   "devDependencies": {
     "@types/node": "^22.16.4",
     "dotenv": "^16.4.7",
-    "tiktoken": "^1.0.17",
-    "tsup": "^8.3.6",
+    "gpt-tokenizer": "3.4.0",
+    "tsup": "^8.5.1",
     "typescript": "^5.7.3",
     "vitest": "^2.0.5"
   },