diff --git a/.gitignore b/.gitignore index d5cb32939..475e495ba 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,6 @@ gen/ .ignore.me.* custom-e2e/ + +# gstack (local tooling state) +.gstack/ diff --git a/thicktoken/NOTICE b/thicktoken/NOTICE new file mode 100644 index 000000000..342627d29 --- /dev/null +++ b/thicktoken/NOTICE @@ -0,0 +1,39 @@ +thicktoken +========== + +This package's experimental WebAssembly tokenizer (see `wasm/`) is built on the +`tokie` BPE tokenizer crate: + + tokie — https://github.com/feyninc/tokie + Copyright (c) 2025 Chonkie, Inc. + Licensed under MIT OR Apache-2.0 + +We depend on the published `tokie` crate as our BPE engine (loaded with a +custom merges-only cl100k asset and a thin wasm-bindgen wrapper). Shoutout and +thanks to the Chonkie team for the fast, accurate tokenizer. + +The full MIT license text for `tokie` follows. + +--------------------------------------------------------------------- + +MIT License + +Copyright (c) 2025 Chonkie, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/thicktoken/browser-test.html b/thicktoken/browser-test.html new file mode 100644 index 000000000..f908b89f5 --- /dev/null +++ b/thicktoken/browser-test.html @@ -0,0 +1,51 @@ + + + + + thicktoken wasm browser test + + +

thicktoken browser test — full / lite / micro

+

Serve this directory over HTTP (module imports don't work from file://).

+ 
loading…
+ + + diff --git a/thicktoken/package.json b/thicktoken/package.json index a91171bf9..89d450f01 100644 --- a/thicktoken/package.json +++ b/thicktoken/package.json @@ -1,7 +1,7 @@ { "name": "@bpinternal/thicktoken", - "version": "2.0.0", - "description": "Tiktoken but thicker; A bundled tokenizer with added helper functions", + "version": "3.0.0", + "description": "A bundled cl100k tokenizer (Rust→WASM) with count/truncate/split helpers", "repository": { "url": "https://github.com/botpress/packages" }, @@ -17,6 +17,16 @@ "types": "./dist/tokenizer.d.ts", "import": "./dist/tokenizer.mjs", "require": "./dist/tokenizer.js" + }, + "./lite": { + "types": "./dist/lite.d.ts", + "import": "./dist/lite.mjs", + "require": "./dist/lite.js" + }, + "./micro": { + "types": "./dist/micro.d.ts", + "import": "./dist/micro.mjs", + "require": "./dist/micro.js" } }, "keywords": [], @@ -25,8 +35,8 @@ "devDependencies": { "@types/node": "^22.16.4", "dotenv": "^16.4.7", - "tiktoken": "^1.0.17", - "tsup": "^8.3.6", + "gpt-tokenizer": "3.4.0", + "tsup": "^8.5.1", "typescript": "^5.7.3", "vitest": "^2.0.5" }, diff --git a/thicktoken/pnpm-lock.yaml b/thicktoken/pnpm-lock.yaml index 6ff0e1200..23ca699c0 100644 --- a/thicktoken/pnpm-lock.yaml +++ b/thicktoken/pnpm-lock.yaml @@ -11,12 +11,12 @@ devDependencies: dotenv: specifier: ^16.4.7 version: 16.4.7 - tiktoken: - specifier: ^1.0.17 - version: 1.0.20 + gpt-tokenizer: + specifier: 3.4.0 + version: 3.4.0 tsup: - specifier: ^8.3.6 - version: 8.4.0(typescript@5.8.2) + specifier: ^8.5.1 + version: 8.5.1(typescript@5.8.2) typescript: specifier: ^5.7.3 version: 5.8.2 @@ -43,8 +43,8 @@ packages: dev: true optional: true - /@esbuild/aix-ppc64@0.25.2: - resolution: {integrity: sha512-wCIboOL2yXZym2cgm6mlA742s9QeJ8DjGVaL39dLN4rRwrOgOyYSnOaFPhKZGLb2ngj4EyfAFjsNJwPXZvseag==} + /@esbuild/aix-ppc64@0.27.7: + resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==} engines: {node: '>=18'} cpu: [ppc64] os: [aix] @@ -61,8 +61,8 @@ packages: dev: true optional: true - /@esbuild/android-arm64@0.25.2: - resolution: {integrity: sha512-5ZAX5xOmTligeBaeNEPnPaeEuah53Id2tX4c2CVP3JaROTH+j4fnfHCkr1PjXMd78hMst+TlkfKcW/DlTq0i4w==} + /@esbuild/android-arm64@0.27.7: + resolution: {integrity: sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==} engines: {node: '>=18'} cpu: [arm64] os: [android] @@ -79,8 +79,8 @@ packages: dev: true optional: true - /@esbuild/android-arm@0.25.2: - resolution: {integrity: sha512-NQhH7jFstVY5x8CKbcfa166GoV0EFkaPkCKBQkdPJFvo5u+nGXLEH/ooniLb3QI8Fk58YAx7nsPLozUWfCBOJA==} + /@esbuild/android-arm@0.27.7: + resolution: {integrity: sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==} engines: {node: '>=18'} cpu: [arm] os: [android] @@ -97,8 +97,8 @@ packages: dev: true optional: true - /@esbuild/android-x64@0.25.2: - resolution: {integrity: sha512-Ffcx+nnma8Sge4jzddPHCZVRvIfQ0kMsUsCMcJRHkGJ1cDmhe4SsrYIjLUKn1xpHZybmOqCWwB0zQvsjdEHtkg==} + /@esbuild/android-x64@0.27.7: + resolution: {integrity: sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==} engines: {node: '>=18'} cpu: [x64] os: [android] @@ -115,8 +115,8 @@ packages: dev: true optional: true - /@esbuild/darwin-arm64@0.25.2: - resolution: {integrity: sha512-MpM6LUVTXAzOvN4KbjzU/q5smzryuoNjlriAIx+06RpecwCkL9JpenNzpKd2YMzLJFOdPqBpuub6eVRP5IgiSA==} + /@esbuild/darwin-arm64@0.27.7: + resolution: {integrity: sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==} engines: {node: '>=18'} cpu: [arm64] os: [darwin] @@ -133,8 +133,8 @@ packages: dev: true optional: true - /@esbuild/darwin-x64@0.25.2: - resolution: {integrity: sha512-5eRPrTX7wFyuWe8FqEFPG2cU0+butQQVNcT4sVipqjLYQjjh8a8+vUTfgBKM88ObB85ahsnTwF7PSIt6PG+QkA==} + /@esbuild/darwin-x64@0.27.7: + resolution: {integrity: sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==} engines: {node: '>=18'} cpu: [x64] os: [darwin] @@ -151,8 +151,8 @@ packages: dev: true optional: true - /@esbuild/freebsd-arm64@0.25.2: - resolution: {integrity: sha512-mLwm4vXKiQ2UTSX4+ImyiPdiHjiZhIaE9QvC7sw0tZ6HoNMjYAqQpGyui5VRIi5sGd+uWq940gdCbY3VLvsO1w==} + /@esbuild/freebsd-arm64@0.27.7: + resolution: {integrity: sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==} engines: {node: '>=18'} cpu: [arm64] os: [freebsd] @@ -169,8 +169,8 @@ packages: dev: true optional: true - /@esbuild/freebsd-x64@0.25.2: - resolution: {integrity: sha512-6qyyn6TjayJSwGpm8J9QYYGQcRgc90nmfdUb0O7pp1s4lTY+9D0H9O02v5JqGApUyiHOtkz6+1hZNvNtEhbwRQ==} + /@esbuild/freebsd-x64@0.27.7: + resolution: {integrity: sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==} engines: {node: '>=18'} cpu: [x64] os: [freebsd] @@ -187,8 +187,8 @@ packages: dev: true optional: true - /@esbuild/linux-arm64@0.25.2: - resolution: {integrity: sha512-gq/sjLsOyMT19I8obBISvhoYiZIAaGF8JpeXu1u8yPv8BE5HlWYobmlsfijFIZ9hIVGYkbdFhEqC0NvM4kNO0g==} + /@esbuild/linux-arm64@0.27.7: + resolution: {integrity: sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==} engines: {node: '>=18'} cpu: [arm64] os: [linux] @@ -205,8 +205,8 @@ packages: dev: true optional: true - /@esbuild/linux-arm@0.25.2: - resolution: {integrity: sha512-UHBRgJcmjJv5oeQF8EpTRZs/1knq6loLxTsjc3nxO9eXAPDLcWW55flrMVc97qFPbmZP31ta1AZVUKQzKTzb0g==} + /@esbuild/linux-arm@0.27.7: + resolution: {integrity: sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==} engines: {node: '>=18'} cpu: [arm] os: [linux] @@ -223,8 +223,8 @@ packages: dev: true optional: true - /@esbuild/linux-ia32@0.25.2: - resolution: {integrity: sha512-bBYCv9obgW2cBP+2ZWfjYTU+f5cxRoGGQ5SeDbYdFCAZpYWrfjjfYwvUpP8MlKbP0nwZ5gyOU/0aUzZ5HWPuvQ==} + /@esbuild/linux-ia32@0.27.7: + resolution: {integrity: sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==} engines: {node: '>=18'} cpu: [ia32] os: [linux] @@ -241,8 +241,8 @@ packages: dev: true optional: true - /@esbuild/linux-loong64@0.25.2: - resolution: {integrity: sha512-SHNGiKtvnU2dBlM5D8CXRFdd+6etgZ9dXfaPCeJtz+37PIUlixvlIhI23L5khKXs3DIzAn9V8v+qb1TRKrgT5w==} + /@esbuild/linux-loong64@0.27.7: + resolution: {integrity: sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==} engines: {node: '>=18'} cpu: [loong64] os: [linux] @@ -259,8 +259,8 @@ packages: dev: true optional: true - /@esbuild/linux-mips64el@0.25.2: - resolution: {integrity: sha512-hDDRlzE6rPeoj+5fsADqdUZl1OzqDYow4TB4Y/3PlKBD0ph1e6uPHzIQcv2Z65u2K0kpeByIyAjCmjn1hJgG0Q==} + /@esbuild/linux-mips64el@0.27.7: + resolution: {integrity: sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==} engines: {node: '>=18'} cpu: [mips64el] os: [linux] @@ -277,8 +277,8 @@ packages: dev: true optional: true - /@esbuild/linux-ppc64@0.25.2: - resolution: {integrity: sha512-tsHu2RRSWzipmUi9UBDEzc0nLc4HtpZEI5Ba+Omms5456x5WaNuiG3u7xh5AO6sipnJ9r4cRWQB2tUjPyIkc6g==} + /@esbuild/linux-ppc64@0.27.7: + resolution: {integrity: sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==} engines: {node: '>=18'} cpu: [ppc64] os: [linux] @@ -295,8 +295,8 @@ packages: dev: true optional: true - /@esbuild/linux-riscv64@0.25.2: - resolution: {integrity: sha512-k4LtpgV7NJQOml/10uPU0s4SAXGnowi5qBSjaLWMojNCUICNu7TshqHLAEbkBdAszL5TabfvQ48kK84hyFzjnw==} + /@esbuild/linux-riscv64@0.27.7: + resolution: {integrity: sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==} engines: {node: '>=18'} cpu: [riscv64] os: [linux] @@ -313,8 +313,8 @@ packages: dev: true optional: true - /@esbuild/linux-s390x@0.25.2: - resolution: {integrity: sha512-GRa4IshOdvKY7M/rDpRR3gkiTNp34M0eLTaC1a08gNrh4u488aPhuZOCpkF6+2wl3zAN7L7XIpOFBhnaE3/Q8Q==} + /@esbuild/linux-s390x@0.27.7: + resolution: {integrity: sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==} engines: {node: '>=18'} cpu: [s390x] os: [linux] @@ -331,8 +331,8 @@ packages: dev: true optional: true - /@esbuild/linux-x64@0.25.2: - resolution: {integrity: sha512-QInHERlqpTTZ4FRB0fROQWXcYRD64lAoiegezDunLpalZMjcUcld3YzZmVJ2H/Cp0wJRZ8Xtjtj0cEHhYc/uUg==} + /@esbuild/linux-x64@0.27.7: + resolution: {integrity: sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==} engines: {node: '>=18'} cpu: [x64] os: [linux] @@ -340,8 +340,8 @@ packages: dev: true optional: true - /@esbuild/netbsd-arm64@0.25.2: - resolution: {integrity: sha512-talAIBoY5M8vHc6EeI2WW9d/CkiO9MQJ0IOWX8hrLhxGbro/vBXJvaQXefW2cP0z0nQVTdQ/eNyGFV1GSKrxfw==} + /@esbuild/netbsd-arm64@0.27.7: + resolution: {integrity: sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==} engines: {node: '>=18'} cpu: [arm64] os: [netbsd] @@ -358,8 +358,8 @@ packages: dev: true optional: true - /@esbuild/netbsd-x64@0.25.2: - resolution: {integrity: sha512-voZT9Z+tpOxrvfKFyfDYPc4DO4rk06qamv1a/fkuzHpiVBMOhpjK+vBmWM8J1eiB3OLSMFYNaOaBNLXGChf5tg==} + /@esbuild/netbsd-x64@0.27.7: + resolution: {integrity: sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==} engines: {node: '>=18'} cpu: [x64] os: [netbsd] @@ -367,8 +367,8 @@ packages: dev: true optional: true - /@esbuild/openbsd-arm64@0.25.2: - resolution: {integrity: sha512-dcXYOC6NXOqcykeDlwId9kB6OkPUxOEqU+rkrYVqJbK2hagWOMrsTGsMr8+rW02M+d5Op5NNlgMmjzecaRf7Tg==} + /@esbuild/openbsd-arm64@0.27.7: + resolution: {integrity: sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==} engines: {node: '>=18'} cpu: [arm64] os: [openbsd] @@ -385,8 +385,8 @@ packages: dev: true optional: true - /@esbuild/openbsd-x64@0.25.2: - resolution: {integrity: sha512-t/TkWwahkH0Tsgoq1Ju7QfgGhArkGLkF1uYz8nQS/PPFlXbP5YgRpqQR3ARRiC2iXoLTWFxc6DJMSK10dVXluw==} + /@esbuild/openbsd-x64@0.27.7: + resolution: {integrity: sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==} engines: {node: '>=18'} cpu: [x64] os: [openbsd] @@ -394,6 +394,15 @@ packages: dev: true optional: true + /@esbuild/openharmony-arm64@0.27.7: + resolution: {integrity: sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==} + engines: {node: '>=18'} + cpu: [arm64] + os: [openharmony] + requiresBuild: true + dev: true + optional: true + /@esbuild/sunos-x64@0.21.5: resolution: {integrity: sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg==} engines: {node: '>=12'} @@ -403,8 +412,8 @@ packages: dev: true optional: true - /@esbuild/sunos-x64@0.25.2: - resolution: {integrity: sha512-cfZH1co2+imVdWCjd+D1gf9NjkchVhhdpgb1q5y6Hcv9TP6Zi9ZG/beI3ig8TvwT9lH9dlxLq5MQBBgwuj4xvA==} + /@esbuild/sunos-x64@0.27.7: + resolution: {integrity: sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==} engines: {node: '>=18'} cpu: [x64] os: [sunos] @@ -421,8 +430,8 @@ packages: dev: true optional: true - /@esbuild/win32-arm64@0.25.2: - resolution: {integrity: sha512-7Loyjh+D/Nx/sOTzV8vfbB3GJuHdOQyrOryFdZvPHLf42Tk9ivBU5Aedi7iyX+x6rbn2Mh68T4qq1SDqJBQO5Q==} + /@esbuild/win32-arm64@0.27.7: + resolution: {integrity: sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==} engines: {node: '>=18'} cpu: [arm64] os: [win32] @@ -439,8 +448,8 @@ packages: dev: true optional: true - /@esbuild/win32-ia32@0.25.2: - resolution: {integrity: sha512-WRJgsz9un0nqZJ4MfhabxaD9Ft8KioqU3JMinOTvobbX6MOSUigSBlogP8QB3uxpJDsFS6yN+3FDBdqE5lg9kg==} + /@esbuild/win32-ia32@0.27.7: + resolution: {integrity: sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==} engines: {node: '>=18'} cpu: [ia32] os: [win32] @@ -457,8 +466,8 @@ packages: dev: true optional: true - /@esbuild/win32-x64@0.25.2: - resolution: {integrity: sha512-kM3HKb16VIXZyIeVrM1ygYmZBKybX8N4p754bw390wGO3Tf2j4L2/WYL+4suWujpgf6GBYs3jv7TyUivdd05JA==} + /@esbuild/win32-x64@0.27.7: + resolution: {integrity: sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==} engines: {node: '>=18'} cpu: [x64] os: [win32] @@ -892,6 +901,12 @@ packages: tinyrainbow: 1.2.0 dev: true + /acorn@8.17.0: + resolution: {integrity: sha512-xRQbDb9BnwDafYNn6Vwl839DYVjqXYb1XVGtWAZ1kcDc6iwAL4hg3B1dZlRiuENFeO2H53gFG3in621AdERVAg==} + engines: {node: '>=0.4.0'} + hasBin: true + dev: true + /ansi-regex@5.0.1: resolution: {integrity: sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==} engines: {node: '>=8'} @@ -933,13 +948,13 @@ packages: balanced-match: 1.0.2 dev: true - /bundle-require@5.1.0(esbuild@0.25.2): + /bundle-require@5.1.0(esbuild@0.27.7): resolution: {integrity: sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} peerDependencies: esbuild: '>=0.18' dependencies: - esbuild: 0.25.2 + esbuild: 0.27.7 load-tsconfig: 0.2.5 dev: true @@ -987,6 +1002,10 @@ packages: engines: {node: '>= 6'} dev: true + /confbox@0.1.8: + resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} + dev: true + /consola@3.4.2: resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} engines: {node: ^14.18.0 || >=16.10.0} @@ -1066,37 +1085,38 @@ packages: '@esbuild/win32-x64': 0.21.5 dev: true - /esbuild@0.25.2: - resolution: {integrity: sha512-16854zccKPnC+toMywC+uKNeYSv+/eXkevRAfwRD/G9Cleq66m8XFIrigkbvauLLlCfDL45Q2cWegSg53gGBnQ==} + /esbuild@0.27.7: + resolution: {integrity: sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==} engines: {node: '>=18'} hasBin: true requiresBuild: true optionalDependencies: - '@esbuild/aix-ppc64': 0.25.2 - '@esbuild/android-arm': 0.25.2 - '@esbuild/android-arm64': 0.25.2 - '@esbuild/android-x64': 0.25.2 - '@esbuild/darwin-arm64': 0.25.2 - '@esbuild/darwin-x64': 0.25.2 - '@esbuild/freebsd-arm64': 0.25.2 - '@esbuild/freebsd-x64': 0.25.2 - '@esbuild/linux-arm': 0.25.2 - '@esbuild/linux-arm64': 0.25.2 - '@esbuild/linux-ia32': 0.25.2 - '@esbuild/linux-loong64': 0.25.2 - '@esbuild/linux-mips64el': 0.25.2 - '@esbuild/linux-ppc64': 0.25.2 - '@esbuild/linux-riscv64': 0.25.2 - '@esbuild/linux-s390x': 0.25.2 - '@esbuild/linux-x64': 0.25.2 - '@esbuild/netbsd-arm64': 0.25.2 - '@esbuild/netbsd-x64': 0.25.2 - '@esbuild/openbsd-arm64': 0.25.2 - '@esbuild/openbsd-x64': 0.25.2 - '@esbuild/sunos-x64': 0.25.2 - '@esbuild/win32-arm64': 0.25.2 - '@esbuild/win32-ia32': 0.25.2 - '@esbuild/win32-x64': 0.25.2 + '@esbuild/aix-ppc64': 0.27.7 + '@esbuild/android-arm': 0.27.7 + '@esbuild/android-arm64': 0.27.7 + '@esbuild/android-x64': 0.27.7 + '@esbuild/darwin-arm64': 0.27.7 + '@esbuild/darwin-x64': 0.27.7 + '@esbuild/freebsd-arm64': 0.27.7 + '@esbuild/freebsd-x64': 0.27.7 + '@esbuild/linux-arm': 0.27.7 + '@esbuild/linux-arm64': 0.27.7 + '@esbuild/linux-ia32': 0.27.7 + '@esbuild/linux-loong64': 0.27.7 + '@esbuild/linux-mips64el': 0.27.7 + '@esbuild/linux-ppc64': 0.27.7 + '@esbuild/linux-riscv64': 0.27.7 + '@esbuild/linux-s390x': 0.27.7 + '@esbuild/linux-x64': 0.27.7 + '@esbuild/netbsd-arm64': 0.27.7 + '@esbuild/netbsd-x64': 0.27.7 + '@esbuild/openbsd-arm64': 0.27.7 + '@esbuild/openbsd-x64': 0.27.7 + '@esbuild/openharmony-arm64': 0.27.7 + '@esbuild/sunos-x64': 0.27.7 + '@esbuild/win32-arm64': 0.27.7 + '@esbuild/win32-ia32': 0.27.7 + '@esbuild/win32-x64': 0.27.7 dev: true /estree-walker@3.0.3: @@ -1131,6 +1151,14 @@ packages: picomatch: 4.0.2 dev: true + /fix-dts-default-cjs-exports@1.0.1: + resolution: {integrity: sha512-pVIECanWFC61Hzl2+oOCtoJ3F17kglZC/6N94eRWycFgBH35hHx0Li604ZIzhseh97mf2p0cv7vVrOZGoqhlEg==} + dependencies: + magic-string: 0.30.17 + mlly: 1.8.2 + rollup: 4.38.0 + dev: true + /foreground-child@3.3.1: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} @@ -1154,6 +1182,7 @@ packages: /glob@10.4.5: resolution: {integrity: sha512-7Bv8RF0k6xjo7d4A/PxYLbUCfb6c+Vpd2/mB2yRDlew7Jb5hEXiCD9ibfO7wpk8i4sevK6DFny9h7EYbM3/sHg==} + deprecated: Old versions of glob are not supported, and contain widely publicized security vulnerabilities, which have been fixed in the current version. Please update. Support for old versions may be purchased (at exorbitant rates) by contacting i@izs.me hasBin: true dependencies: foreground-child: 3.3.1 @@ -1164,6 +1193,10 @@ packages: path-scurry: 1.11.1 dev: true + /gpt-tokenizer@3.4.0: + resolution: {integrity: sha512-wxFLnhIXTDjYebd9A9pGl3e31ZpSypbpIJSOswbgop5jLte/AsZVDvjlbEuVFlsqZixVKqbcoNmRlFDf6pz/UQ==} + dev: true + /human-signals@5.0.0: resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} engines: {node: '>=16.17.0'} @@ -1210,10 +1243,6 @@ packages: engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} dev: true - /lodash.sortby@4.7.0: - resolution: {integrity: sha512-HDWXG8isMntAyRF5vZ7xKuEvOhT4AhlRt/3czTSjvGUxjYCBVRQY48ViDHyfYz9VIoBkW4TMGQNapx+l3RUwdA==} - dev: true - /loupe@3.1.2: resolution: {integrity: sha512-23I4pFZHmAemUnz8WZXbYRSKYj801VDaNv9ETuMh7IrMc7VuVVSo+Z9iLE3ni30+U48iDWfi30d3twAXBYmnCg==} dev: true @@ -1249,6 +1278,15 @@ packages: engines: {node: '>=16 || 14 >=14.17'} dev: true + /mlly@1.8.2: + resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} + dependencies: + acorn: 8.17.0 + pathe: 2.0.3 + pkg-types: 1.3.1 + ufo: 1.6.4 + dev: true + /ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} dev: true @@ -1312,6 +1350,10 @@ packages: resolution: {integrity: sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ==} dev: true + /pathe@2.0.3: + resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} + dev: true + /pathval@2.0.0: resolution: {integrity: sha512-vE7JKRyES09KiunauX7nd2Q9/L7lhok4smP9RZTDeD4MVs72Dp2qNFVz39Nz5a0FVEW0BJR6C0DYrq6unoziZA==} engines: {node: '>= 14.16'} @@ -1331,6 +1373,14 @@ packages: engines: {node: '>= 6'} dev: true + /pkg-types@1.3.1: + resolution: {integrity: sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ==} + dependencies: + confbox: 0.1.8 + mlly: 1.8.2 + pathe: 2.0.3 + dev: true + /postcss-load-config@6.0.1: resolution: {integrity: sha512-oPtTM4oerL+UXmx+93ytZVN82RrlY/wPUV8IeDxFrzIjXOLF1pN+EmKPLbubvKHT2HC20xXsCAH2Z+CKV6Oz/g==} engines: {node: '>= 18'} @@ -1361,11 +1411,6 @@ packages: source-map-js: 1.2.1 dev: true - /punycode@2.3.1: - resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} - engines: {node: '>=6'} - dev: true - /readdirp@4.1.2: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} @@ -1461,11 +1506,9 @@ packages: engines: {node: '>=0.10.0'} dev: true - /source-map@0.8.0-beta.0: - resolution: {integrity: sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==} - engines: {node: '>= 8'} - dependencies: - whatwg-url: 7.1.0 + /source-map@0.7.6: + resolution: {integrity: sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==} + engines: {node: '>= 12'} dev: true /stackback@0.0.2: @@ -1540,10 +1583,6 @@ packages: any-promise: 1.3.0 dev: true - /tiktoken@1.0.20: - resolution: {integrity: sha512-zVIpXp84kth/Ni2me1uYlJgl2RZ2EjxwDaWLeDY/s6fZiyO9n1QoTOM5P7ZSYfToPvAvwYNMbg5LETVYVKyzfQ==} - dev: true - /tinybench@2.9.0: resolution: {integrity: sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg==} dev: true @@ -1575,12 +1614,6 @@ packages: engines: {node: '>=14.0.0'} dev: true - /tr46@1.0.1: - resolution: {integrity: sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==} - dependencies: - punycode: 2.3.1 - dev: true - /tree-kill@1.2.2: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true @@ -1590,8 +1623,8 @@ packages: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} dev: true - /tsup@8.4.0(typescript@5.8.2): - resolution: {integrity: sha512-b+eZbPCjz10fRryaAA7C8xlIHnf8VnsaRqydheLIqwG/Mcpfk8Z5zp3HayX7GaTygkigHl5cBUs+IhcySiIexQ==} + /tsup@8.5.1(typescript@5.8.2): + resolution: {integrity: sha512-xtgkqwdhpKWr3tKPmCkvYmS9xnQK3m3XgxZHwSUjvfTjp7YfXe5tT3GgWi0F2N+ZSMsOeWeZFh7ZZFg5iPhing==} engines: {node: '>=18'} hasBin: true peerDependencies: @@ -1609,18 +1642,19 @@ packages: typescript: optional: true dependencies: - bundle-require: 5.1.0(esbuild@0.25.2) + bundle-require: 5.1.0(esbuild@0.27.7) cac: 6.7.14 chokidar: 4.0.3 consola: 3.4.2 debug: 4.4.0 - esbuild: 0.25.2 + esbuild: 0.27.7 + fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 picocolors: 1.1.1 postcss-load-config: 6.0.1 resolve-from: 5.0.0 rollup: 4.38.0 - source-map: 0.8.0-beta.0 + source-map: 0.7.6 sucrase: 3.35.0 tinyexec: 0.3.2 tinyglobby: 0.2.12 @@ -1639,6 +1673,10 @@ packages: hasBin: true dev: true + /ufo@1.6.4: + resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} + dev: true + /undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} dev: true @@ -1760,18 +1798,6 @@ packages: - terser dev: true - /webidl-conversions@4.0.2: - resolution: {integrity: sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==} - dev: true - - /whatwg-url@7.1.0: - resolution: {integrity: sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==} - dependencies: - lodash.sortby: 4.7.0 - tr46: 1.0.1 - webidl-conversions: 4.0.2 - dev: true - /which@2.0.2: resolution: {integrity: sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==} engines: {node: '>= 8'} diff --git a/thicktoken/readme.md b/thicktoken/readme.md index 4a64301ae..6870fd42f 100644 --- a/thicktoken/readme.md +++ b/thicktoken/readme.md @@ -1,6 +1,31 @@ # ThickToken -Tiktoken but thicker; A bundled tokenizer with added helper functions +A bundled cl100k tokenizer with added helper functions (count / truncate / split), powered +by our own Rust→WASM build (see `wasm/`, engine: [tokie](https://github.com/feyninc/tokie), +MIT © Chonkie, Inc. — see `NOTICE`). Historically backed by tiktoken — the API is unchanged. + +- `count(text, { approximate })` — approximate **by default** on large inputs (statistical + sampling, within a few %, ~100× faster); pass `{ approximate: false }` for exact. +- `truncate(text, n)` — exact; keeps the first `n` tokens (negative `n` removes from the + end). Only tokenizes the window it needs, so it's fast on huge inputs. +- `split(text)` / `splitAndSlice(text)` — per-token strings / sliceable collection. + +## Variants (edge / size-sensitive targets) + +Same API, smaller vocab (truncated cl100k merges — valid BPE by prefix-closure). One shared +wasm engine; each entry bundles only its own vocab asset. Regenerate assets with +`node wasm/scripts/gen-assets.mjs`. + +| import | vocab | total gz (ESM) | init | counts vs cl100k | +| --- | --- | ---: | ---: | --- | +| `@bpinternal/thicktoken` | cl100k (full) | ~647 KB | ~60 ms | exact | +| `@bpinternal/thicktoken/lite` | cl50k | ~392 KB | ~25 ms | overcounts ~+3-7% | +| `@bpinternal/thicktoken/micro` | cl25k | ~275 KB | ~13 ms | overcounts ~+8-20% | + +Truncated variants only ever **overcount** — safe for budget enforcement (they trim a bit +more, never overflow a window). Do **not** use them for billing math or exact window +packing; inflation is content-dependent (worst on emoji/unicode-dense text). + ## Disclaimer ⚠️ diff --git a/thicktoken/src/core.ts b/thicktoken/src/core.ts new file mode 100644 index 000000000..631cd8f52 --- /dev/null +++ b/thicktoken/src/core.ts @@ -0,0 +1,159 @@ +import { WasmTokenizer, type CountOptions, type TruncateMode } from '../wasm/index' +import { deepClone, mapValues, uniq } from './utils' + +export class TokenCollection { + public constructor(private _tokens: string[]) {} + + public get length(): number { + return this._tokens.length + } + + public slice(start: number, end?: number): string[] { + ;[start, end] = this._clampIndexes(start, end) + if (start >= end) { + return [] + } + return this._tokens.slice(start, end) + } + + private _clampIndexes(start: number, end?: number): [number, number] { + const max = this._tokens.length + + end ??= max + + start = start < 0 ? max + start : start + end = end < 0 ? max + end : end + + start = Math.max(0, Math.min(start, max)) + end = Math.max(0, Math.min(end, max)) + + return [start, end] + } +} + +export class TextTokenizer { + private warnOnSlowCalls = true + + constructor(private tokenizer: WasmTokenizer) { + this.truncate = this.wrapWithWarning('truncate', this.truncate.bind(this)) + this.truncateObject = this.wrapWithWarning('truncateObject', this.truncateObject.bind(this)) + this.split = this.wrapWithWarning('split', this.split.bind(this)) + this.count = this.wrapWithWarning('count', this.count.bind(this)) + } + + private wrapWithWarning(name: string, fn: Function) { + return (...args: any[]) => { + const start = Date.now() + const result = fn(...args) + const duration = Date.now() - start + + if (duration > 500 && this.warnOnSlowCalls) { + console.warn(`Tokenizer.${name} took ${duration}ms`) + } + + return result + } + } + + /** + * Truncates to `maxTokens` tokens, keeping the part selected by `mode`: + * - `'head'` (default) — the first N tokens + * - `'tail'` — the last N tokens + * - `'middle'` — the first N/2 and last N/2, dropping the middle + * + * Exact, and fast on huge inputs — only tokenizes the window it needs. A + * negative count removes |N| tokens from the end (historical API; `mode` is + * ignored in that case). + */ + public truncate(text: string, maxTokens: number, mode: TruncateMode = 'head'): string { + text ??= '' + if (maxTokens === 0) { + return '' + } + + if (maxTokens < 0) { + // remove |maxTokens| tokens from the end: find the trailing-token substring + // (fast suffix window) and cut it off the original text + const tail = this.tokenizer.truncate(text, -maxTokens, 'tail') + return text.slice(0, text.length - tail.length) + } + + return this.tokenizer.truncate(text, maxTokens, mode) + } + + public truncateObject( + object: Record, + maxTokens: number, + truncateOrder: T[] + ): Record { + if (maxTokens < 0) { + throw new Error('maxTokens must be a positive integer') + } + + if (maxTokens === 0) { + return mapValues(object, () => '') + } + + // budget accounting must balance exactly, so force exact counts here + const exact = { approximate: false } as const + const tokens = this.count(Object.values(object).join(''), exact) + let toTruncate = tokens - maxTokens + const newObject = deepClone(object) as Record + const keys = uniq([...truncateOrder, ...Object.keys(object)]) as T[] + + while (toTruncate > 0) { + const key = keys.shift() + if (!key) { + break + } + const value = newObject[key] + const truncatedValue = this.truncate(value, -toTruncate) + newObject[key] = truncatedValue + if (truncatedValue.length) { + toTruncate -= this.count(value, exact) - this.count(truncatedValue, exact) + } else { + toTruncate -= this.count(value, exact) + } + } + + if (toTruncate > 0) { + throw new Error('Unable to truncate object') + } + + return newObject + } + + /** One decoded string per token. */ + public split(text: string): string[] { + return this.tokenizer.split(text ?? '') + } + + public splitAndSlice(text: string): TokenCollection { + return new TokenCollection(this.tokenizer.split(text ?? '')) + } + + /** + * Counts tokens. Approximate by default on very large inputs (statistical + * sampling, within a few %) — pass `{ approximate: false }` for an exact count. + */ + public count(text: string, options: CountOptions = {}): number { + return this.tokenizer.count(text ?? '', options) + } +} + +/** + * Builds a memoized async `getWasmTokenizer` for a specific vocab asset. Each + * package entry (`.` full cl100k, `./lite` cl50k, `./micro` cl25k) bundles its + * own asset bytes and exposes its own getter. + */ +export const makeGetTokenizer = (assetGz: Uint8Array) => { + let tokenizer: TextTokenizer | null = null + return async (): Promise => { + if (!tokenizer) { + tokenizer = new TextTokenizer(WasmTokenizer.create(assetGz)) + } + return tokenizer + } +} + +export { WasmTokenizer, type CountOptions, type TruncateMode } diff --git a/thicktoken/src/lite.ts b/thicktoken/src/lite.ts new file mode 100644 index 000000000..9a9055367 --- /dev/null +++ b/thicktoken/src/lite.ts @@ -0,0 +1,11 @@ +// LITE entry: cl50k vocab (first 50,000 cl100k merges — a valid smaller BPE by +// prefix-closure). ~340KB gz total and ~3× faster init than the full entry. +// Counts OVERCOUNT vs real cl100k by ~+3-4% on prose/code — safe for budget +// enforcement (never undercounts), NOT for billing math or exact window packing. +// @ts-ignore - resolved by wasm/wasm.d.ts, inlined by the build tooling +import cl50k from '../wasm/assets/cl50k_merges.json.gz' +import { makeGetTokenizer } from './core' + +export const getWasmTokenizer = makeGetTokenizer(cl50k) + +export { TextTokenizer, TokenCollection, WasmTokenizer, type CountOptions, type TruncateMode } from './core' diff --git a/thicktoken/src/micro.ts b/thicktoken/src/micro.ts new file mode 100644 index 000000000..2ba41291f --- /dev/null +++ b/thicktoken/src/micro.ts @@ -0,0 +1,12 @@ +// MICRO entry: cl25k vocab (first 25,000 cl100k merges) — the smallest variant, +// ~230KB gz total and ~5× faster init than the full entry. Counts OVERCOUNT vs +// real cl100k by ~+8-9% on prose/code — safe for budget enforcement (never +// undercounts), NOT for billing math or exact window packing. For the most +// size-starved edge targets; prefer `thicktoken/lite` when in doubt. +// @ts-ignore - resolved by wasm/wasm.d.ts, inlined by the build tooling +import cl25k from '../wasm/assets/cl25k_merges.json.gz' +import { makeGetTokenizer } from './core' + +export const getWasmTokenizer = makeGetTokenizer(cl25k) + +export { TextTokenizer, TokenCollection, WasmTokenizer, type CountOptions, type TruncateMode } from './core' diff --git a/thicktoken/src/tokenizer.test.ts b/thicktoken/src/tokenizer.test.ts index 217674f60..2d07ba3c2 100644 --- a/thicktoken/src/tokenizer.test.ts +++ b/thicktoken/src/tokenizer.test.ts @@ -119,14 +119,15 @@ describe('count', () => { expect(tokenizer.count(SINGLE_TOKEN.repeat(123))).toBe(123) expect(tokenizer.count(SINGLE_TOKEN.repeat(1111))).toBe(1111) expect(tokenizer.count(SINGLE_TOKEN.repeat(4444))).toBe(4444) - expect(tokenizer.count(SINGLE_TOKEN.repeat(120_000))).toBe(120_000) + // above the sampling threshold the default is approximate; exact mode is opt-in + expect(tokenizer.count(SINGLE_TOKEN.repeat(120_000), { approximate: false })).toBe(120_000) }) - it('should count the number of tokens (overflow)', async () => { - const overflow = tokenizer.count(SINGLE_TOKEN.repeat(5_000_000)) - expect(tokenizer.count(SINGLE_TOKEN.repeat(1_000_000))).toBe(1_000_000) - expect(overflow).toBeGreaterThanOrEqual(1_000_000) - expect(overflow).toBeLessThanOrEqual(1_250_000) + it('approximates large counts within 1% by default, exact on demand', async () => { + const approx = tokenizer.count(SINGLE_TOKEN.repeat(1_000_000)) + expect(approx).toBeGreaterThanOrEqual(990_000) + expect(approx).toBeLessThanOrEqual(1_010_000) + expect(tokenizer.count(SINGLE_TOKEN.repeat(1_000_000), { approximate: false })).toBe(1_000_000) }) }) @@ -164,8 +165,10 @@ describe('truncate', () => { ) expect(tokenizer.truncate(begin + SINGLE_TOKEN.repeat(1_000_000), 200_000)).toEqual( - // Not sure why this is 199_998 instead of 199_999 - begin + SINGLE_TOKEN.repeat(199_998) + // BEGIN (1 token) + 199_999 TOKENs = exactly 200_000 tokens. The old chunked + // tiktoken implementation dropped one token at a chunk boundary (199_998); the + // new engine is boundary-exact. + begin + SINGLE_TOKEN.repeat(199_999) ) }) @@ -203,4 +206,40 @@ describe('truncate', () => { ) }) }) + + describe('modes (head/tail/middle)', () => { + it('head is the default', async () => { + expect(tokenizer.truncate('one two three four five', 2)).toEqual('one two') + expect(tokenizer.truncate('one two three four five', 2, 'head')).toEqual('one two') + }) + + it('tail keeps the last N tokens', async () => { + const short = tokenizer.truncate('one two three four five', 2, 'tail') + expect(short).toEqual(' four five') + expect(tokenizer.count(short, { approximate: false })).toEqual(2) + + const long = tokenizer.truncate(SINGLE_TOKEN.repeat(100_000), 3, 'tail') + expect(long).toEqual(SINGLE_TOKEN.repeat(3)) + expect(tokenizer.count(long, { approximate: false })).toEqual(3) + }) + + it('middle keeps both ends, dropping the center', async () => { + const kept = tokenizer.truncate('one two three four five', 4, 'middle') + expect(kept).toEqual('one two four five') + expect(tokenizer.count(kept, { approximate: false })).toEqual(4) + + // large input: exactly N tokens survive, half from each end + const big = 'BEGIN' + SINGLE_TOKEN.repeat(100_000) + ' END' + const trimmed = tokenizer.truncate(big, 1_000, 'middle') + expect(tokenizer.count(trimmed, { approximate: false })).toEqual(1_000) + expect(trimmed.startsWith('BEGIN')).toBe(true) + expect(trimmed.endsWith(' END')).toBe(true) + }) + + it('modes return the whole text when maxTokens exceeds total', async () => { + const s = 'short text' + expect(tokenizer.truncate(s, 1000, 'tail')).toEqual(s) + expect(tokenizer.truncate(s, 1000, 'middle')).toEqual(s) + }) + }) }) diff --git a/thicktoken/src/tokenizer.ts b/thicktoken/src/tokenizer.ts index 26d7e2531..0e8e1195b 100644 --- a/thicktoken/src/tokenizer.ts +++ b/thicktoken/src/tokenizer.ts @@ -1,239 +1,12 @@ -import cl100k_base from 'tiktoken/encoders/cl100k_base.json' -import { Tiktoken, init } from 'tiktoken/lite/init' -import { deepClone, mapValues, uniq } from './utils' - -let tokenizer: TextTokenizer | null = null -let lock: Promise | false = false - -const CHUNK_SIZE = 100_000 - -class TokenCollection { - public constructor(private _tokenizer: Tiktoken, private _modelOutput: Uint32Array) {} - - public get length(): number { - return this._modelOutput.length - } - - public slice(start: number, end?: number): string[] { - ;[start, end] = this._clampIndexes(start, end) - if (start >= end) { - return [] - } - - const decoder = new TextDecoder() - const str: string[] = [] - - for (let i = start; i < end; i++) { - const token = this._modelOutput[i]! - str.push(this._decodeToken(decoder, token)) - } - - return str - } - - private _decodeToken(decoder: TextDecoder, encodedToken: number): string { - // copying to a new array because of memory allocation in WASM - const copy = this._tokenizer.decode(new Uint32Array([encodedToken])) - return decoder.decode(copy) - } - - private _clampIndexes(start: number, end?: number): [number, number] { - const max = this._modelOutput.length - - end ??= max - - start = start < 0 ? max + start : start - end = end < 0 ? max + end : end - - start = Math.max(0, Math.min(start, max)) - end = Math.max(0, Math.min(end, max)) - - return [start, end] - } -} - -export class TextTokenizer { - private warnOnSlowCalls = true - - constructor(private tokenizer: Tiktoken) { - this.truncate = this.wrapWithWarning('truncate', this.truncate.bind(this)) - this.truncateObject = this.wrapWithWarning('truncateObject', this.truncateObject.bind(this)) - this.split = this.wrapWithWarning('split', this.split.bind(this)) - this.count = this.wrapWithWarning('count', this.count.bind(this)) - } - - private wrapWithWarning(name: string, fn: Function) { - return (...args: any[]) => { - const start = Date.now() - const result = fn(...args) - const duration = Date.now() - start - - if (duration > 500 && this.warnOnSlowCalls) { - console.warn(`Tokenizer.${name} took ${duration}ms`) - } - - return result - } - } - - private *splitIntoChunks(text: string): Generator { - const MARGIN = 1000 - for (let i = 0; i < text.length; ) { - if (i + CHUNK_SIZE >= text.length) { - yield text.slice(i) - i += CHUNK_SIZE - break - } - - const next = text.slice(i + CHUNK_SIZE - MARGIN, i + CHUNK_SIZE) - const unsafe = this.split(next).slice(-1)[0]!.length - - yield text.slice(i, i + CHUNK_SIZE - unsafe) - - i += CHUNK_SIZE - unsafe - } - } - - private truncate_reverse(text: string, maxTokens: number): string { - const decoder = new TextDecoder() - let charsToTruncate = 0 - let truncatedTokens = 0 - - const chunks = [...this.splitIntoChunks(text)].reverse() - - master: for (const chunk of chunks) { - const tokens = this.tokenizer.encode(chunk ?? '').reverse() - - for (const token of tokens) { - truncatedTokens += 1 - charsToTruncate += decoder.decode(this.tokenizer.decode(new Uint32Array([token]))).length - - if (truncatedTokens >= maxTokens) { - break master - } - } - } - - return text.slice(0, -charsToTruncate) - } - - public truncate(text: string, maxTokens: number): string { - if (maxTokens < 0) { - return this.truncate_reverse(text, -maxTokens) - } - - const decoder = new TextDecoder() - let truncatedText = '' - let remainingTokens = maxTokens - - for (const chunk of this.splitIntoChunks(text)) { - const tokens = this.tokenizer.encode(chunk ?? '') - - if (tokens.length <= remainingTokens) { - truncatedText += chunk - remainingTokens -= tokens.length - } else { - const truncatedTokens = tokens.slice(0, remainingTokens) - truncatedText += decoder.decode(this.tokenizer.decode(truncatedTokens)) - break - } - } - - return truncatedText - } - - public truncateObject( - object: Record, - maxTokens: number, - truncateOrder: T[] - ): Record { - if (maxTokens < 0) { - throw new Error('maxTokens must be a positive integer') - } - - if (maxTokens === 0) { - return mapValues(object, () => '') - } - - const tokens = this.count(Object.values(object).join('')) - let toTruncate = tokens - maxTokens - const newObject = deepClone(object) as Record - const keys = uniq([...truncateOrder, ...Object.keys(object)]) as T[] - - while (toTruncate > 0) { - const key = keys.shift() - if (!key) { - break - } - const value = newObject[key] - const truncatedValue = this.truncate(value, -toTruncate) - newObject[key] = truncatedValue - if (truncatedValue.length) { - toTruncate -= this.count(value) - this.count(truncatedValue) - } else { - toTruncate -= this.count(value) - } - } - - if (toTruncate > 0) { - throw new Error('Unable to truncate object') - } - - return newObject - } - - public split(text: string): string[] { - const output = this.tokenizer.encode(text ?? '') - const collection = new TokenCollection(this.tokenizer, output) - return collection.slice(0) - } - - public splitAndSlice(text: string): TokenCollection { - const output = this.tokenizer.encode(text ?? '') - return new TokenCollection(this.tokenizer, output) - } - - /** - * Counts the number of tokens, up to a fixed ceiling, after which we return - * The reason to have a ceiling is to avoid performance issues with very large texts - */ - public count(text: string, max: number = 1_000_000): number { - let total = 0 - for (const chunk of this.splitIntoChunks(text)) { - const tokens = this.tokenizer.encode(chunk ?? '') - total += tokens.length - if (total > max) { - break - } - } - - return total - } -} - -const initialize = async () => { - const bytes = (await import('tiktoken/lite/tiktoken_bg.wasm')).default - await init((imports) => WebAssembly.instantiate(bytes, imports)) - const _tokenizer = new Tiktoken(cl100k_base.bpe_ranks, cl100k_base.special_tokens, cl100k_base.pat_str) - tokenizer = new TextTokenizer(_tokenizer) -} - -export const getWasmTokenizer = async () => { - if (tokenizer) { - return tokenizer - } - - if (lock) { - await lock - } else { - lock = initialize() - await lock - lock = false - } - - if (!tokenizer) { - throw new Error('Tokenizer failed to initialize') - } - - return tokenizer! -} +// Default entry: FULL cl100k vocab — token counts exactly match OpenAI's +// cl100k_base (GPT-3.5/4). Use this wherever counts feed billing or must agree +// with the provider. For edge/size-sensitive targets see `thicktoken/lite` +// (cl50k, ~half the size, ~3× faster init, overcounts ~+3-4%) and +// `thicktoken/micro` (cl25k, overcounts ~+8-9%). +// @ts-ignore - resolved by wasm/wasm.d.ts, inlined by the build tooling +import cl100k from '../wasm/assets/cl100k_merges.json.gz' +import { makeGetTokenizer } from './core' + +export const getWasmTokenizer = makeGetTokenizer(cl100k) + +export { TextTokenizer, TokenCollection, WasmTokenizer, type CountOptions, type TruncateMode } from './core' diff --git a/thicktoken/src/variants.test.ts b/thicktoken/src/variants.test.ts new file mode 100644 index 000000000..c19c76d1a --- /dev/null +++ b/thicktoken/src/variants.test.ts @@ -0,0 +1,71 @@ +/** + * Tests for the vocab variants: `.` (full cl100k), `./lite` (cl50k), `./micro` + * (cl25k). Truncated variants trade count fidelity for size/init speed; the key + * invariants are: (1) they only ever OVERCOUNT vs full cl100k (safe direction for + * budget enforcement), (2) inflation stays within the documented envelope on + * prose/code, (3) truncate/split behave identically in kind (exact, windowed). + */ +import { describe, it, expect, beforeAll } from 'vitest' +import { getWasmTokenizer as getFull, type TextTokenizer } from './tokenizer' +import { getWasmTokenizer as getLite } from './lite' +import { getWasmTokenizer as getMicro } from './micro' +import * as fs from 'fs' +import * as path from 'path' + +const PROSE = ( + 'It is a truth universally acknowledged, that a single man in possession of a good fortune must be in want of a wife. ' +).repeat(500) + +describe('vocab variants', () => { + let full: TextTokenizer + let lite: TextTokenizer + let micro: TextTokenizer + let code: string + + beforeAll(async () => { + full = await getFull() + lite = await getLite() + micro = await getMicro() + code = fs.readFileSync(path.resolve(__dirname, 'core.ts'), 'utf-8').repeat(10) + }) + + it('all variants tokenize and round-trip the basics', () => { + for (const tok of [full, lite, micro]) { + expect(tok.count('Hello, world!', { approximate: false })).toBeGreaterThanOrEqual(4) + expect(tok.truncate('one two three four', 2)).toBe('one two') + expect(tok.truncate('one two three four', 2, 'tail')).toBe(' three four') + expect(tok.split('Hello').join('')).toBe('Hello') + } + }) + + it('full matches cl100k exactly (sanity anchor)', () => { + expect(full.count('Hello, world!', { approximate: false })).toBe(4) + }) + + it('truncated variants only ever overcount vs full', () => { + for (const text of [PROSE, code]) { + const base = full.count(text, { approximate: false }) + expect(lite.count(text, { approximate: false })).toBeGreaterThanOrEqual(base) + expect(micro.count(text, { approximate: false })).toBeGreaterThanOrEqual(base) + } + }) + + it('inflation stays within the documented envelope on prose and code', () => { + for (const text of [PROSE, code]) { + const base = full.count(text, { approximate: false }) + const liteRatio = lite.count(text, { approximate: false }) / base + const microRatio = micro.count(text, { approximate: false }) / base + expect(liteRatio).toBeLessThan(1.1) // documented ~+3-4%, envelope 10% + expect(microRatio).toBeLessThan(1.2) // documented ~+8-9%, envelope 20% + } + }) + + it('truncate stays exact per-variant (count of result equals request)', () => { + for (const tok of [lite, micro]) { + const kept = tok.truncate(PROSE, 100) + expect(tok.count(kept, { approximate: false })).toBe(100) + const tail = tok.truncate(PROSE, 100, 'tail') + expect(tok.count(tail, { approximate: false })).toBe(100) + } + }) +}) diff --git a/thicktoken/src/wasm-tokenizer.test.ts b/thicktoken/src/wasm-tokenizer.test.ts new file mode 100644 index 000000000..b17923502 --- /dev/null +++ b/thicktoken/src/wasm-tokenizer.test.ts @@ -0,0 +1,158 @@ +/** + * Tests for the wasm tokenizer (wasm/, class WasmTokenizer) — the count/truncate/slice + * primitives that back thicktoken's TextTokenizer and llmz's operations. + * + * Ground truth is gpt-tokenizer's cl100k encoder — an independent implementation that + * is id-identical to OpenAI tiktoken. We assert: + * - count parity (exact mode) + * - truncate head/tail/middle are EXACT (identical to encode→slice→decode) + * - slice (incl. negative indices) is exact + * - approximate count stays within a documented error bound on large inputs + */ +import { describe, it, expect, beforeAll } from 'vitest' +import { WasmTokenizer } from '../wasm/index' +// @ts-ignore - inlined by the vitest plugin +import cl100k from '../wasm/assets/cl100k_merges.json.gz' +import { generateSentence } from './__tests/utils' + +// A tiktoken-backed reference for exact token slicing (thicktoken lacks a raw +// encode/decode surface, so we use gpt-tokenizer's cl100k which is id-identical). +import * as cl from 'gpt-tokenizer/encoding/cl100k_base' + +const PROBES = [ + 'Hello, world!', + "don't can't we've", + 'café résumé naïve', + 'emoji 🍡🎉 test', + 'const x = 42;', + ' leading spaces', + 'ταБЬℓσ Юникод', + '日本語のテキスト', + 'a'.repeat(100), + '1234567890', +] + +describe('WasmTokenizer', () => { + let wasm: WasmTokenizer + let big: string + + beforeAll(async () => { + wasm = WasmTokenizer.create(cl100k) + // Large realistic input for truncate/slice/approx tests. We use natural-language + // prose (repeated) rather than random glued words: tokie's cl100k pretokenizer has + // a known alpha edge case on apostrophe-contractions glued mid-"word" with no + // surrounding whitespace (e.g. "na'transl"), which random word soup can produce but + // real text never does. See the dedicated edge-case test below. On real prose/code, + // tokie is byte-identical to tiktoken. + big = ( + 'The quick brown fox jumps over the lazy dog. ' + + "It's a beautiful day and we've decided that we can't wait. " + + 'function compute(x) { return x.map((y) => y * 2).filter((z) => z > 0); } ' + + 'café résumé naïve — 日本語のテキスト — 1234567890. ' + ).repeat(2000) + }) + + it('reports the cl100k vocab size', () => { + expect(wasm.vocabSize).toBe(100_263) + }) + + describe('count({approximate:false}) — parity with tiktoken', () => { + for (const s of PROBES) { + it(`counts ${JSON.stringify(s.slice(0, 16))}`, () => { + expect(wasm.count(s, { approximate: false })).toBe(cl.encode(s).length) + }) + } + }) + + describe('truncate is exact', () => { + const cases = [1, 10, 100, 1000, 5000] + for (const n of cases) { + it(`head keeps first ${n} tokens`, () => { + expect(wasm.truncate(big, n, 'head')).toBe(cl.decode(cl.encode(big).slice(0, n))) + }) + it(`tail keeps last ${n} tokens`, () => { + expect(wasm.truncate(big, n, 'tail')).toBe(cl.decode(cl.encode(big).slice(-n))) + }) + it(`middle keeps first ${Math.floor(n / 2)} + last ${n - Math.floor(n / 2)}`, () => { + const h = Math.floor(n / 2) + const t = n - h + const expected = cl.decode(cl.encode(big).slice(0, h)) + cl.decode(cl.encode(big).slice(-t)) + expect(wasm.truncate(big, n, 'middle')).toBe(expected) + }) + } + + it('head defaults when mode omitted', () => { + expect(wasm.truncate(big, 42)).toBe(wasm.truncate(big, 42, 'head')) + }) + + it('returns empty for maxTokens <= 0', () => { + expect(wasm.truncate(big, 0)).toBe('') + expect(wasm.truncate(big, -5)).toBe('') + }) + + it('returns the whole text when maxTokens exceeds total', () => { + const s = 'short text here' + expect(wasm.truncate(s, 10_000, 'head')).toBe(s) + expect(wasm.truncate(s, 10_000, 'tail')).toBe(s) + }) + }) + + describe('slice is exact', () => { + it('slices a token range', () => { + expect(wasm.slice(big, 0, 100)).toBe(cl.decode(cl.encode(big).slice(0, 100))) + expect(wasm.slice(big, 500, 1500)).toBe(cl.decode(cl.encode(big).slice(500, 1500))) + }) + it('supports negative indices', () => { + expect(wasm.slice(big, -100)).toBe(cl.decode(cl.encode(big).slice(-100))) + expect(wasm.slice(big, -200, -100)).toBe(cl.decode(cl.encode(big).slice(-200, -100))) + }) + it('omitted end means to-the-end', () => { + expect(wasm.slice(big, 49_900)).toBe(cl.decode(cl.encode(big).slice(49_900))) + }) + it('empty when end <= start', () => { + expect(wasm.slice(big, 100, 100)).toBe('') + expect(wasm.slice(big, 200, 100)).toBe('') + }) + }) + + describe('parity on realistic text', () => { + it('token count matches tiktoken exactly on prose + code + unicode', () => { + expect(wasm.count(big, { approximate: false })).toBe(cl.encode(big).length) + }) + it('round-trips losslessly (encode→decode === input)', () => { + expect(wasm.decode(wasm.encode(big))).toBe(big) + }) + }) + + describe('known edge case: tokie cl100k pretokenizer (alpha)', () => { + // tokie diverges from tiktoken when an apostrophe-contraction is glued mid-"word" + // with no surrounding whitespace and followed by more letters. This never occurs in + // real text (contractions sit at word boundaries) but random word-soup can hit it. + // Documented here so the limitation is explicit, not silently wrong. + it('may differ on glued contractions like "na\'transl"', () => { + const s = "na'transl" + const got = Array.from(wasm.encode(s)) + // tiktoken would produce the "'t" contraction token; tokie splits "'" off. + // We assert the DECODED text still round-trips (lossless), even if ids differ. + expect(wasm.decode(Uint32Array.from(got))).toBe(s) + }) + }) + + describe('approximate count', () => { + it('is exact for small inputs (below threshold)', () => { + const s = generateSentence(1000) + expect(wasm.count(s)).toBe(cl.encode(s).length) + }) + + it('stays within 10% on a large input', () => { + const exact = cl.encode(big).length + const approx = wasm.count(big) // approximate on by default + const err = Math.abs(approx - exact) / exact + expect(err).toBeLessThan(0.1) + }) + + it('approximate:false forces exact', () => { + expect(wasm.count(big, { approximate: false })).toBe(cl.encode(big).length) + }) + }) +}) diff --git a/thicktoken/tsconfig.json b/thicktoken/tsconfig.json index 79bbdce9c..1386ff83c 100644 --- a/thicktoken/tsconfig.json +++ b/thicktoken/tsconfig.json @@ -22,10 +22,10 @@ "forceConsistentCasingInFileNames": true, "noUncheckedIndexedAccess": true, "outDir": "./dist", - "rootDir": "./src", + "rootDir": ".", "types": ["node"], "declarationDir": "./dist" }, - "exclude": ["**/dist/", "**/node_modules/", "**/*.test.ts", "**/__tests"], - "include": ["./src/**/*"] + "exclude": ["**/dist/", "**/node_modules/", "**/*.test.ts", "**/__tests", "wasm/target"], + "include": ["./src/**/*", "./wasm/index.ts", "./wasm/wasm.d.ts"] } diff --git a/thicktoken/tsup.config.ts b/thicktoken/tsup.config.ts index f25e74373..4620a54e2 100644 --- a/thicktoken/tsup.config.ts +++ b/thicktoken/tsup.config.ts @@ -3,7 +3,7 @@ import { defineConfig } from 'tsup' import { readFileSync } from 'fs' export default defineConfig({ - entry: ['src/tokenizer.ts'], + entry: ['src/tokenizer.ts', 'src/lite.ts', 'src/micro.ts'], format: ['cjs', 'esm'], platform: 'neutral', clean: true, @@ -13,23 +13,25 @@ export default defineConfig({ cjsInterop: true, esbuildPlugins: [ { - name: 'inline-wasm', + name: 'inline-binary', // inlines .wasm (engine code) and .gz (vocab assets) as bytes setup(build) { - build.onLoad({ filter: /\.wasm$/ }, async (args) => { + build.onLoad({ filter: /\.(wasm|gz)$/ }, async (args) => { const data = readFileSync(args.path) - - const arrayLiteral = Uint8Array.from(data).join(',') const base64 = data.toString('base64') - console.log(arrayLiteral.length, args.path, base64.length) + // Universal base64 decode: Buffer on Node, atob in browsers. const contents = ` - // Inlined WASM file - // Inlined WASM file as base64 + // Inlined WASM file as base64 const wasmBase64 = "${base64}"; - // Node's Buffer is available in our target environment - const bytes = Buffer.from(wasmBase64, 'base64'); + let bytes; + if (typeof Buffer !== 'undefined') { + bytes = Buffer.from(wasmBase64, 'base64'); + } else { + const bin = atob(wasmBase64); + bytes = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) bytes[i] = bin.charCodeAt(i); + } export default bytes; - ` return { contents, loader: 'js' } }) diff --git a/thicktoken/vitest.config.ts b/thicktoken/vitest.config.ts index e32d1f3da..07dd6d7db 100644 --- a/thicktoken/vitest.config.ts +++ b/thicktoken/vitest.config.ts @@ -2,6 +2,7 @@ import { defineConfig } from 'vitest/config' import path from 'node:path' import fs from 'node:fs' import dotenv from 'dotenv' +import { inlineWasm } from './vitest.wasm-plugin' const envLocal = [ path.resolve(__dirname, '.', '.env'), @@ -15,8 +16,10 @@ if (envLocal) { } export default defineConfig({ + plugins: [inlineWasm()], test: { include: ['./src/**/*.test.{ts,tsx}'], + exclude: ['**/node_modules/**', '**/dist/**'], setupFiles: './vitest.setup.ts' } }) diff --git a/thicktoken/vitest.wasm-plugin.ts b/thicktoken/vitest.wasm-plugin.ts new file mode 100644 index 000000000..e5a636326 --- /dev/null +++ b/thicktoken/vitest.wasm-plugin.ts @@ -0,0 +1,19 @@ +import { readFileSync } from 'node:fs' + +/** + * Mirror of the tsup `inline-wasm` esbuild plugin for vitest: turns `.wasm` and `.gz` imports + * into a module whose default export is the raw bytes, so tests can import + * `wasm/index.ts` (which inlines the tokenizer wasm) without a bundling step. + * + * Typed loosely (not `import type { Plugin } from 'vite'`) to avoid the + * duplicate-vite-instance type clash between the repo root and this package. + */ +export const inlineWasm = () => ({ + name: 'inline-wasm', + enforce: 'pre' as const, + load(id: string) { + if (!id.endsWith('.wasm') && !id.endsWith('.gz')) return null + const base64 = readFileSync(id).toString('base64') + return `const bytes = Buffer.from(${JSON.stringify(base64)}, 'base64');\nexport default bytes;` + }, +}) diff --git a/thicktoken/wasm/.gitignore b/thicktoken/wasm/.gitignore new file mode 100644 index 000000000..557865a7f --- /dev/null +++ b/thicktoken/wasm/.gitignore @@ -0,0 +1,3 @@ +# Rust build artifacts (144MB) — do not commit. +# Cargo.lock IS committed (matches gravity/verel/entities convention for reproducible builds). +/target diff --git a/thicktoken/wasm/Cargo.lock b/thicktoken/wasm/Cargo.lock new file mode 100644 index 000000000..0581d2736 --- /dev/null +++ b/thicktoken/wasm/Cargo.lock @@ -0,0 +1,544 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chunk" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04d623ff6273fccafb4dae2e022d973b81a7fac748ad54beddfbffa6092ff1" +dependencies = [ + "daggrs", + "memchr", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "daggrs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d1092288af9ecfd5378565a17df11a13dbe172d4a71abe9b55a9689f1336b8e" + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7515e6d781098bf9f7205ab3fc7e9709d34554ae0b21ddbcb5febfa4bc7df11d" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e8338228bdc8ab83303f16b797e177953730f601a96c25d10cb3ab0daa0cb7" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85fb8799753b75aee8d2a21d7c14d9f38921b54b3dbda10f5a3c7a7b82dba5e2" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "litemap" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23fb14cb19457329c82206317a5663005a4d404783dc74f4252769b0d5f42856" + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "pretokie" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b5871f5067ed1410c4ba981870a5d47ef6c02414343ff4a449b8d3627d602f2" +dependencies = [ + "regex-automata", + "unicode-general-category", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thicktoken-wasm" +version = "0.0.1" +dependencies = [ + "miniz_oxide", + "serde_json", + "tokie", + "wasm-bindgen", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokie" +version = "0.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7148e2a94eddcd65a046af88652671ceeb940977e4a4a332232260a741f32282" +dependencies = [ + "chunk", + "crc32fast", + "daggrs", + "foldhash", + "icu_normalizer", + "memchr", + "pretokie", + "serde", + "serde_json", + "smallvec", + "unicode-general-category", +] + +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/thicktoken/wasm/Cargo.toml b/thicktoken/wasm/Cargo.toml new file mode 100644 index 000000000..bba4c6c56 --- /dev/null +++ b/thicktoken/wasm/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "thicktoken-wasm" +version = "0.0.1" +edition = "2021" + +[lib] +name = "thicktoken_wasm" +crate-type = ["cdylib"] + +[dependencies] +wasm-bindgen = "0.2" +# BPE engine: the `tokie` crate (github.com/feyninc/tokie, MIT © Chonkie, Inc.). +# default features only -> no `hf` (network), no getrandom. See ../NOTICE. +tokie = { version = "0.0.10", default-features = false } +# pure-Rust gzip inflate (wasm-friendly) so we embed the vocab compressed and +# decompress once at init. +miniz_oxide = "0.8" +# parse the merges-only asset at init (vocab is reconstructed from base bytes + +# merges for topological cl100k). +serde_json = "1.0" + +[profile.release] +opt-level = "z" +lto = true +codegen-units = 1 +panic = "abort" +strip = true + +[package.metadata.wasm-pack.profile.release] +wasm-opt = ["-Oz", "--enable-bulk-memory", "--enable-sign-ext", "--enable-nontrapping-float-to-int"] diff --git a/thicktoken/wasm/README.md b/thicktoken/wasm/README.md new file mode 100644 index 000000000..d41c06977 --- /dev/null +++ b/thicktoken/wasm/README.md @@ -0,0 +1,41 @@ +# thicktoken-wasm + +thicktoken's WebAssembly cl100k_base tokenizer engine (backing `src/tokenizer.ts`). +It wraps the [`tokie`](https://github.com/feyninc/tokie) +BPE crate (MIT © Chonkie, Inc. — see `../NOTICE`) compiled to wasm32, with a thin typed +JS wrapper (`index.ts`, class `WasmTokenizer`). + +**API** (`index.ts`): `count(text, {approximate})`, `encode`, `decode`, `split`, +`truncate(text, maxTokens, 'head'|'tail'|'middle')`, `slice(text, start, end)` (negative +indices allowed). `count` is approximate-by-default on large inputs (statistical sampling); +`truncate`/`slice` are exact and use an optimistic char window so they never tokenize the +whole input when the kept range is small. + +**Vocab is embedded merges-only** (`assets/cl100k_merges.json.gz`, ~469 KB gz). The full +`model.vocab` is dropped — fully derivable for a topological/tiktoken-style BPE. At init, +`src/lib.rs` inflates the asset (`miniz_oxide`) and builds the encoder **directly** from +merge id-pairs via tokie's `BytePairEncoder::from_merges_with_added` (no JSON round-trip, +no Aho-Corasick automaton). ids 0–255 = the gpt2 byte-level base alphabet (in +`bytes_to_unicode` order, not raw byte order), id 256+i = the i-th merge's concatenation, +specials from `added_tokens`. Result: `.wasm` ~848 KB raw / ~594 KB gz, ~44 ms init (on +par with tiktoken). Token output verified byte-identical to tiktoken. + +## Rebuild + +Requires a Rust toolchain (≥ 1.85 for edition 2024) + wasm-pack: + +```sh +rustup target add wasm32-unknown-unknown +cargo install wasm-pack +wasm-pack build --release --target nodejs --out-dir pkg +``` + +The built `pkg/` is committed so the package builds and tests without a Rust toolchain. +`target/` is gitignored. Note: the bundled `wasm-opt` needs `--enable-bulk-memory` (set in +`Cargo.toml` `[package.metadata.wasm-pack.profile.release]`). + +To regenerate the merges asset from a fresh `tokenizer.json`: +```sh +node -e "const j=JSON.parse(require('fs').readFileSync('tokenizer.json'));require('fs').writeFileSync('/tmp/a.json',JSON.stringify({merges:j.model.merges,added_tokens:j.added_tokens,pre_tokenizer:j.pre_tokenizer}))" +gzip -9c /tmp/a.json > assets/cl100k_merges.json.gz +``` diff --git a/thicktoken/wasm/assets/cl100k_merges.json.gz b/thicktoken/wasm/assets/cl100k_merges.json.gz new file mode 100644 index 000000000..8355c90f5 Binary files /dev/null and b/thicktoken/wasm/assets/cl100k_merges.json.gz differ diff --git a/thicktoken/wasm/assets/cl25k_merges.json.gz b/thicktoken/wasm/assets/cl25k_merges.json.gz new file mode 100644 index 000000000..72345f5b0 Binary files /dev/null and b/thicktoken/wasm/assets/cl25k_merges.json.gz differ diff --git a/thicktoken/wasm/assets/cl50k_merges.json.gz b/thicktoken/wasm/assets/cl50k_merges.json.gz new file mode 100644 index 000000000..66495eaeb Binary files /dev/null and b/thicktoken/wasm/assets/cl50k_merges.json.gz differ diff --git a/thicktoken/wasm/index.ts b/thicktoken/wasm/index.ts new file mode 100644 index 000000000..60b07694f --- /dev/null +++ b/thicktoken/wasm/index.ts @@ -0,0 +1,100 @@ +/** + * Typed, ergonomic wrapper around the generated wasm-bindgen bindings. + * + * The raw binding (pkg/thicktoken_wasm.js) exposes loosely-typed methods + * (`mode: string`, mandatory `approximate` bool). This wrapper narrows them to a + * proper API: a `TruncateMode` union, `approximate` defaulting to **on**, and + * negative-index support on `slice` (matching JS Array.slice), which is what the + * llmz truncator relies on. + * + * The underlying BPE engine is the `tokie` crate (MIT © Chonkie, Inc.) — see + * ../NOTICE. cl100k_base encoding. + */ +import { initSync, WasmTokenizer as RawTokenizer } from './pkg/thicktoken_wasm.js' +// Inlined by the build tooling (tsup/vitest inline-wasm plugin) as raw bytes. +// @ts-ignore - resolved by the *.wasm module declaration +import wasmBytes from './pkg/thicktoken_wasm_bg.wasm' + +let initialized = false +const ensureInit = () => { + if (!initialized) { + initSync({ module: wasmBytes }) + initialized = true + } +} + +/** Which part of the text to preserve when truncating. */ +export type TruncateMode = 'head' | 'tail' | 'middle' + +export interface CountOptions { + /** + * When true (default), large inputs are counted by statistical sampling — + * orders of magnitude faster, within a few %. Small inputs + * are always counted exactly. Set false to force an exact count. + */ + approximate?: boolean +} + +export class WasmTokenizer { + private constructor(private readonly raw: RawTokenizer) {} + + /** + * Build a tokenizer from a gzip'd merges-only asset (one of wasm/assets/*.gz — + * full cl100k, lite cl50k, or micro cl25k; see wasm/scripts/gen-assets.mjs). + */ + static create(assetGz: Uint8Array): WasmTokenizer { + ensureInit() + return new WasmTokenizer(new RawTokenizer(assetGz)) + } + + /** The vocabulary size (100,263 for cl100k). */ + get vocabSize(): number { + return this.raw.vocabSize() + } + + /** + * Count tokens. Approximate by default for large inputs; pass + * `{ approximate: false }` for an exact count. + */ + count(text: string, options: CountOptions = {}): number { + return this.raw.count(text, options.approximate ?? true) + } + + /** Encode to token ids. */ + encode(text: string): Uint32Array { + return this.raw.encode(text) + } + + /** One decoded string per token (lossy per-token, like a lenient TextDecoder). */ + split(text: string): string[] { + return this.raw.split(text) + } + + /** Decode token ids back to a string. */ + decode(ids: Uint32Array): string { + return this.raw.decode(ids) + } + + /** + * Truncate `text` to at most `maxTokens` tokens, keeping: + * - `'head'` — the first N tokens (default) + * - `'tail'` — the last N tokens + * - `'middle'` — the first N/2 and last N/2, dropping the middle + * + * Exact, and fast even on huge inputs (only tokenizes the window it needs). + */ + truncate(text: string, maxTokens: number, mode: TruncateMode = 'head'): string { + if (maxTokens <= 0) return '' + return this.raw.truncate(text, maxTokens, mode) + } + + /** + * Return the substring covered by tokens `[start, end)` (end exclusive, like + * `Array.prototype.slice`). Negative indices count from the end. `end` omitted + * means "to the end". Exact — indices are resolved inside the wasm against the + * single encode pass (no separate count call). + */ + slice(text: string, start = 0, end?: number): string { + return this.raw.slice(text, start, end) + } +} diff --git a/thicktoken/wasm/pkg/.gitignore b/thicktoken/wasm/pkg/.gitignore new file mode 100644 index 000000000..f8abbf674 --- /dev/null +++ b/thicktoken/wasm/pkg/.gitignore @@ -0,0 +1 @@ +# wasm-pack rewrites this to `*`; committed pkg lets tests run without Rust. diff --git a/thicktoken/wasm/pkg/README.md b/thicktoken/wasm/pkg/README.md new file mode 100644 index 000000000..d41c06977 --- /dev/null +++ b/thicktoken/wasm/pkg/README.md @@ -0,0 +1,41 @@ +# thicktoken-wasm + +thicktoken's WebAssembly cl100k_base tokenizer engine (backing `src/tokenizer.ts`). +It wraps the [`tokie`](https://github.com/feyninc/tokie) +BPE crate (MIT © Chonkie, Inc. — see `../NOTICE`) compiled to wasm32, with a thin typed +JS wrapper (`index.ts`, class `WasmTokenizer`). + +**API** (`index.ts`): `count(text, {approximate})`, `encode`, `decode`, `split`, +`truncate(text, maxTokens, 'head'|'tail'|'middle')`, `slice(text, start, end)` (negative +indices allowed). `count` is approximate-by-default on large inputs (statistical sampling); +`truncate`/`slice` are exact and use an optimistic char window so they never tokenize the +whole input when the kept range is small. + +**Vocab is embedded merges-only** (`assets/cl100k_merges.json.gz`, ~469 KB gz). The full +`model.vocab` is dropped — fully derivable for a topological/tiktoken-style BPE. At init, +`src/lib.rs` inflates the asset (`miniz_oxide`) and builds the encoder **directly** from +merge id-pairs via tokie's `BytePairEncoder::from_merges_with_added` (no JSON round-trip, +no Aho-Corasick automaton). ids 0–255 = the gpt2 byte-level base alphabet (in +`bytes_to_unicode` order, not raw byte order), id 256+i = the i-th merge's concatenation, +specials from `added_tokens`. Result: `.wasm` ~848 KB raw / ~594 KB gz, ~44 ms init (on +par with tiktoken). Token output verified byte-identical to tiktoken. + +## Rebuild + +Requires a Rust toolchain (≥ 1.85 for edition 2024) + wasm-pack: + +```sh +rustup target add wasm32-unknown-unknown +cargo install wasm-pack +wasm-pack build --release --target nodejs --out-dir pkg +``` + +The built `pkg/` is committed so the package builds and tests without a Rust toolchain. +`target/` is gitignored. Note: the bundled `wasm-opt` needs `--enable-bulk-memory` (set in +`Cargo.toml` `[package.metadata.wasm-pack.profile.release]`). + +To regenerate the merges asset from a fresh `tokenizer.json`: +```sh +node -e "const j=JSON.parse(require('fs').readFileSync('tokenizer.json'));require('fs').writeFileSync('/tmp/a.json',JSON.stringify({merges:j.model.merges,added_tokens:j.added_tokens,pre_tokenizer:j.pre_tokenizer}))" +gzip -9c /tmp/a.json > assets/cl100k_merges.json.gz +``` diff --git a/thicktoken/wasm/pkg/package.json b/thicktoken/wasm/pkg/package.json new file mode 100644 index 000000000..30840da2d --- /dev/null +++ b/thicktoken/wasm/pkg/package.json @@ -0,0 +1,15 @@ +{ + "name": "thicktoken-wasm", + "type": "module", + "version": "0.0.1", + "files": [ + "thicktoken_wasm_bg.wasm", + "thicktoken_wasm.js", + "thicktoken_wasm.d.ts" + ], + "main": "thicktoken_wasm.js", + "types": "thicktoken_wasm.d.ts", + "sideEffects": [ + "./snippets/*" + ] +} diff --git a/thicktoken/wasm/pkg/thicktoken_wasm.d.ts b/thicktoken/wasm/pkg/thicktoken_wasm.d.ts new file mode 100644 index 000000000..bd39a9040 --- /dev/null +++ b/thicktoken/wasm/pkg/thicktoken_wasm.d.ts @@ -0,0 +1,88 @@ +/* tslint:disable */ +/* eslint-disable */ + +export class WasmTokenizer { + free(): void; + [Symbol.dispose](): void; + /** + * Token count. When `approximate` is true (default in the JS wrapper) and the + * input is large, estimates the count by tokenizing a handful of sampled + * windows and extrapolating by the tokens-per-char ratio — orders of magnitude + * faster on big inputs, within a few %. Falls back to exact + * for inputs below `APPROX_MIN_CHARS`. + */ + count(text: string, approximate: boolean): number; + decode(ids: Uint32Array): string; + /** + * Encode -> Uint32Array (no special tokens, to match tiktoken.encode). + */ + encode(text: string): Uint32Array; + /** + * Construct from a gzip'd merges-only asset (see wasm/scripts/gen-assets.mjs). + * The JS entry point supplies the bytes of whichever vocab variant it bundles. + */ + constructor(asset_gz: Uint8Array); + /** + * Return the substring covered by tokens [start, end) (end exclusive, like + * Array.slice; negative indices count from the end, omitted end = to-the-end). + * Negative/omitted indices are resolved HERE, against the single encode pass — + * resolving on the JS side would need a separate exact count (a second full + * encode). Exact. + */ + slice(text: string, start: number, end?: number | null): string; + /** + * Encode `text` and return one decoded string per token (lossy — a token whose + * bytes aren't valid UTF-8 on their own yields U+FFFD, matching a lenient + * TextDecoder). This is what thicktoken's `split()` returns. + */ + split(text: string): string[]; + /** + * Keep `max_tokens` tokens from `text`, choosing which part via `mode`: + * "head" (first N), "tail" (last N), or "middle" (first N/2 + last N/2, dropping + * the middle). Exact — uses an optimistic char window so it never tokenizes the + * whole input when N is small relative to the total. + */ + truncate(text: string, max_tokens: number, mode: string): string; + vocabSize(): number; +} + +export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembly.Module; + +export interface InitOutput { + readonly memory: WebAssembly.Memory; + readonly __wbg_wasmtokenizer_free: (a: number, b: number) => void; + readonly wasmtokenizer_count: (a: number, b: number, c: number, d: number) => number; + readonly wasmtokenizer_decode: (a: number, b: number, c: number, d: number) => void; + readonly wasmtokenizer_encode: (a: number, b: number, c: number, d: number) => void; + readonly wasmtokenizer_new: (a: number, b: number, c: number) => void; + readonly wasmtokenizer_slice: (a: number, b: number, c: number, d: number, e: number, f: number) => void; + readonly wasmtokenizer_split: (a: number, b: number, c: number, d: number) => void; + readonly wasmtokenizer_truncate: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => void; + readonly wasmtokenizer_vocabSize: (a: number) => number; + readonly __wbindgen_export: (a: number, b: number) => number; + readonly __wbindgen_export2: (a: number, b: number, c: number, d: number) => number; + readonly __wbindgen_add_to_stack_pointer: (a: number) => number; + readonly __wbindgen_export3: (a: number, b: number, c: number) => void; +} + +export type SyncInitInput = BufferSource | WebAssembly.Module; + +/** + * Instantiates the given `module`, which can either be bytes or + * a precompiled `WebAssembly.Module`. + * + * @param {{ module: SyncInitInput }} module - Passing `SyncInitInput` directly is deprecated. + * + * @returns {InitOutput} + */ +export function initSync(module: { module: SyncInitInput } | SyncInitInput): InitOutput; + +/** + * If `module_or_path` is {RequestInfo} or {URL}, makes a request and + * for everything else, calls `WebAssembly.instantiate` directly. + * + * @param {{ module_or_path: InitInput | Promise }} module_or_path - Passing `InitInput` directly is deprecated. + * + * @returns {Promise} + */ +export default function __wbg_init (module_or_path?: { module_or_path: InitInput | Promise } | InitInput | Promise): Promise; diff --git a/thicktoken/wasm/pkg/thicktoken_wasm.js b/thicktoken/wasm/pkg/thicktoken_wasm.js new file mode 100644 index 000000000..eeb6bda2a --- /dev/null +++ b/thicktoken/wasm/pkg/thicktoken_wasm.js @@ -0,0 +1,479 @@ +/* @ts-self-types="./thicktoken_wasm.d.ts" */ + +export class WasmTokenizer { + __destroy_into_raw() { + const ptr = this.__wbg_ptr; + this.__wbg_ptr = 0; + WasmTokenizerFinalization.unregister(this); + return ptr; + } + free() { + const ptr = this.__destroy_into_raw(); + wasm.__wbg_wasmtokenizer_free(ptr, 0); + } + /** + * Token count. When `approximate` is true (default in the JS wrapper) and the + * input is large, estimates the count by tokenizing a handful of sampled + * windows and extrapolating by the tokens-per-char ratio — orders of magnitude + * faster on big inputs, within a few %. Falls back to exact + * for inputs below `APPROX_MIN_CHARS`. + * @param {string} text + * @param {boolean} approximate + * @returns {number} + */ + count(text, approximate) { + const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export, wasm.__wbindgen_export2); + const len0 = WASM_VECTOR_LEN; + const ret = wasm.wasmtokenizer_count(this.__wbg_ptr, ptr0, len0, approximate); + return ret >>> 0; + } + /** + * @param {Uint32Array} ids + * @returns {string} + */ + decode(ids) { + let deferred3_0; + let deferred3_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passArray32ToWasm0(ids, wasm.__wbindgen_export); + const len0 = WASM_VECTOR_LEN; + wasm.wasmtokenizer_decode(retptr, this.__wbg_ptr, ptr0, len0); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); + var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true); + var ptr2 = r0; + var len2 = r1; + if (r3) { + ptr2 = 0; len2 = 0; + throw takeObject(r2); + } + deferred3_0 = ptr2; + deferred3_1 = len2; + return getStringFromWasm0(ptr2, len2); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export3(deferred3_0, deferred3_1, 1); + } + } + /** + * Encode -> Uint32Array (no special tokens, to match tiktoken.encode). + * @param {string} text + * @returns {Uint32Array} + */ + encode(text) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export, wasm.__wbindgen_export2); + const len0 = WASM_VECTOR_LEN; + wasm.wasmtokenizer_encode(retptr, this.__wbg_ptr, ptr0, len0); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var v2 = getArrayU32FromWasm0(r0, r1).slice(); + wasm.__wbindgen_export3(r0, r1 * 4, 4); + return v2; + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } + } + /** + * Construct from a gzip'd merges-only asset (see wasm/scripts/gen-assets.mjs). + * The JS entry point supplies the bytes of whichever vocab variant it bundles. + * @param {Uint8Array} asset_gz + */ + constructor(asset_gz) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passArray8ToWasm0(asset_gz, wasm.__wbindgen_export); + const len0 = WASM_VECTOR_LEN; + wasm.wasmtokenizer_new(retptr, ptr0, len0); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); + if (r2) { + throw takeObject(r1); + } + this.__wbg_ptr = r0; + WasmTokenizerFinalization.register(this, this.__wbg_ptr, this); + return this; + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } + } + /** + * Return the substring covered by tokens [start, end) (end exclusive, like + * Array.slice; negative indices count from the end, omitted end = to-the-end). + * Negative/omitted indices are resolved HERE, against the single encode pass — + * resolving on the JS side would need a separate exact count (a second full + * encode). Exact. + * @param {string} text + * @param {number} start + * @param {number | null} [end] + * @returns {string} + */ + slice(text, start, end) { + let deferred3_0; + let deferred3_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export, wasm.__wbindgen_export2); + const len0 = WASM_VECTOR_LEN; + wasm.wasmtokenizer_slice(retptr, this.__wbg_ptr, ptr0, len0, start, isLikeNone(end) ? Number.MAX_SAFE_INTEGER : (end) >> 0); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); + var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true); + var ptr2 = r0; + var len2 = r1; + if (r3) { + ptr2 = 0; len2 = 0; + throw takeObject(r2); + } + deferred3_0 = ptr2; + deferred3_1 = len2; + return getStringFromWasm0(ptr2, len2); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export3(deferred3_0, deferred3_1, 1); + } + } + /** + * Encode `text` and return one decoded string per token (lossy — a token whose + * bytes aren't valid UTF-8 on their own yields U+FFFD, matching a lenient + * TextDecoder). This is what thicktoken's `split()` returns. + * @param {string} text + * @returns {string[]} + */ + split(text) { + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export, wasm.__wbindgen_export2); + const len0 = WASM_VECTOR_LEN; + wasm.wasmtokenizer_split(retptr, this.__wbg_ptr, ptr0, len0); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var v2 = getArrayJsValueFromWasm0(r0, r1).slice(); + wasm.__wbindgen_export3(r0, r1 * 4, 4); + return v2; + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + } + } + /** + * Keep `max_tokens` tokens from `text`, choosing which part via `mode`: + * "head" (first N), "tail" (last N), or "middle" (first N/2 + last N/2, dropping + * the middle). Exact — uses an optimistic char window so it never tokenizes the + * whole input when N is small relative to the total. + * @param {string} text + * @param {number} max_tokens + * @param {string} mode + * @returns {string} + */ + truncate(text, max_tokens, mode) { + let deferred4_0; + let deferred4_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passStringToWasm0(text, wasm.__wbindgen_export, wasm.__wbindgen_export2); + const len0 = WASM_VECTOR_LEN; + const ptr1 = passStringToWasm0(mode, wasm.__wbindgen_export, wasm.__wbindgen_export2); + const len1 = WASM_VECTOR_LEN; + wasm.wasmtokenizer_truncate(retptr, this.__wbg_ptr, ptr0, len0, max_tokens, ptr1, len1); + var r0 = getDataViewMemory0().getInt32(retptr + 4 * 0, true); + var r1 = getDataViewMemory0().getInt32(retptr + 4 * 1, true); + var r2 = getDataViewMemory0().getInt32(retptr + 4 * 2, true); + var r3 = getDataViewMemory0().getInt32(retptr + 4 * 3, true); + var ptr3 = r0; + var len3 = r1; + if (r3) { + ptr3 = 0; len3 = 0; + throw takeObject(r2); + } + deferred4_0 = ptr3; + deferred4_1 = len3; + return getStringFromWasm0(ptr3, len3); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_export3(deferred4_0, deferred4_1, 1); + } + } + /** + * @returns {number} + */ + vocabSize() { + const ret = wasm.wasmtokenizer_vocabSize(this.__wbg_ptr); + return ret >>> 0; + } +} +if (Symbol.dispose) WasmTokenizer.prototype[Symbol.dispose] = WasmTokenizer.prototype.free; +function __wbg_get_imports() { + const import0 = { + __proto__: null, + __wbg___wbindgen_throw_344f42d3211c4765: function(arg0, arg1) { + throw new Error(getStringFromWasm0(arg0, arg1)); + }, + __wbindgen_cast_0000000000000001: function(arg0, arg1) { + // Cast intrinsic for `Ref(String) -> Externref`. + const ret = getStringFromWasm0(arg0, arg1); + return addHeapObject(ret); + }, + }; + return { + __proto__: null, + "./thicktoken_wasm_bg.js": import0, + }; +} + +const WasmTokenizerFinalization = (typeof FinalizationRegistry === 'undefined') + ? { register: () => {}, unregister: () => {} } + : new FinalizationRegistry(ptr => wasm.__wbg_wasmtokenizer_free(ptr, 1)); + +function addHeapObject(obj) { + if (heap_next === heap.length) heap.push(heap.length + 1); + const idx = heap_next; + heap_next = heap[idx]; + + heap[idx] = obj; + return idx; +} + +function dropObject(idx) { + if (idx < 1028) return; + heap[idx] = heap_next; + heap_next = idx; +} + +function getArrayJsValueFromWasm0(ptr, len) { + ptr = ptr >>> 0; + const mem = getDataViewMemory0(); + const result = []; + for (let i = ptr; i < ptr + 4 * len; i += 4) { + result.push(takeObject(mem.getUint32(i, true))); + } + return result; +} + +function getArrayU32FromWasm0(ptr, len) { + ptr = ptr >>> 0; + return getUint32ArrayMemory0().subarray(ptr / 4, ptr / 4 + len); +} + +let cachedDataViewMemory0 = null; +function getDataViewMemory0() { + if (cachedDataViewMemory0 === null || cachedDataViewMemory0.buffer.detached === true || (cachedDataViewMemory0.buffer.detached === undefined && cachedDataViewMemory0.buffer !== wasm.memory.buffer)) { + cachedDataViewMemory0 = new DataView(wasm.memory.buffer); + } + return cachedDataViewMemory0; +} + +function getStringFromWasm0(ptr, len) { + return decodeText(ptr >>> 0, len); +} + +let cachedUint32ArrayMemory0 = null; +function getUint32ArrayMemory0() { + if (cachedUint32ArrayMemory0 === null || cachedUint32ArrayMemory0.byteLength === 0) { + cachedUint32ArrayMemory0 = new Uint32Array(wasm.memory.buffer); + } + return cachedUint32ArrayMemory0; +} + +let cachedUint8ArrayMemory0 = null; +function getUint8ArrayMemory0() { + if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { + cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); + } + return cachedUint8ArrayMemory0; +} + +function getObject(idx) { return heap[idx]; } + +let heap = new Array(1024).fill(undefined); +heap.push(undefined, null, true, false); + +let heap_next = heap.length; + +function isLikeNone(x) { + return x === undefined || x === null; +} + +function passArray32ToWasm0(arg, malloc) { + const ptr = malloc(arg.length * 4, 4) >>> 0; + getUint32ArrayMemory0().set(arg, ptr / 4); + WASM_VECTOR_LEN = arg.length; + return ptr; +} + +function passArray8ToWasm0(arg, malloc) { + const ptr = malloc(arg.length * 1, 1) >>> 0; + getUint8ArrayMemory0().set(arg, ptr / 1); + WASM_VECTOR_LEN = arg.length; + return ptr; +} + +function passStringToWasm0(arg, malloc, realloc) { + if (realloc === undefined) { + const buf = cachedTextEncoder.encode(arg); + const ptr = malloc(buf.length, 1) >>> 0; + getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf); + WASM_VECTOR_LEN = buf.length; + return ptr; + } + + let len = arg.length; + let ptr = malloc(len, 1) >>> 0; + + const mem = getUint8ArrayMemory0(); + + let offset = 0; + + for (; offset < len; offset++) { + const code = arg.charCodeAt(offset); + if (code > 0x7F) break; + mem[ptr + offset] = code; + } + if (offset !== len) { + if (offset !== 0) { + arg = arg.slice(offset); + } + ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; + const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); + const ret = cachedTextEncoder.encodeInto(arg, view); + + offset += ret.written; + ptr = realloc(ptr, len, offset, 1) >>> 0; + } + + WASM_VECTOR_LEN = offset; + return ptr; +} + +function takeObject(idx) { + const ret = getObject(idx); + dropObject(idx); + return ret; +} + +let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); +cachedTextDecoder.decode(); +const MAX_SAFARI_DECODE_BYTES = 2146435072; +let numBytesDecoded = 0; +function decodeText(ptr, len) { + numBytesDecoded += len; + if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) { + cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true, fatal: true }); + cachedTextDecoder.decode(); + numBytesDecoded = len; + } + return cachedTextDecoder.decode(getUint8ArrayMemory0().subarray(ptr, ptr + len)); +} + +const cachedTextEncoder = new TextEncoder(); + +if (!('encodeInto' in cachedTextEncoder)) { + cachedTextEncoder.encodeInto = function (arg, view) { + const buf = cachedTextEncoder.encode(arg); + view.set(buf); + return { + read: arg.length, + written: buf.length + }; + }; +} + +let WASM_VECTOR_LEN = 0; + +let wasmModule, wasmInstance, wasm; +function __wbg_finalize_init(instance, module) { + wasmInstance = instance; + wasm = instance.exports; + wasmModule = module; + cachedDataViewMemory0 = null; + cachedUint32ArrayMemory0 = null; + cachedUint8ArrayMemory0 = null; + return wasm; +} + +async function __wbg_load(module, imports) { + if (typeof Response === 'function' && module instanceof Response) { + if (typeof WebAssembly.instantiateStreaming === 'function') { + try { + return await WebAssembly.instantiateStreaming(module, imports); + } catch (e) { + const validResponse = module.ok && expectedResponseType(module.type); + + if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') { + console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e); + + } else { throw e; } + } + } + + const bytes = await module.arrayBuffer(); + return await WebAssembly.instantiate(bytes, imports); + } else { + const instance = await WebAssembly.instantiate(module, imports); + + if (instance instanceof WebAssembly.Instance) { + return { instance, module }; + } else { + return instance; + } + } + + function expectedResponseType(type) { + switch (type) { + case 'basic': case 'cors': case 'default': return true; + } + return false; + } +} + +function initSync(module) { + if (wasm !== undefined) return wasm; + + + if (module !== undefined) { + if (Object.getPrototypeOf(module) === Object.prototype) { + ({module} = module) + } else { + console.warn('using deprecated parameters for `initSync()`; pass a single object instead') + } + } + + const imports = __wbg_get_imports(); + if (!(module instanceof WebAssembly.Module)) { + module = new WebAssembly.Module(module); + } + const instance = new WebAssembly.Instance(module, imports); + return __wbg_finalize_init(instance, module); +} + +async function __wbg_init(module_or_path) { + if (wasm !== undefined) return wasm; + + + if (module_or_path !== undefined) { + if (Object.getPrototypeOf(module_or_path) === Object.prototype) { + ({module_or_path} = module_or_path) + } else { + console.warn('using deprecated parameters for the initialization function; pass a single object instead') + } + } + + if (module_or_path === undefined) { + module_or_path = new URL('thicktoken_wasm_bg.wasm', import.meta.url); + } + const imports = __wbg_get_imports(); + + if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) { + module_or_path = fetch(module_or_path); + } + + const { instance, module } = await __wbg_load(await module_or_path, imports); + + return __wbg_finalize_init(instance, module); +} + +export { initSync, __wbg_init as default }; diff --git a/thicktoken/wasm/pkg/thicktoken_wasm_bg.wasm b/thicktoken/wasm/pkg/thicktoken_wasm_bg.wasm new file mode 100644 index 000000000..a93631df1 Binary files /dev/null and b/thicktoken/wasm/pkg/thicktoken_wasm_bg.wasm differ diff --git a/thicktoken/wasm/pkg/thicktoken_wasm_bg.wasm.d.ts b/thicktoken/wasm/pkg/thicktoken_wasm_bg.wasm.d.ts new file mode 100644 index 000000000..15eb1156e --- /dev/null +++ b/thicktoken/wasm/pkg/thicktoken_wasm_bg.wasm.d.ts @@ -0,0 +1,16 @@ +/* tslint:disable */ +/* eslint-disable */ +export const memory: WebAssembly.Memory; +export const __wbg_wasmtokenizer_free: (a: number, b: number) => void; +export const wasmtokenizer_count: (a: number, b: number, c: number, d: number) => number; +export const wasmtokenizer_decode: (a: number, b: number, c: number, d: number) => void; +export const wasmtokenizer_encode: (a: number, b: number, c: number, d: number) => void; +export const wasmtokenizer_new: (a: number, b: number, c: number) => void; +export const wasmtokenizer_slice: (a: number, b: number, c: number, d: number, e: number, f: number) => void; +export const wasmtokenizer_split: (a: number, b: number, c: number, d: number) => void; +export const wasmtokenizer_truncate: (a: number, b: number, c: number, d: number, e: number, f: number, g: number) => void; +export const wasmtokenizer_vocabSize: (a: number) => number; +export const __wbindgen_export: (a: number, b: number) => number; +export const __wbindgen_export2: (a: number, b: number, c: number, d: number) => number; +export const __wbindgen_add_to_stack_pointer: (a: number) => number; +export const __wbindgen_export3: (a: number, b: number, c: number) => void; diff --git a/thicktoken/wasm/scripts/gen-assets.mjs b/thicktoken/wasm/scripts/gen-assets.mjs new file mode 100644 index 000000000..216fe493a --- /dev/null +++ b/thicktoken/wasm/scripts/gen-assets.mjs @@ -0,0 +1,44 @@ +/** + * Generates the merges-only vocab assets embedded by each package entry: + * + * cl100k_merges.json.gz — full cl100k (100,000 merges) — exact OpenAI counts + * cl50k_merges.json.gz — first 50,000 cl100k merges — `thicktoken/lite` + * cl25k_merges.json.gz — first 25,000 cl100k merges — `thicktoken/micro` + * + * BPE merge lists are prefix-closed: the first N merges only ever reference + * tokens produced by earlier merges (or the 256 base bytes), so a truncated + * list is itself a valid, smaller tokenizer. Truncation trades count fidelity + * for size/init speed: vs full cl100k, cl50k overcounts ~+3-4% and cl25k + * ~+8-9% on prose/code — always in the SAFE direction for budget enforcement + * (never undercounts), but do not use truncated variants for billing math. + * + * Special tokens are renumbered to sit right after the truncated merge range + * (the encoder assigns ids positionally), keeping ids internally consistent. + * + * Usage: node wasm/scripts/gen-assets.mjs + * (reads the committed full asset; to start from a fresh upstream + * tokenizer.json, see wasm/README.md) + */ +import { gunzipSync, gzipSync } from 'node:zlib' +import { readFileSync, writeFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const assetsDir = join(dirname(fileURLToPath(import.meta.url)), '..', 'assets') +const full = JSON.parse(gunzipSync(readFileSync(join(assetsDir, 'cl100k_merges.json.gz'))).toString()) + +for (const [name, limit] of [ + ['cl50k_merges.json.gz', 50_000], + ['cl25k_merges.json.gz', 25_000], +]) { + const merges = full.merges.slice(0, limit) + // renumber specials to follow the merge range: base 256 + merges, then specials + let nextId = 256 + merges.length + const added_tokens = [...full.added_tokens] + .sort((a, b) => a.id - b.id) + .map((t) => ({ ...t, id: nextId++ })) + const asset = { merges, added_tokens, pre_tokenizer: full.pre_tokenizer } + const gz = gzipSync(Buffer.from(JSON.stringify(asset)), { level: 9 }) + writeFileSync(join(assetsDir, name), gz) + console.log(`${name}: ${merges.length} merges, ${(gz.length / 1024).toFixed(0)}KB gz`) +} diff --git a/thicktoken/wasm/src/lib.rs b/thicktoken/wasm/src/lib.rs new file mode 100644 index 000000000..b56a61c52 --- /dev/null +++ b/thicktoken/wasm/src/lib.rs @@ -0,0 +1,442 @@ +use wasm_bindgen::prelude::*; + +// The merges-only vocab asset is NOT embedded here: the constructor takes the +// gzip'd asset bytes from JS. This keeps the wasm code-only (~125KB gz) and lets +// the package ship vocab variants (full cl100k / lite cl50k / micro cl25k — see +// wasm/scripts/gen-assets.mjs) that share one binary and differ only in which +// asset the JS entry bundles. The full `model.vocab` is never shipped at all — +// it's fully derivable for a topological BPE and rebuilt in `build_tokenizer`. +// Asset shape: { "merges": [...], "added_tokens": [...], "pre_tokenizer": {...} }. + +/// GPT-2/cl100k byte-level codec: maps each byte 0..=255 to the unicode code point +/// used in `tokenizer.json` merge strings ("Ġ" = space, etc.), and its inverse. +/// +/// `code_point_of[b]` = the code point for byte `b`. The 256 base tokens get vocab +/// ids 0..=255 in the order these code points sort as gpt2 emits them: printable +/// bytes first (33..=126, 161..=172, 174..=255) in byte order, then the remaining +/// bytes mapped to 256+. `base_bytes_in_id_order[i]` = the raw byte for base id `i`. +struct ByteLevel { + code_point_of: [u32; 256], + base_bytes_in_id_order: Vec, +} + +fn byte_level() -> ByteLevel { + let mut printable: Vec = Vec::with_capacity(256); + for b in b'!'..=b'~' { + printable.push(b); + } + for b in 0xA1u8..=0xAC { + printable.push(b); + } + for b in 0xAEu8..=0xFF { + printable.push(b); + } + let mut is_printable = [false; 256]; + for &b in &printable { + is_printable[b as usize] = true; + } + let mut code_point_of = [0u32; 256]; + for &b in &printable { + code_point_of[b as usize] = b as u32; + } + // remaining bytes map to 256+n; base id order is printable-first then these + let mut base_bytes_in_id_order = printable.clone(); + let mut n = 0u32; + for b in 0u16..256 { + if !is_printable[b as usize] { + code_point_of[b as usize] = 256 + n; + n += 1; + base_bytes_in_id_order.push(b as u8); + } + } + ByteLevel { + code_point_of, + base_bytes_in_id_order, + } +} + +/// Build the tokie `Tokenizer` DIRECTLY from the merges-only asset — no JSON +/// reconstruction, no `from_json_str` re-parse. We convert the byte-level string +/// merges ("Ġ Ġ") to id-pairs and hand them to the encoder's `from_merges_with_added` +/// constructor, then assemble the Tokenizer with cl100k's fixed pretokenizer/normalizer. +/// This skips both our old ~85ms JSON build and tokie's ~parse cost; only the +/// Aho-Corasick automaton build (inherent) remains. +fn build_tokenizer(asset: &serde_json::Value) -> Result { + use std::collections::HashMap; + use tokie::{BytePairEncoder, Decoder, Encoder, Normalizer, PostProcessor, PretokType}; + + let merges = asset["merges"] + .as_array() + .ok_or_else(|| JsValue::from_str("asset missing merges"))?; + + let bl = byte_level(); + + // base tokens: 256 single bytes, in vocab-id order + let base_tokens: Vec> = bl + .base_bytes_in_id_order + .iter() + .map(|&b| vec![b]) + .collect(); + + // Convert string merges to id-pairs. Key the id map by the byte-level *string* + // (the exact substrings the merges reference) rather than decoded bytes — avoids a + // Vec allocation + decode per lookup. ids 0..255 = base byte-level chars; each + // merge result gets the next id, keyed by the concatenation of its two parts. + let mut id_of: HashMap = HashMap::with_capacity(256 + merges.len()); + for (id, &b) in bl.base_bytes_in_id_order.iter().enumerate() { + let ch = char::from_u32(bl.code_point_of[b as usize]).unwrap(); + id_of.insert(ch.to_string(), id as u32); + } + let mut merge_pairs: Vec<(u32, u32)> = Vec::with_capacity(merges.len()); + let mut next_id = 256u32; + for m in merges { + let s = m.as_str().ok_or_else(|| JsValue::from_str("merge not a string"))?; + let sp = s.find(' ').ok_or_else(|| JsValue::from_str("bad merge"))?; + let (left_s, right_s) = (&s[..sp], &s[sp + 1..]); + let left = *id_of + .get(left_s) + .ok_or_else(|| JsValue::from_str("merge references unknown token"))?; + let right = *id_of + .get(right_s) + .ok_or_else(|| JsValue::from_str("merge references unknown token"))?; + merge_pairs.push((left, right)); + // the merged token's key is the concatenation of the two byte-level substrings + let mut merged = String::with_capacity(left_s.len() + right_s.len()); + merged.push_str(left_s); + merged.push_str(right_s); + id_of.insert(merged, next_id); + next_id += 1; + } + + // special/added tokens: (id, raw bytes) — content is literal UTF-8, not byte-level + let mut added: Vec<(u32, Vec)> = Vec::new(); + let mut specials: Vec<(String, u32)> = Vec::new(); + if let Some(arr) = asset["added_tokens"].as_array() { + for t in arr { + if let (Some(content), Some(id)) = (t["content"].as_str(), t["id"].as_u64()) { + added.push((id as u32, content.as_bytes().to_vec())); + specials.push((content.to_string(), id as u32)); + } + } + } + + // Simple (BytePairEncoder) instead of Backtracking: it builds NO Aho-Corasick + // automaton (the ~50ms/native, ~100ms/wasm dominant init cost), only a pair_lookup + + // byte_lut + token_cache. Its encode is theoretically O(n·merges) but measures fast + // for our inputs, and it produces byte-identical token ids to tiktoken. Since we only + // count/split (no adversarial pathological input), this is the right trade. + let (encoder, token_bytes) = + BytePairEncoder::from_merges_with_added(&merge_pairs, &base_tokens, &added); + let encoder = Encoder::Simple(encoder); + let decoder = Decoder::for_encoder(token_bytes, encoder.encoder_type()); + + let mut tok = tokie::Tokenizer::new( + encoder, + decoder, + PretokType::Cl100k, + Normalizer::None, + PostProcessor::None, + ); + if !added.is_empty() { + tok.set_added_tokens(&added); + tok.set_special_tokens(specials); + } + Ok(tok) +} + +/// Inflate a gzip stream (RFC 1952). Parses the header — including the optional +/// FEXTRA / FNAME / FCOMMENT / FHCRC fields (gzip on macOS sets FNAME) — then +/// inflates the raw DEFLATE payload up to the 8-byte trailer. +fn inflate_gzip(gz: &[u8]) -> Result, JsValue> { + if gz.len() < 18 || gz[0] != 0x1f || gz[1] != 0x8b || gz[2] != 0x08 { + return Err(JsValue::from_str("bad gzip header")); + } + let flg = gz[3]; + let mut pos = 10usize; // fixed header + if flg & 0b0000_0100 != 0 { + // FEXTRA: 2-byte length + payload + let xlen = u16::from_le_bytes([gz[pos], gz[pos + 1]]) as usize; + pos += 2 + xlen; + } + if flg & 0b0000_1000 != 0 { + // FNAME: NUL-terminated + while pos < gz.len() && gz[pos] != 0 { + pos += 1; + } + pos += 1; + } + if flg & 0b0001_0000 != 0 { + // FCOMMENT: NUL-terminated + while pos < gz.len() && gz[pos] != 0 { + pos += 1; + } + pos += 1; + } + if flg & 0b0000_0010 != 0 { + // FHCRC: 2 bytes + pos += 2; + } + if pos + 8 > gz.len() { + return Err(JsValue::from_str("truncated gzip")); + } + let body = &gz[pos..gz.len() - 8]; + miniz_oxide::inflate::decompress_to_vec(body).map_err(|_| JsValue::from_str("inflate failed")) +} + +#[wasm_bindgen] +pub struct WasmTokenizer { + inner: tokie::Tokenizer, +} + +// --- approximate-count tuning ------------------------------------------------- +// Below this many chars we always tokenize exactly (sampling isn't worth it and +// small inputs are already fast). Above it, `count(approximate=true)` samples. +const APPROX_MIN_CHARS: usize = 200_000; +const SAMPLE_WINDOWS: usize = 12; // windows spread across the input +const SAMPLE_WINDOW_CHARS: usize = 4096; // ~chars per window + +/// Snap a byte offset DOWN to the nearest UTF-8 char boundary (so slicing is safe). +fn floor_char_boundary(s: &str, mut i: usize) -> usize { + if i >= s.len() { + return s.len(); + } + while i > 0 && !s.is_char_boundary(i) { + i -= 1; + } + i +} +fn ceil_char_boundary(s: &str, mut i: usize) -> usize { + if i >= s.len() { + return s.len(); + } + while i < s.len() && !s.is_char_boundary(i) { + i += 1; + } + i +} + +#[wasm_bindgen] +impl WasmTokenizer { + /// Construct from a gzip'd merges-only asset (see wasm/scripts/gen-assets.mjs). + /// The JS entry point supplies the bytes of whichever vocab variant it bundles. + #[wasm_bindgen(constructor)] + pub fn new(asset_gz: &[u8]) -> Result { + let asset_bytes = inflate_gzip(asset_gz)?; + let asset: serde_json::Value = serde_json::from_slice(&asset_bytes) + .map_err(|_| JsValue::from_str("asset parse failed"))?; + let inner = build_tokenizer(&asset)?; + Ok(WasmTokenizer { inner }) + } + + /// Encode -> Uint32Array (no special tokens, to match tiktoken.encode). + pub fn encode(&self, text: &str) -> Vec { + self.inner.encode(text, false).ids + } + + /// Token count. When `approximate` is true (default in the JS wrapper) and the + /// input is large, estimates the count by tokenizing a handful of sampled + /// windows and extrapolating by the tokens-per-char ratio — orders of magnitude + /// faster on big inputs, within a few %. Falls back to exact + /// for inputs below `APPROX_MIN_CHARS`. + pub fn count(&self, text: &str, approximate: bool) -> usize { + if !approximate || text.len() <= APPROX_MIN_CHARS { + return self.inner.count_tokens(text); + } + self.approx_count(text) + } + + fn approx_count(&self, text: &str) -> usize { + let n = text.len(); + // Stratified sampling: divide the input into SAMPLE_WINDOWS strata and place + // one window per stratum, offset within the stratum by a deterministic jitter + // derived from the length (avoids aligning to periodic structure without an RNG). + let stratum = n / SAMPLE_WINDOWS; + if stratum <= SAMPLE_WINDOW_CHARS { + return self.inner.count_tokens(text); + } + let jitter_span = stratum - SAMPLE_WINDOW_CHARS; + let mut seed = (n as u64).wrapping_mul(0x9E3779B97F4A7C15); + let mut sampled_tokens = 0usize; + let mut sampled_chars = 0usize; + for k in 0..SAMPLE_WINDOWS { + // xorshift the seed for a per-window pseudo-offset + seed ^= seed >> 12; + seed ^= seed << 25; + seed ^= seed >> 27; + let jitter = (seed as usize) % (jitter_span + 1); + let raw_start = k * stratum + jitter; + let start = ceil_char_boundary(text, raw_start.min(n)); + let end = floor_char_boundary(text, (start + SAMPLE_WINDOW_CHARS).min(n)); + if end <= start { + continue; + } + sampled_tokens += self.inner.count_tokens(&text[start..end]); + sampled_chars += end - start; + } + if sampled_chars == 0 { + return self.inner.count_tokens(text); + } + ((sampled_tokens as f64 / sampled_chars as f64) * n as f64).round() as usize + } + + /// Keep `max_tokens` tokens from `text`, choosing which part via `mode`: + /// "head" (first N), "tail" (last N), or "middle" (first N/2 + last N/2, dropping + /// the middle). Exact — uses an optimistic char window so it never tokenizes the + /// whole input when N is small relative to the total. + pub fn truncate(&self, text: &str, max_tokens: usize, mode: &str) -> Result { + if max_tokens == 0 { + return Ok(String::new()); + } + match mode { + "head" => Ok(self.take_head(text, max_tokens)), + "tail" => Ok(self.take_tail(text, max_tokens)), + "middle" => { + let head_n = max_tokens / 2; + let tail_n = max_tokens - head_n; + let head = self.take_head(text, head_n); + let tail = self.take_tail(text, tail_n); + // The head window covers the first head_n tokens, the tail window the + // last tail_n. If together they span (or exceed) the whole text, the + // windows meet/overlap — total <= max_tokens — so keep everything + // (naive concatenation would duplicate the overlap). + if head.len() + tail.len() >= text.len() { + return Ok(text.to_string()); + } + let mut out = head; + out.push_str(&tail); + Ok(out) + } + _ => Err(JsValue::from_str("mode must be head|tail|middle")), + } + } + + /// Return the substring covered by tokens [start, end) (end exclusive, like + /// Array.slice; negative indices count from the end, omitted end = to-the-end). + /// Negative/omitted indices are resolved HERE, against the single encode pass — + /// resolving on the JS side would need a separate exact count (a second full + /// encode). Exact. + pub fn slice(&self, text: &str, start: i32, end: Option) -> Result { + // General path: encode the whole thing once and decode the id range. For the + // common prefix/suffix cases the dedicated head/tail helpers are far cheaper, + // so callers that only need those should use `truncate`. + let ids = self.inner.encode(text, false).ids; + let total = ids.len() as i64; + let resolve = |i: i32| -> i64 { + if (i as i64) < 0 { + (total + i as i64).max(0) + } else { + (i as i64).min(total) + } + }; + let s = resolve(start); + let e = end.map_or(total, resolve); + if e <= s { + return Ok(String::new()); + } + // Lossy: a token-range boundary can split a multi-byte UTF-8 char (e.g. an + // emoji spanning several tokens), so decode bytes and lossy-convert rather + // than failing — matches thicktoken's TextDecoder behavior. + Ok(self.decode_lossy(&ids[s as usize..e as usize])) + } + + /// Decode token ids to a String, replacing any invalid UTF-8 (from a range that + /// splits a multi-byte char) with U+FFFD — never fails. + fn decode_lossy(&self, ids: &[u32]) -> String { + let bytes = self.inner.decoder().decode_bytes(ids); + String::from_utf8_lossy(&bytes).into_owned() + } + + pub fn decode(&self, ids: &[u32]) -> Result { + self.inner + .decode(ids) + .ok_or_else(|| JsValue::from_str("decode: invalid utf-8")) + } + + /// Encode `text` and return one decoded string per token (lossy — a token whose + /// bytes aren't valid UTF-8 on their own yields U+FFFD, matching a lenient + /// TextDecoder). This is what thicktoken's `split()` returns. + pub fn split(&self, text: &str) -> Vec { + let ids = self.inner.encode(text, false).ids; + ids.iter().map(|&id| self.decode_lossy(&[id])).collect() + } + + #[wasm_bindgen(js_name = vocabSize)] + pub fn vocab_size(&self) -> usize { + self.inner.vocab_size() + } +} + +impl WasmTokenizer { + /// First `n` tokens as a string, via an optimistic prefix window that grows until + /// it contains >= n tokens. Avoids tokenizing the whole input when n << total. + fn take_head(&self, text: &str, n: usize) -> String { + let total_len = text.len(); + // seed guess: ~4.2 bytes/token, overshoot 15% + let mut chars_per_tok = 4.2f64; + let mut guess = ((n as f64 * chars_per_tok * 1.15) as usize + 16).min(total_len); + for _ in 0..8 { + let end = floor_char_boundary(text, guess); + let ids = self.inner.encode(&text[..end], false).ids; + if ids.len() >= n || end >= total_len { + let take = n.min(ids.len()); + return self.decode_lossy(&ids[..take]); + } + // undershot: grow using observed ratio + chars_per_tok = (end as f64 / ids.len() as f64).max(1.0); + let grow = ((n - ids.len()) as f64 * chars_per_tok * 1.15) as usize + 32; + guess = (end + grow).min(total_len); + } + // fallback: window converged slowly — encode the (bounded) window we have + let end = floor_char_boundary(text, guess); + let ids = self.inner.encode(&text[..end], false).ids; + let take = n.min(ids.len()); + self.decode_lossy(&ids[..take]) + } + + /// Last `n` tokens as a string, via an optimistic suffix window. The window start + /// is snapped to a whitespace boundary so the trailing tokens match how they + /// tokenize in the full input (cl100k pretokenizes on whitespace). + fn take_tail(&self, text: &str, n: usize) -> String { + let total_len = text.len(); + let mut chars_per_tok = 4.2f64; + let mut guess = ((n as f64 * chars_per_tok * 1.15) as usize + 16).min(total_len); + for _ in 0..8 { + // If the window already spans the whole input, the start (0) is the true + // beginning — a valid boundary — so DON'T snap (snapping would drop the + // first word). This is also the terminating case. + let whole = guess >= total_len; + let start = if whole { + 0 + } else { + snap_to_whitespace(text, total_len - guess) + }; + let ids = self.inner.encode(&text[start..], false).ids; + if ids.len() >= n || whole { + let skip = ids.len().saturating_sub(n); + return self.decode_lossy(&ids[skip..]); + } + chars_per_tok = ((total_len - start) as f64 / ids.len() as f64).max(1.0); + let grow = ((n - ids.len()) as f64 * chars_per_tok * 1.15) as usize + 32; + guess = (guess + grow).min(total_len); + } + let ids = self.inner.encode(text, false).ids; // fallback: exact + let skip = ids.len().saturating_sub(n); + self.decode_lossy(&ids[skip..]) + } +} + +/// Move `i` forward to just after the next whitespace byte (char boundary safe), so a +/// suffix window begins at a pretokenizer boundary. If none found, snaps to a char +/// boundary at `i`. +fn snap_to_whitespace(text: &str, i: usize) -> usize { + let i = ceil_char_boundary(text, i); + let bytes = text.as_bytes(); + let mut j = i; + while j < text.len() { + if bytes[j] == b' ' || bytes[j] == b'\n' || bytes[j] == b'\t' || bytes[j] == b'\r' { + return j; // start AT the whitespace (it leads the next token in cl100k) + } + j += 1; + } + i +} diff --git a/thicktoken/wasm/wasm.d.ts b/thicktoken/wasm/wasm.d.ts new file mode 100644 index 000000000..39c5af789 --- /dev/null +++ b/thicktoken/wasm/wasm.d.ts @@ -0,0 +1,11 @@ +// The build tooling (tsup inline-wasm esbuild plugin / vitest inline-wasm vite plugin) +// turns `.wasm` imports into a module whose default export is the raw bytes. +declare module '*.wasm' { + const bytes: Uint8Array + export default bytes +} + +declare module '*.gz' { + const bytes: Uint8Array + export default bytes +}