From 9794c89f7276000e3738fc2f8aced983374fd8c4 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Wed, 20 May 2026 14:18:51 +0100 Subject: [PATCH 1/5] fx --- resources/litert-js/src/download-models.mjs | 84 +++++++------ .../transformers-js/src/download-models.mjs | 118 +++++++++--------- 2 files changed, 101 insertions(+), 101 deletions(-) diff --git a/resources/litert-js/src/download-models.mjs b/resources/litert-js/src/download-models.mjs index cdae08c..7014215 100644 --- a/resources/litert-js/src/download-models.mjs +++ b/resources/litert-js/src/download-models.mjs @@ -37,52 +37,54 @@ async function downloadModels() { console.log(`Starting TFLite model downloads to: **${MODEL_DIR}**`); - for (const modelInfo of MODELS_TO_DOWNLOAD) { - const { repo, filename, url } = modelInfo; + await Promise.all(MODELS_TO_DOWNLOAD.map(modelInfo => downloadModel(modelInfo, cache))); + console.log('TFLite download process finished.'); +} + +async function downloadModel(modelInfo, cache) { + const { repo, filename, url } = modelInfo; + + const cacheKey = `${repo}-${filename}`; + if (cache.has(cacheKey)) { + console.log(`Model ${filename} from ${repo} already cached. Skipping.`); + return; + } + + const modelUrl = url; + const outputPath = path.join(MODEL_DIR, path.basename(filename)); - const cacheKey = `${repo}-${filename}`; - if (cache.has(cacheKey)) { - console.log(`Model ${filename} from ${repo} already cached. Skipping.`); - continue; + console.log(`\nAttempting to download **${filename}** from **${repo}**...`); + console.log(`URL: ${modelUrl}`); + + try { + const response = await fetch(modelUrl); + + if (!response.ok) { + throw new Error(`Failed to fetch: ${response.statusText} (${response.status})`); } - const modelUrl = url; - const outputPath = path.join(MODEL_DIR, path.basename(filename)); - - console.log(`\nAttempting to download **${filename}** from **${repo}**...`); - console.log(`URL: ${modelUrl}`); - - try { - const response = await fetch(modelUrl); - - if (!response.ok) { - throw new Error(`Failed to fetch: ${response.statusText} (${response.status})`); - } - - const fileStream = fs.createWriteStream(outputPath); - await new Promise((resolve, reject) => { - response.body.pipe(fileStream); - response.body.on('error', reject); - fileStream.on('finish', resolve); - }); - - console.log(`Successfully downloaded **${filename}** to **${outputPath}**`); - - if (path.extname(filename) === '.zip') { - console.log(`Extracting **${filename}**...`); - const zip = new AdmZip(outputPath); - zip.extractAllTo(MODEL_DIR, true); - console.log(`Successfully extracted **${filename}** to **${MODEL_DIR}**`); - fs.unlinkSync(outputPath); - console.log(`Deleted zip file **${outputPath}**`); - } - - cache.put(cacheKey); - } catch (err) { - console.error(`Model download failed for ${repo}/${filename}:`, err.message); + const fileStream = fs.createWriteStream(outputPath); + await new Promise((resolve, reject) => { + response.body.pipe(fileStream); + response.body.on('error', reject); + fileStream.on('finish', resolve); + }); + + console.log(`Successfully downloaded **${filename}** to **${outputPath}**`); + + if (path.extname(filename) === '.zip') { + console.log(`Extracting **${filename}**...`); + const zip = new AdmZip(outputPath); + zip.extractAllTo(MODEL_DIR, true); + console.log(`Successfully extracted **${filename}** to **${MODEL_DIR}**`); + fs.unlinkSync(outputPath); + console.log(`Deleted zip file **${outputPath}**`); } + + cache.put(cacheKey); + } catch (err) { + console.error(`Model download failed for ${repo}/${filename}:`, err.message); } - console.log('TFLite download process finished.'); } downloadModels().catch(err => { diff --git a/resources/transformers-js/src/download-models.mjs b/resources/transformers-js/src/download-models.mjs index 99d61aa..4670bda 100644 --- a/resources/transformers-js/src/download-models.mjs +++ b/resources/transformers-js/src/download-models.mjs @@ -82,28 +82,7 @@ async function downloadModels() { try { // Download models that work with pipeline - for (const modelInfo of MODELS_TO_DOWNLOAD) { - const { id: modelId, task: modelTask, dtype: modelDType } = modelInfo; - - const cacheKey = `${modelId}-${modelTask}-${modelDType}`; - if (cache.has(cacheKey)) { - console.log(`Model ${modelId} (${modelTask}, dtype: ${modelDType}) already cached. Skipping.`); - continue; - } - - console.log(`Downloading files for ${modelId} (${modelTask}, dtype: ${modelDType})...`); - - await pipeline( - modelTask, - modelId, - { - cache_dir: env.localModelPath, - dtype: modelDType - }); - - console.log(`Successfully downloaded and cached ${modelId}`); - cache.put(cacheKey); - } + await Promise.all(MODELS_TO_DOWNLOAD.map(modelInfo => downloadPipelineModel(modelInfo, cache))); // Download Xenova/mobileclip_s0 models via components console.log(`Checking Xenova/mobileclip_s0 models...`); @@ -145,45 +124,12 @@ async function downloadModels() { if (!fs.existsSync(kokoroModelPath)) { fs.mkdirSync(kokoroModelPath, { recursive: true }); } - - for (const filename of KOKORO_FILES) { - const cacheKey = `${KOKORO_REPO}-${filename}`; - if (cache.has(cacheKey)) { - console.log(` ${filename} already exists, skipping.`); - continue; - } - const isOnnxFile = filename.endsWith('.onnx') || filename.endsWith('.onnx_data'); - const modelUrl = getHuggingFaceUrl(KOKORO_REPO, filename); - let outputPath; - - if (isOnnxFile) { - const onnxDir = path.join(kokoroModelPath, 'onnx'); - if (!fs.existsSync(onnxDir)) { - fs.mkdirSync(onnxDir, { recursive: true }); - } - outputPath = path.join(onnxDir, filename); - } else { - outputPath = path.join(kokoroModelPath, filename); - } - - console.log(` Downloading ${filename}...`); - try { - const response = await fetch(modelUrl); - if (!response.ok) { - throw new Error(`Failed to fetch ${filename}: ${response.statusText}`); - } - const fileStream = fs.createWriteStream(outputPath); - await new Promise((resolve, reject) => { - response.body.pipe(fileStream); - response.body.on('error', reject); - fileStream.on('finish', resolve); - }); - - cache.put(cacheKey); - } catch (err) { - console.error(` Failed to download ${filename}:`, err.message); - } + const onnxDir = path.join(kokoroModelPath, 'onnx'); + if (!fs.existsSync(onnxDir)) { + fs.mkdirSync(onnxDir, { recursive: true }); } + + await Promise.all(KOKORO_FILES.map(filename => downloadKokoroFile(filename, cache, kokoroModelPath, onnxDir))); console.log(`Successfully checked all files for ${KOKORO_REPO}`); } catch (err) { @@ -194,6 +140,58 @@ async function downloadModels() { env.allowRemoteModels = originalAllowRemote; } +async function downloadPipelineModel(modelInfo, cache) { + const { id: modelId, task: modelTask, dtype: modelDType } = modelInfo; + + const cacheKey = `${modelId}-${modelTask}-${modelDType}`; + if (cache.has(cacheKey)) { + console.log(`Model ${modelId} (${modelTask}, dtype: ${modelDType}) already cached. Skipping.`); + return; + } + + console.log(`Downloading files for ${modelId} (${modelTask}, dtype: ${modelDType})...`); + + await pipeline( + modelTask, + modelId, + { + cache_dir: env.localModelPath, + dtype: modelDType + }); + + console.log(`Successfully downloaded and cached ${modelId}`); + cache.put(cacheKey); +} + +async function downloadKokoroFile(filename, cache, kokoroModelPath, onnxDir) { + const cacheKey = `${KOKORO_REPO}-${filename}`; + if (cache.has(cacheKey)) { + console.log(` ${filename} already exists, skipping.`); + return; + } + const isOnnxFile = filename.endsWith('.onnx') || filename.endsWith('.onnx_data'); + const modelUrl = getHuggingFaceUrl(KOKORO_REPO, filename); + let outputPath = isOnnxFile ? path.join(onnxDir, filename) : path.join(kokoroModelPath, filename); + + console.log(` Downloading ${filename}...`); + try { + const response = await fetch(modelUrl); + if (!response.ok) { + throw new Error(`Failed to fetch ${filename}: ${response.statusText}`); + } + const fileStream = fs.createWriteStream(outputPath); + await new Promise((resolve, reject) => { + response.body.pipe(fileStream); + response.body.on('error', reject); + fileStream.on('finish', resolve); + }); + + cache.put(cacheKey); + } catch (err) { + console.error(` Failed to download ${filename}:`, err.message); + } +} + downloadModels().catch(err => { console.error("Download process terminated.", err); process.exit(1); From 4d5c55f2de829a59ef8030607d03c04dbb00e813 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Wed, 20 May 2026 14:30:12 +0100 Subject: [PATCH 2/5] fix --- about.html | 4 +- index.html | 4 +- package.json | 1 + resources/experimental/package-lock.json | 5 ++ resources/litert-js/package-lock.json | 10 ++- resources/transformers-js/package-lock.json | 5 ++ .../transformers-js/src/download-models.mjs | 89 ++++++++++--------- script/clean.mjs | 45 ++++++++++ 8 files changed, 118 insertions(+), 45 deletions(-) create mode 100644 script/clean.mjs diff --git a/about.html b/about.html index 8c085f6..661c7b2 100644 --- a/about.html +++ b/about.html @@ -18,8 +18,8 @@

About the Web AI Compute Benchmark

Benchmark Version

    -
  • Version: 0.2.0 (2026-05-05)
    • -
  • Git commit: 8907dfe
    • +
  • Version: 0.2.0 (2026-05-20)
    • +
  • Git commit: 9794c89

Library Versions

    diff --git a/index.html b/index.html index 83fdbbf..a52023b 100644 --- a/index.html +++ b/index.html @@ -28,7 +28,7 @@ Additionally, the first iteration of the benchmark may take several minutes to complete.

    - Version: 0.2.0 (2026-05-05) (commit: 8907dfe) + Version: 0.2.0 (2026-05-20) (commit: 9794c89)

    @@ -89,7 +89,7 @@

    Score

    -

    Detailed Results 0.2.0 (2026-05-05)

    +

    Detailed Results 0.2.0 (2026-05-20)

    Non-standard Parameters

    diff --git a/package.json b/package.json index 3910dee..e8c39e6 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "license": "SEE LICENSE IN LICENSE", "scripts": { "build": "node script/build.mjs", + "clean": "node script/clean.mjs", "dev": "node tests/server.mjs", "lint:check": "eslint **/*.{js,mjs,jsx,ts,tsx}", "lint:fix": "eslint \"**/*.{js,mjs,jsx,ts,tsx}\" --fix", diff --git a/resources/experimental/package-lock.json b/resources/experimental/package-lock.json index 3d244fa..d3f9d3f 100644 --- a/resources/experimental/package-lock.json +++ b/resources/experimental/package-lock.json @@ -908,6 +908,7 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -941,6 +942,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -1064,6 +1066,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -3163,6 +3166,7 @@ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.1.tgz", "integrity": "sha512-Gdj3X74CLJJ8zy4URmK42W7wTZUJrqL+z8nyGEr4dTN0kb3nVs+ZvjbTOqRYPD7qX4tUmwyHL9Q9K6T1seW6Yw==", "license": "MIT", + "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -3212,6 +3216,7 @@ "integrity": "sha512-MfwFQ6SfwinsUVi0rNJm7rHZ31GyTcpVE5pgVA3hwFRb7COD4TzjUUwhGWKfO50+xdc2MQPuEBBJoqIMGt3JDw==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.6.1", "@webpack-cli/configtest": "^3.0.1", diff --git a/resources/litert-js/package-lock.json b/resources/litert-js/package-lock.json index 1610b57..8f30d90 100644 --- a/resources/litert-js/package-lock.json +++ b/resources/litert-js/package-lock.json @@ -283,6 +283,7 @@ "resolved": "https://registry.npmjs.org/@tensorflow/tfjs-core/-/tfjs-core-4.22.0.tgz", "integrity": "sha512-LEkOyzbknKFoWUwfkr59vSB68DMJ4cjwwHgicXN0DUi3a0Vh1Er3JQqCI1Hl86GGZQvY8ezVrtDIvqR1ZFW55A==", "license": "Apache-2.0", + "peer": true, "dependencies": { "@types/long": "^4.0.1", "@types/offscreencanvas": "~2019.7.0", @@ -713,6 +714,7 @@ "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -757,6 +759,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -923,6 +926,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -2535,6 +2539,7 @@ "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -2764,7 +2769,8 @@ "version": "3.0.5", "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==", - "license": "MIT" + "license": "MIT", + "peer": true }, "node_modules/semver": { "version": "7.7.3", @@ -3168,6 +3174,7 @@ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.2.tgz", "integrity": "sha512-dRXm0a2qcHPUBEzVk8uph0xWSjV/xZxenQQbLwnwP7caQCYpqG1qddwlyEkIDkYn0K8tvmcrZ+bOrzoQ3HxCDw==", "license": "MIT", + "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -3216,6 +3223,7 @@ "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-6.0.1.tgz", "integrity": "sha512-MfwFQ6SfwinsUVi0rNJm7rHZ31GyTcpVE5pgVA3hwFRb7COD4TzjUUwhGWKfO50+xdc2MQPuEBBJoqIMGt3JDw==", "dev": true, + "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.6.1", "@webpack-cli/configtest": "^3.0.1", diff --git a/resources/transformers-js/package-lock.json b/resources/transformers-js/package-lock.json index ca665e9..29f8063 100644 --- a/resources/transformers-js/package-lock.json +++ b/resources/transformers-js/package-lock.json @@ -876,6 +876,7 @@ "version": "8.15.0", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -908,6 +909,7 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.18.0.tgz", "integrity": "sha512-PlXPeEWMXMZ7sPYOHqmDyCJzcfNrUr3fGNKtezX14ykXOEIvyK81d+qydx89KY5O71FKMPaQ2vBfBFI5NHR63A==", "license": "MIT", + "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -1026,6 +1028,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "baseline-browser-mapping": "^2.9.0", "caniuse-lite": "^1.0.30001759", @@ -3039,6 +3042,7 @@ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.105.2.tgz", "integrity": "sha512-dRXm0a2qcHPUBEzVk8uph0xWSjV/xZxenQQbLwnwP7caQCYpqG1qddwlyEkIDkYn0K8tvmcrZ+bOrzoQ3HxCDw==", "license": "MIT", + "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -3087,6 +3091,7 @@ "resolved": "https://registry.npmjs.org/webpack-cli/-/webpack-cli-6.0.1.tgz", "integrity": "sha512-MfwFQ6SfwinsUVi0rNJm7rHZ31GyTcpVE5pgVA3hwFRb7COD4TzjUUwhGWKfO50+xdc2MQPuEBBJoqIMGt3JDw==", "dev": true, + "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.6.1", "@webpack-cli/configtest": "^3.0.1", diff --git a/resources/transformers-js/src/download-models.mjs b/resources/transformers-js/src/download-models.mjs index 4670bda..8ed3a53 100644 --- a/resources/transformers-js/src/download-models.mjs +++ b/resources/transformers-js/src/download-models.mjs @@ -81,44 +81,6 @@ async function downloadModels() { env.allowRemoteModels = true; try { - // Download models that work with pipeline - await Promise.all(MODELS_TO_DOWNLOAD.map(modelInfo => downloadPipelineModel(modelInfo, cache))); - - // Download Xenova/mobileclip_s0 models via components - console.log(`Checking Xenova/mobileclip_s0 models...`); - for (const modelInfo of MOBILECLIP_MODELS_TO_DOWNLOAD) { - const className = modelInfo.modelClass.name; - const cacheKey = `mobileclip-${className}-${modelInfo.dtype || ''}`; - if (cache.has(cacheKey)) { - console.log(`Model ${className} (dtype: ${modelInfo.dtype}) already cached. Skipping.`); - continue; - } - - console.log(`Downloading Xenova/mobileclip_s0 (${className}${modelInfo.dtype ? `, dtype: ${modelInfo.dtype}` : ''})...`); - await modelInfo.modelClass.from_pretrained("Xenova/mobileclip_s0", { - cache_dir: env.localModelPath, - dtype: modelInfo.dtype - }); - - cache.put(cacheKey); - } - console.log(`Successfully checked Xenova/mobileclip_s0`); - - // Download Xenova/sam-vit-base models - console.log(`Checking Xenova/sam-vit-base models...`); - if (!cache.has('SAM-SamModel-fp32')) { - console.log(`Downloading Xenova/sam-vit-base (SamModel, fp32)...`); - await SamModel.from_pretrained("Xenova/sam-vit-base", { cache_dir: env.localModelPath, dtype: 'fp32' }); - cache.put('SAM-SamModel-fp32'); - } - if (!cache.has('SAM-SamProcessor-default')) { - console.log(`Downloading Xenova/sam-vit-base (SamProcessor)...`); - await SamProcessor.from_pretrained("Xenova/sam-vit-base", { cache_dir: env.localModelPath }); - cache.put('SAM-SamProcessor-default'); - } - console.log(`Successfully checked Xenova/sam-vit-base`); - - // Download onnx-community/Kokoro-82M-v1.0-ONNX model console.log(`Starting manual download check for ${KOKORO_REPO}...`); const kokoroModelPath = path.join(MODEL_DIR, KOKORO_REPO); if (!fs.existsSync(kokoroModelPath)) { @@ -129,8 +91,22 @@ async function downloadModels() { fs.mkdirSync(onnxDir, { recursive: true }); } - await Promise.all(KOKORO_FILES.map(filename => downloadKokoroFile(filename, cache, kokoroModelPath, onnxDir))); - console.log(`Successfully checked all files for ${KOKORO_REPO}`); + console.log(`Downloading all Transformers.js models in parallel...`); + await Promise.all([ + // Download models that work with pipeline + ...MODELS_TO_DOWNLOAD.map(modelInfo => downloadPipelineModel(modelInfo, cache)), + + // Download Xenova/mobileclip_s0 models via components + ...MOBILECLIP_MODELS_TO_DOWNLOAD.map(modelInfo => downloadMobileClipModel(modelInfo, cache)), + + // Download Xenova/sam-vit-base models + downloadSamModel(cache), + downloadSamProcessor(cache), + + // Download onnx-community/Kokoro-82M-v1.0-ONNX model + ...KOKORO_FILES.map(filename => downloadKokoroFile(filename, cache, kokoroModelPath, onnxDir)) + ]); + console.log(`Successfully checked and downloaded all models.`); } catch (err) { console.error("Model download failed:", err); @@ -163,6 +139,39 @@ async function downloadPipelineModel(modelInfo, cache) { cache.put(cacheKey); } +async function downloadMobileClipModel(modelInfo, cache) { + const className = modelInfo.modelClass.name; + const cacheKey = `mobileclip-${className}-${modelInfo.dtype || ''}`; + if (cache.has(cacheKey)) { + console.log(`Model ${className} (dtype: ${modelInfo.dtype}) already cached. Skipping.`); + return; + } + + console.log(`Downloading Xenova/mobileclip_s0 (${className}${modelInfo.dtype ? `, dtype: ${modelInfo.dtype}` : ''})...`); + await modelInfo.modelClass.from_pretrained("Xenova/mobileclip_s0", { + cache_dir: env.localModelPath, + dtype: modelInfo.dtype + }); + + cache.put(cacheKey); +} + +async function downloadSamModel(cache) { + if (!cache.has('SAM-SamModel-fp32')) { + console.log(`Downloading Xenova/sam-vit-base (SamModel, fp32)...`); + await SamModel.from_pretrained("Xenova/sam-vit-base", { cache_dir: env.localModelPath, dtype: 'fp32' }); + cache.put('SAM-SamModel-fp32'); + } +} + +async function downloadSamProcessor(cache) { + if (!cache.has('SAM-SamProcessor-default')) { + console.log(`Downloading Xenova/sam-vit-base (SamProcessor)...`); + await SamProcessor.from_pretrained("Xenova/sam-vit-base", { cache_dir: env.localModelPath }); + cache.put('SAM-SamProcessor-default'); + } +} + async function downloadKokoroFile(filename, cache, kokoroModelPath, onnxDir) { const cacheKey = `${KOKORO_REPO}-${filename}`; if (cache.has(cacheKey)) { diff --git a/script/clean.mjs b/script/clean.mjs new file mode 100644 index 0000000..38ba8cd --- /dev/null +++ b/script/clean.mjs @@ -0,0 +1,45 @@ +// Copyright 2026 Google LLC +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file or at +// https://developers.google.com/open-source/licenses/bsd + +import {defaultSuites} from "../resources/default-tests.mjs"; +import {logInfo} from "./helper.mjs"; +import fs from "node:fs"; +import path from "node:path"; + +const workloadDirs = new Set(); + +for (const suite of defaultSuites) { + const parts = suite.url.split("/"); + const workloadDir = parts.slice(0, parts.indexOf("dist")).join("/"); + workloadDirs.add(workloadDir); +} + +logInfo(`CLEANING ${workloadDirs.size} WORKLOADS`); + +for (const workloadDir of workloadDirs) { + const modelsDir = path.join(workloadDir, "models"); + const distDir = path.join(workloadDir, "dist"); + + if (fs.existsSync(distDir)) { + logInfo(` Removing ${distDir}...`); + fs.rmSync(distDir, { recursive: true, force: true }); + } + + if (fs.existsSync(modelsDir)) { + logInfo(` Cleaning ${modelsDir}...`); + for (const item of fs.readdirSync(modelsDir)) { + if (item === "gemma") { + logInfo(` Preserving ${path.join(modelsDir, item)}...`); + continue; + } + const itemPath = path.join(modelsDir, item); + logInfo(` Removing ${itemPath}...`); + fs.rmSync(itemPath, { recursive: true, force: true }); + } + } +} + +logInfo("Clean successfully completed."); From 1a56b848817c33f2cd8d0a72a7b1afe5a1701940 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Wed, 1 Jul 2026 16:14:13 +0200 Subject: [PATCH 3/5] fx --- index.html | 6 - .../transformers-js/src/download-models.mjs | 108 ------------------ 2 files changed, 114 deletions(-) diff --git a/index.html b/index.html index d261b89..e907a51 100644 --- a/index.html +++ b/index.html @@ -89,13 +89,7 @@

    Score

    -<<<<<<< HEAD -

    Detailed Results 0.2.0 (2026-05-20)

    -||||||| 3d66d0b -

    Detailed Results 0.2.0 (2026-05-05)

    -=======

    Detailed Results 0.2.0 (2026-06-17)

    ->>>>>>> main

    Non-standard Parameters

    diff --git a/resources/transformers-js/src/download-models.mjs b/resources/transformers-js/src/download-models.mjs index 2d1d916..a83dfa8 100644 --- a/resources/transformers-js/src/download-models.mjs +++ b/resources/transformers-js/src/download-models.mjs @@ -82,68 +82,6 @@ async function downloadModels() { env.allowRemoteModels = true; try { -<<<<<<< HEAD -||||||| 3d66d0b - // Download models that work with pipeline - for (const modelInfo of MODELS_TO_DOWNLOAD) { - const { id: modelId, task: modelTask, dtype: modelDType } = modelInfo; - - const cacheKey = `${modelId}-${modelTask}-${modelDType}`; - if (cache.has(cacheKey)) { - console.log(`Model ${modelId} (${modelTask}, dtype: ${modelDType}) already cached. Skipping.`); - continue; - } - - console.log(`Downloading files for ${modelId} (${modelTask}, dtype: ${modelDType})...`); - - await pipeline( - modelTask, - modelId, - { - cache_dir: env.localModelPath, - dtype: modelDType - }); - - console.log(`Successfully downloaded and cached ${modelId}`); - cache.put(cacheKey); - } - - // Download Xenova/mobileclip_s0 models via components - console.log(`Checking Xenova/mobileclip_s0 models...`); - for (const modelInfo of MOBILECLIP_MODELS_TO_DOWNLOAD) { - const className = modelInfo.modelClass.name; - const cacheKey = `mobileclip-${className}-${modelInfo.dtype || ''}`; - if (cache.has(cacheKey)) { - console.log(`Model ${className} (dtype: ${modelInfo.dtype}) already cached. Skipping.`); - continue; - } - - console.log(`Downloading Xenova/mobileclip_s0 (${className}${modelInfo.dtype ? `, dtype: ${modelInfo.dtype}` : ''})...`); - await modelInfo.modelClass.from_pretrained("Xenova/mobileclip_s0", { - cache_dir: env.localModelPath, - dtype: modelInfo.dtype - }); - - cache.put(cacheKey); - } - console.log(`Successfully checked Xenova/mobileclip_s0`); - - // Download Xenova/sam-vit-base models - console.log(`Checking Xenova/sam-vit-base models...`); - if (!cache.has('SAM-SamModel-fp32')) { - console.log(`Downloading Xenova/sam-vit-base (SamModel, fp32)...`); - await SamModel.from_pretrained("Xenova/sam-vit-base", { cache_dir: env.localModelPath, dtype: 'fp32' }); - cache.put('SAM-SamModel-fp32'); - } - if (!cache.has('SAM-SamProcessor-default')) { - console.log(`Downloading Xenova/sam-vit-base (SamProcessor)...`); - await SamProcessor.from_pretrained("Xenova/sam-vit-base", { cache_dir: env.localModelPath }); - cache.put('SAM-SamProcessor-default'); - } - console.log(`Successfully checked Xenova/sam-vit-base`); - - // Download onnx-community/Kokoro-82M-v1.0-ONNX model -======= // Download models that work with pipeline for (const modelInfo of MODELS_TO_DOWNLOAD) { const { id: modelId, task: modelTask, dtype: modelDType } = modelInfo; @@ -203,56 +141,11 @@ async function downloadModels() { console.log(`Successfully checked Xenova/sam-vit-base`); // Download onnx-community/Kokoro-82M-v1.0-ONNX model ->>>>>>> main console.log(`Starting manual download check for ${KOKORO_REPO}...`); const kokoroModelPath = path.join(MODEL_DIR, KOKORO_REPO); if (!fs.existsSync(kokoroModelPath)) { fs.mkdirSync(kokoroModelPath, { recursive: true }); } -<<<<<<< HEAD - const onnxDir = path.join(kokoroModelPath, 'onnx'); - if (!fs.existsSync(onnxDir)) { - fs.mkdirSync(onnxDir, { recursive: true }); -||||||| 3d66d0b - - for (const filename of KOKORO_FILES) { - const cacheKey = `${KOKORO_REPO}-${filename}`; - if (cache.has(cacheKey)) { - console.log(` ${filename} already exists, skipping.`); - continue; - } - const isOnnxFile = filename.endsWith('.onnx') || filename.endsWith('.onnx_data'); - const modelUrl = getHuggingFaceUrl(KOKORO_REPO, filename); - let outputPath; - - if (isOnnxFile) { - const onnxDir = path.join(kokoroModelPath, 'onnx'); - if (!fs.existsSync(onnxDir)) { - fs.mkdirSync(onnxDir, { recursive: true }); - } - outputPath = path.join(onnxDir, filename); - } else { - outputPath = path.join(kokoroModelPath, filename); - } - - console.log(` Downloading ${filename}...`); - try { - const response = await fetch(modelUrl); - if (!response.ok) { - throw new Error(`Failed to fetch ${filename}: ${response.statusText}`); - } - const fileStream = fs.createWriteStream(outputPath); - await new Promise((resolve, reject) => { - response.body.pipe(fileStream); - response.body.on('error', reject); - fileStream.on('finish', resolve); - }); - - cache.put(cacheKey); - } catch (err) { - console.error(` Failed to download ${filename}:`, err.message); - } -======= for (const filename of KOKORO_FILES) { const cacheKey = `${KOKORO_REPO}-${filename}`; @@ -293,7 +186,6 @@ async function downloadModels() { } catch (err) { console.error(` Failed to download ${filename} after retries:`, err.message); } ->>>>>>> main } console.log(`Downloading all Transformers.js models in parallel...`); From 6713f104e516f14c69919fa69791d311d1b05ef8 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Wed, 1 Jul 2026 16:29:55 +0200 Subject: [PATCH 4/5] Fix model download scripts to run in parallel and add retries --- .../experimental/src/download-models.mjs | 51 ++++--- resources/litert-js/src/download-models.mjs | 75 +++------- .../transformers-js/src/download-models.mjs | 135 +++--------------- 3 files changed, 67 insertions(+), 194 deletions(-) diff --git a/resources/experimental/src/download-models.mjs b/resources/experimental/src/download-models.mjs index e6dafde..b27d04b 100644 --- a/resources/experimental/src/download-models.mjs +++ b/resources/experimental/src/download-models.mjs @@ -32,29 +32,11 @@ async function downloadModels() { env.allowRemoteModels = true; try { - // Download models that work with pipeline - for (const modelInfo of MODELS_TO_DOWNLOAD) { - const { id: modelId, task: modelTask, dtype: modelDType } = modelInfo; - - const cacheKey = `${modelId}-${modelTask}-${modelDType}`; - if (cache.has(cacheKey)) { - console.log(`Model ${modelId} (${modelTask}, dtype: ${modelDType}) already cached. Skipping.`); - continue; - } - - console.log(`Downloading files for ${modelId} (${modelTask}, dtype: ${modelDType})...`); - - await retry(() => pipeline( - modelTask, - modelId, - { - cache_dir: env.localModelPath, - dtype: modelDType - })); - - console.log(`Successfully downloaded and cached ${modelId}`); - cache.put(cacheKey); - } + console.log(`Downloading all experimental models in parallel...`); + await Promise.all( + MODELS_TO_DOWNLOAD.map(modelInfo => downloadPipelineModel(modelInfo, cache)) + ); + console.log(`Successfully checked and downloaded all models.`); } catch (err) { console.error("Model download failed:", err); @@ -64,6 +46,29 @@ async function downloadModels() { env.allowRemoteModels = originalAllowRemote; } +async function downloadPipelineModel(modelInfo, cache) { + const { id: modelId, task: modelTask, dtype: modelDType } = modelInfo; + + const cacheKey = `${modelId}-${modelTask}-${modelDType}`; + if (cache.has(cacheKey)) { + console.log(`Model ${modelId} (${modelTask}, dtype: ${modelDType}) already cached. Skipping.`); + return; + } + + console.log(`Downloading files for ${modelId} (${modelTask}, dtype: ${modelDType})...`); + + await retry(() => pipeline( + modelTask, + modelId, + { + cache_dir: env.localModelPath, + dtype: modelDType + })); + + console.log(`Successfully downloaded and cached ${modelId}`); + cache.put(cacheKey); +} + downloadModels().catch(err => { console.error("Download process terminated."); process.exit(1); diff --git a/resources/litert-js/src/download-models.mjs b/resources/litert-js/src/download-models.mjs index 53b45c5..2a2961d 100644 --- a/resources/litert-js/src/download-models.mjs +++ b/resources/litert-js/src/download-models.mjs @@ -39,52 +39,13 @@ async function downloadModels() { console.log(`Starting TFLite model downloads to: **${MODEL_DIR}**`); - for (const modelInfo of MODELS_TO_DOWNLOAD) { - const { repo, filename, url } = modelInfo; - - const cacheKey = `${repo}-${filename}`; - if (cache.has(cacheKey)) { - console.log(`Model ${filename} from ${repo} already cached. Skipping.`); - continue; - } - - const modelUrl = url; - const outputPath = path.join(MODEL_DIR, path.basename(filename)); - - console.log(`\nAttempting to download **${filename}** from **${repo}**...`); - console.log(`URL: ${modelUrl}`); - - try { - await retry(async () => { - const response = await fetch(modelUrl); - - if (!response.ok) { - throw new Error(`Failed to fetch: ${response.statusText} (${response.status})`); - } - - const fileStream = fs.createWriteStream(outputPath); - await new Promise((resolve, reject) => { - response.body.pipe(fileStream); - response.body.on('error', reject); - fileStream.on('finish', resolve); - }); - }); - - console.log(`Successfully downloaded **${filename}** to **${outputPath}**`); - - if (path.extname(filename) === '.zip') { - console.log(`Extracting **${filename}**...`); - const zip = new AdmZip(outputPath); - zip.extractAllTo(MODEL_DIR, true); - console.log(`Successfully extracted **${filename}** to **${MODEL_DIR}**`); - fs.unlinkSync(outputPath); - console.log(`Deleted zip file **${outputPath}**`); - } - - cache.put(cacheKey); - } catch (err) { - console.error(`Model download failed for ${repo}/${filename} after retries:`, err.message); - } + try { + await Promise.all( + MODELS_TO_DOWNLOAD.map(modelInfo => downloadModel(modelInfo, cache)) + ); + } catch (err) { + console.error("TFLite model download failed:", err); + throw err; } console.log('TFLite download process finished.'); } @@ -105,17 +66,19 @@ async function downloadModel(modelInfo, cache) { console.log(`URL: ${modelUrl}`); try { - const response = await fetch(modelUrl); + await retry(async () => { + const response = await fetch(modelUrl); - if (!response.ok) { - throw new Error(`Failed to fetch: ${response.statusText} (${response.status})`); - } + if (!response.ok) { + throw new Error(`Failed to fetch: ${response.statusText} (${response.status})`); + } - const fileStream = fs.createWriteStream(outputPath); - await new Promise((resolve, reject) => { - response.body.pipe(fileStream); - response.body.on('error', reject); - fileStream.on('finish', resolve); + const fileStream = fs.createWriteStream(outputPath); + await new Promise((resolve, reject) => { + response.body.pipe(fileStream); + response.body.on('error', reject); + fileStream.on('finish', resolve); + }); }); console.log(`Successfully downloaded **${filename}** to **${outputPath}**`); @@ -131,7 +94,7 @@ async function downloadModel(modelInfo, cache) { cache.put(cacheKey); } catch (err) { - console.error(`Model download failed for ${repo}/${filename}:`, err.message); + console.error(`Model download failed for ${repo}/${filename} after retries:`, err.message); } } diff --git a/resources/transformers-js/src/download-models.mjs b/resources/transformers-js/src/download-models.mjs index a83dfa8..f427ea7 100644 --- a/resources/transformers-js/src/download-models.mjs +++ b/resources/transformers-js/src/download-models.mjs @@ -82,110 +82,13 @@ async function downloadModels() { env.allowRemoteModels = true; try { - // Download models that work with pipeline - for (const modelInfo of MODELS_TO_DOWNLOAD) { - const { id: modelId, task: modelTask, dtype: modelDType } = modelInfo; - - const cacheKey = `${modelId}-${modelTask}-${modelDType}`; - if (cache.has(cacheKey)) { - console.log(`Model ${modelId} (${modelTask}, dtype: ${modelDType}) already cached. Skipping.`); - continue; - } - - console.log(`Downloading files for ${modelId} (${modelTask}, dtype: ${modelDType})...`); - - await retry(() => pipeline( - modelTask, - modelId, - { - cache_dir: env.localModelPath, - dtype: modelDType - })); - - console.log(`Successfully downloaded and cached ${modelId}`); - cache.put(cacheKey); - } - - // Download Xenova/mobileclip_s0 models via components - console.log(`Checking Xenova/mobileclip_s0 models...`); - for (const modelInfo of MOBILECLIP_MODELS_TO_DOWNLOAD) { - const className = modelInfo.modelClass.name; - const cacheKey = `mobileclip-${className}-${modelInfo.dtype || ''}`; - if (cache.has(cacheKey)) { - console.log(`Model ${className} (dtype: ${modelInfo.dtype}) already cached. Skipping.`); - continue; - } - - console.log(`Downloading Xenova/mobileclip_s0 (${className}${modelInfo.dtype ? `, dtype: ${modelInfo.dtype}` : ''})...`); - await retry(() => modelInfo.modelClass.from_pretrained("Xenova/mobileclip_s0", { - cache_dir: env.localModelPath, - dtype: modelInfo.dtype - })); - - cache.put(cacheKey); - } - console.log(`Successfully checked Xenova/mobileclip_s0`); - - // Download Xenova/sam-vit-base models - console.log(`Checking Xenova/sam-vit-base models...`); - if (!cache.has('SAM-SamModel-fp32')) { - console.log(`Downloading Xenova/sam-vit-base (SamModel, fp32)...`); - await retry(() => SamModel.from_pretrained("Xenova/sam-vit-base", { cache_dir: env.localModelPath, dtype: 'fp32' })); - cache.put('SAM-SamModel-fp32'); - } - if (!cache.has('SAM-SamProcessor-default')) { - console.log(`Downloading Xenova/sam-vit-base (SamProcessor)...`); - await retry(() => SamProcessor.from_pretrained("Xenova/sam-vit-base", { cache_dir: env.localModelPath })); - cache.put('SAM-SamProcessor-default'); - } - console.log(`Successfully checked Xenova/sam-vit-base`); - - // Download onnx-community/Kokoro-82M-v1.0-ONNX model - console.log(`Starting manual download check for ${KOKORO_REPO}...`); const kokoroModelPath = path.join(MODEL_DIR, KOKORO_REPO); if (!fs.existsSync(kokoroModelPath)) { fs.mkdirSync(kokoroModelPath, { recursive: true }); } - - for (const filename of KOKORO_FILES) { - const cacheKey = `${KOKORO_REPO}-${filename}`; - if (cache.has(cacheKey)) { - console.log(` ${filename} already exists, skipping.`); - continue; - } - const isOnnxFile = filename.endsWith('.onnx') || filename.endsWith('.onnx_data'); - const modelUrl = getHuggingFaceUrl(KOKORO_REPO, filename); - let outputPath; - - if (isOnnxFile) { - const onnxDir = path.join(kokoroModelPath, 'onnx'); - if (!fs.existsSync(onnxDir)) { - fs.mkdirSync(onnxDir, { recursive: true }); - } - outputPath = path.join(onnxDir, filename); - } else { - outputPath = path.join(kokoroModelPath, filename); - } - - console.log(` Downloading ${filename}...`); - try { - await retry(async () => { - const response = await fetch(modelUrl); - if (!response.ok) { - throw new Error(`Failed to fetch ${filename}: ${response.statusText}`); - } - const fileStream = fs.createWriteStream(outputPath); - await new Promise((resolve, reject) => { - response.body.pipe(fileStream); - response.body.on('error', reject); - fileStream.on('finish', resolve); - }); - }); - - cache.put(cacheKey); - } catch (err) { - console.error(` Failed to download ${filename} after retries:`, err.message); - } + const onnxDir = path.join(kokoroModelPath, 'onnx'); + if (!fs.existsSync(onnxDir)) { + fs.mkdirSync(onnxDir, { recursive: true }); } console.log(`Downloading all Transformers.js models in parallel...`); @@ -224,13 +127,13 @@ async function downloadPipelineModel(modelInfo, cache) { console.log(`Downloading files for ${modelId} (${modelTask}, dtype: ${modelDType})...`); - await pipeline( + await retry(() => pipeline( modelTask, modelId, { cache_dir: env.localModelPath, dtype: modelDType - }); + })); console.log(`Successfully downloaded and cached ${modelId}`); cache.put(cacheKey); @@ -245,10 +148,10 @@ async function downloadMobileClipModel(modelInfo, cache) { } console.log(`Downloading Xenova/mobileclip_s0 (${className}${modelInfo.dtype ? `, dtype: ${modelInfo.dtype}` : ''})...`); - await modelInfo.modelClass.from_pretrained("Xenova/mobileclip_s0", { + await retry(() => modelInfo.modelClass.from_pretrained("Xenova/mobileclip_s0", { cache_dir: env.localModelPath, dtype: modelInfo.dtype - }); + })); cache.put(cacheKey); } @@ -256,7 +159,7 @@ async function downloadMobileClipModel(modelInfo, cache) { async function downloadSamModel(cache) { if (!cache.has('SAM-SamModel-fp32')) { console.log(`Downloading Xenova/sam-vit-base (SamModel, fp32)...`); - await SamModel.from_pretrained("Xenova/sam-vit-base", { cache_dir: env.localModelPath, dtype: 'fp32' }); + await retry(() => SamModel.from_pretrained("Xenova/sam-vit-base", { cache_dir: env.localModelPath, dtype: 'fp32' })); cache.put('SAM-SamModel-fp32'); } } @@ -264,7 +167,7 @@ async function downloadSamModel(cache) { async function downloadSamProcessor(cache) { if (!cache.has('SAM-SamProcessor-default')) { console.log(`Downloading Xenova/sam-vit-base (SamProcessor)...`); - await SamProcessor.from_pretrained("Xenova/sam-vit-base", { cache_dir: env.localModelPath }); + await retry(() => SamProcessor.from_pretrained("Xenova/sam-vit-base", { cache_dir: env.localModelPath })); cache.put('SAM-SamProcessor-default'); } } @@ -281,15 +184,17 @@ async function downloadKokoroFile(filename, cache, kokoroModelPath, onnxDir) { console.log(` Downloading ${filename}...`); try { - const response = await fetch(modelUrl); - if (!response.ok) { - throw new Error(`Failed to fetch ${filename}: ${response.statusText}`); - } - const fileStream = fs.createWriteStream(outputPath); - await new Promise((resolve, reject) => { - response.body.pipe(fileStream); - response.body.on('error', reject); - fileStream.on('finish', resolve); + await retry(async () => { + const response = await fetch(modelUrl); + if (!response.ok) { + throw new Error(`Failed to fetch ${filename}: ${response.statusText}`); + } + const fileStream = fs.createWriteStream(outputPath); + await new Promise((resolve, reject) => { + response.body.pipe(fileStream); + response.body.on('error', reject); + fileStream.on('finish', resolve); + }); }); cache.put(cacheKey); From 2cf80676ecda37808f8465f1d1ed57c470999a29 Mon Sep 17 00:00:00 2001 From: Camillo Bruni Date: Wed, 1 Jul 2026 16:33:13 +0200 Subject: [PATCH 5/5] fix-readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index e3b28a2..a092141 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,7 @@ See the Speedometer repo for a more detailed explanation, e.g., in which phases - Workloads are in `resources/transformers-js/` and `resources/litert-js`. - Shared files are in `resources/shared/`, which is depended-upon as a local package. - The default suite / tests to run are in `resources/default-tests.mjs`. +- Cleaning build artifacts and cached models (optional): Run `npm run clean` in the root directory. This will delete the `dist` directories and clear cached models for all workloads (while preserving the large `gemma` model to avoid unnecessary re-downloads). ## How to Run Individual Workload