Add cactus bench: CPU throughput benchmarking for Mac and Android

cactus-engine/tests/CMakeLists.txt

@@ -46,3 +46,6 @@ endif()
 
 add_executable(transcribe transcribe.cpp)
 target_link_libraries(transcribe PRIVATE cactus_engine)
+
+add_executable(llm_bench llm_bench.cpp)
+target_link_libraries(llm_bench PRIVATE cactus_engine)

cactus-engine/tests/android/CMakeLists.txt

@@ -18,6 +18,9 @@ foreach(TEST_FILE ${TEST_SOURCES})
     target_link_libraries(${TEST_NAME} PRIVATE cactus_engine)
 endforeach()
 
+add_executable(llm_bench ${TESTS_DIR}/llm_bench.cpp)
+target_link_libraries(llm_bench PRIVATE cactus_engine)
+
 if(TARGET test_curl AND EXISTS "${CACTUS_CURL_ROOT}/include/curl/curl.h")
     target_include_directories(test_curl PRIVATE "${CACTUS_CURL_ROOT}/include")
 endif()

cactus-engine/tests/llm_bench.cpp

@@ -0,0 +1,121 @@
+#include "cactus_engine.h"
+
+#include <cstdint>
+#include <fstream>
+#include <iostream>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+namespace {
+
+std::vector<uint32_t> load_tokens(const std::string& path) {
+    std::ifstream in(path);
+    if (!in) {
+        throw std::runtime_error("failed to open token input: " + path);
+    }
+    std::string text((std::istreambuf_iterator<char>(in)), std::istreambuf_iterator<char>());
+    std::vector<uint32_t> tokens;
+    std::stringstream ss(text);
+    std::string item;
+    while (std::getline(ss, item, ',')) {
+        if (item.empty()) continue;
+        tokens.push_back(static_cast<uint32_t>(std::stoul(item)));
+    }
+    if (tokens.empty()) {
+        throw std::runtime_error("token input is empty: " + path);
+    }
+    return tokens;
+}
+
+std::string escape_json(const std::string& s) {
+    std::string out;
+    out.reserve(s.size() + 16);
+    for (char c : s) {
+        switch (c) {
+            case '"':  out += "\\\""; break;
+            case '\\': out += "\\\\"; break;
+            case '\n': out += "\\n";  break;
+            case '\r': out += "\\r";  break;
+            case '\t': out += "\\t";  break;
+            default:   out += c;      break;
+        }
+    }
+    return out;
+}
+
+std::string complete_options(size_t max_tokens) {
+    return R"({"max_tokens": )" + std::to_string(max_tokens) +
+           R"(, "temperature": 0.0, "top_p": 0.0, "top_k": 1, "telemetry_enabled": false,)"
+           R"( "stop_sequences": ["<|im_end|>", "<end_of_turn>"]})";
+}
+
+int usage(const char* prog) {
+    std::cerr << "Usage: " << prog << " <model_dir> --tokens <ids.csv> --decode <n>\n"
+              << "       " << prog << " <model_dir> --prompt <text> --max-tokens <n> [--warmup]\n";
+    return 2;
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+    if (argc < 2) return usage(argv[0]);
+    std::string model_dir = argv[1];
+    std::string tokens_path;
+    std::string prompt;
+    size_t decode_tokens = 32;
+    size_t max_tokens = 32;
+    bool warmup = false;
+
+    for (int i = 2; i < argc; i++) {
+        std::string arg = argv[i];
+        if (arg == "--tokens" && i + 1 < argc) tokens_path = argv[++i];
+        else if (arg == "--decode" && i + 1 < argc) decode_tokens = std::stoul(argv[++i]);
+        else if (arg == "--prompt" && i + 1 < argc) prompt = argv[++i];
+        else if (arg == "--max-tokens" && i + 1 < argc) max_tokens = std::stoul(argv[++i]);
+        else if (arg == "--warmup") warmup = true;
+        else return usage(argv[0]);
+    }
+    if (tokens_path.empty() == prompt.empty()) return usage(argv[0]);
+
+    try {
+        cactus_log_set_level(2);
+        cactus_model_t model = cactus_init(model_dir.c_str(), nullptr, false);
+        if (!model) {
+            const char* error = cactus_get_last_error();
+            std::cerr << (error ? error : "cactus_init failed") << "\n";
+            return 1;
+        }
+
+        std::vector<char> buffer(65536);
+        bool ok;
+        if (!tokens_path.empty()) {
+            auto tokens = load_tokens(tokens_path);
+            int rc = cactus_benchmark_tokens(model, tokens.data(), tokens.size(),
+                                             decode_tokens, buffer.data(), buffer.size());
+            ok = rc >= 0;
+        } else {
+            std::string messages = R"([{"role": "user", "content": ")" + escape_json(prompt) + R"("}])";
+            if (warmup) {
+                cactus_complete(model, messages.c_str(), buffer.data(), buffer.size(),
+                                complete_options(16).c_str(), nullptr, nullptr, nullptr, nullptr, 0);
+                cactus_reset(model);
+            }
+            int rc = cactus_complete(model, messages.c_str(), buffer.data(), buffer.size(),
+                                     complete_options(max_tokens).c_str(), nullptr, nullptr, nullptr, nullptr, 0);
+            ok = rc > 0;
+        }
+        cactus_destroy(model);
+        if (!ok) {
+            const char* error = cactus_get_last_error();
+            std::cerr << (error ? error : buffer.data()) << "\n";
+            return 1;
+        }
+        std::cout << buffer.data() << "\n";
+        return 0;
+    } catch (const std::exception& exc) {
+        std::cerr << "Error: " << exc.what() << "\n";
+        return 1;
+    }
+}

docs/benchmarking.md

@@ -0,0 +1,97 @@
+# Benchmarking
+
+Reproducible CPU inference benchmarks via `cactus bench`. Runs locally by
+default; `--android` cross-builds for arm64-v8a and runs on a connected
+device over adb. The bundle is downloaded or converted automatically if
+missing.
+
+```
+cactus bench                                   # gemma-4-e2b-it, local
+cactus bench --android                         # same workload on a device
+cactus bench Qwen/Qwen3-0.6B --android
+cactus bench --prompt "Explain relativity:"    # chat-template path
+```
+
+Two workloads:
+
+- **Token mode** (default) — prefill exactly `--prefill-tokens` raw token
+  IDs (default 512 = 4 full 128-token prefill chunks, zero padding) +
+  `--decode-tokens` decode (default 32), via `cactus_benchmark_tokens`.
+  No tokenizer, template, or padding variance: quote these numbers.
+  Each round runs in a fresh process; the first (warmup) round is
+  discarded. Token IDs are synthetic (1000, 1001, …); keep
+  `--prefill-tokens` well below the model's vocabulary size.
+- **Prompt mode** (`--prompt`) — a full `cactus_complete` through the
+  chat-template path, greedy decoding, `--max-tokens` decode budget.
+  Each round runs a discarded in-process warmup generation first.
+
+Both runs always set `CACTUS_DISABLE_CLOUD_HANDOFF=1` (no routing probe or
+network fallback mid-generation) and `CACTUS_NO_CLOUD_TELE=1` (no telemetry
+uploads during rounds). Results are not valid without them.
+
+Reference numbers, token mode 512+32, default configuration
+(verified 2026-06-09):
+
+| Device | gemma-4-e2b-it (decode / prefill tps) |
+|---|---|
+| Pixel 10a (Tensor G4) | 10.8 / 75 |
+| Samsung S25 (8 Elite) | 22–24 / 200+ |
+
+## Device preparation (Android)
+
+1. **Charging.** `adb shell dumpsys battery` must show `AC/USB powered:
+   true` (the CLI warns if not). A discharging pack current-limits the
+   SoC and silently costs 15–45% decode. On Pixels, also check the
+   "Charge connected device" setting: it puts the USB port in source
+   role, so the phone discharges even with a cable attached.
+2. **Cool device, short batches.** Run 1 warmup + 3 measured rounds from
+   rest; back-to-back hot batches read 20–30% lower. Rest a few minutes
+   between batches.
+3. **Quiet device.** One benchmark per device at a time; no other adb
+   workloads during a run. This applies equally to local runs: a
+   concurrent compile costs ~4× on compute-bound prefill while leaving
+   memory-bound decode plausible-looking.
+
+## Core configuration
+
+Cactus's multi-core scheduling is deliberately **balanced, not
+maximum-throughput**: workers are pinned to performance cores and work is
+distributed so the device stays responsive and inside its sustained power
+envelope. Do not expect (or force) all-core saturation — on phones that
+trades thermals and UX for little gain, and on asymmetric SoCs it can lose
+outright.
+
+- **Default (mixed-core)** — no extra setup; this is the configuration
+  behind the reference table. Best for ~2B-class models (gemma-4-e2b).
+- **Single-core (prime only)** — `--cpu-mask <hex>` (Android only), e.g.
+  `--cpu-mask 80` for cpu7 on a 1-prime + 3-mid + 4-little SoC. Small
+  models whose decode is bound by single-stream memory bandwidth can run
+  *faster* this way: on Pixel 10a at the 512+32 spec, qwen3-0.6b decodes
+  ~26 tps single-prime vs ~18 mixed-core, while gemma-4-e2b is better
+  mixed (10.8 vs ~5.6). Report which configuration a number came from.
+
+## Interpreting prefill numbers
+
+The transpiled prefill graph processes fixed 128-token chunks; prompts are
+padded up to a chunk multiple. Prompt-mode runs with short prompts (e.g. a
+~30-token prompt) therefore understate true prefill throughput by ≈
+chunk/prompt (~4.3×). Token mode uses exact multiples of full chunks with
+zero padding — quote its prefill numbers.
+
+## If your numbers come out low: check bundle vintage
+
+Bundles converted with the **current** converter need none of this. If you
+are benchmarking artifacts produced by an **older release tag or converter
+snapshot**, two known issues silently depress results:
+
+- **Row-major LM head.** Older converters stored orthogonal CQ4 output
+  heads row-major (header flags=2) instead of 4-row-interleaved (flags=6);
+  that path costs ~2.4× more per byte — measured −35% decode on Pixel,
+  −30% on Samsung. Check the flags word (bytes 4–7 of the `.weights`
+  file); fix without reconversion:
+  `cd python && python3 -m cactus.convert.interleave_orthogonal_cq4 IN.weights OUT.weights`
+  Interleaved artifacts require an engine from 2026-05-26 or later.
+- **Pre-chunked-prefill text bundles.** Text-model bundles converted
+  before chunked prefill became the default show low long-prompt prefill
+  despite healthy short-prompt rates (seen with a June-2026 qwen3 bundle:
+  ~21–38 tps at 512 tokens). Re-convert with the current converter.

python/cactus/cli/__init__.py

@@ -27,6 +27,7 @@
 from .list import cmd_list
 
 from .auth import cmd_auth
+from .bench import cmd_bench
 from .clean import cmd_clean
 
 
@@ -177,6 +178,15 @@ def create_parser():
     --android                          run on connected Android
     --enable-telemetry                 send cloud telemetry (off by default)
 
+  cactus bench [model]                 benchmark inference throughput
+    --android                          run on connected Android (default: local)
+    --prefill-tokens <n>               exact prefill length (default: 512)
+    --decode-tokens <n>                decode budget per round (default: 32)
+    --rounds <n>                       measured rounds (default: 3)
+    --prompt <text>                    benchmark the chat path instead
+    --cpu-mask <hex>                   taskset core mask (Android only)
+    --output <csv>                     write per-round results
+
   cactus clean                         delete build artifacts, weights, venv
   cactus --help                        show this help
 
@@ -294,6 +304,30 @@ def create_parser():
     test_parser.add_argument("--enable-telemetry", action="store_true",
                              help="Enable cloud telemetry (disabled by default in tests)")
 
+    bench_parser = subparsers.add_parser("bench",
+                                         help="Benchmark inference throughput (local by default, --android for a device)")
+    bench_parser.add_argument("model_id", nargs="?", default=DEFAULT_MODEL_ID,
+                              type=_hf_id_or_path,
+                              help=f"HF model ID or bundle path (default: {DEFAULT_MODEL_ID})")
+    bench_parser.add_argument("--android", action="store_true",
+                              help="Build for arm64-v8a and run on a connected Android device")
+    bench_parser.add_argument("--serial", default=None,
+                              help="adb device serial (required with multiple devices)")
+    bench_parser.add_argument("--prefill-tokens", type=_positive_int, default=512,
+                              help="Exact prefill length in tokens (default: 512; use multiples of 128)")
+    bench_parser.add_argument("--decode-tokens", type=_positive_int, default=32,
+                              help="Decode budget per round (default: 32)")
+    bench_parser.add_argument("--rounds", type=_positive_int, default=3,
+                              help="Measured rounds after 1 warmup (default: 3)")
+    bench_parser.add_argument("--prompt", default=None,
+                              help="Benchmark the chat-template path with this prompt instead of raw token IDs")
+    bench_parser.add_argument("--max-tokens", type=_positive_int, default=32,
+                              help="Decode budget per round in prompt mode (default: 32)")
+    bench_parser.add_argument("--cpu-mask", default=None,
+                              help="taskset hex mask for core-restricted runs (Android only)")
+    bench_parser.add_argument("--output", default=None,
+                              help="Write per-round results to a CSV file")
+
     auth_parser = subparsers.add_parser("auth", help="Manage cloud API key")
     auth_parser.add_argument("--clear", action="store_true",
                              help="Remove the saved API key")
@@ -404,13 +438,14 @@ def create_parser():
     "list":       cmd_list,
 
     "auth":           cmd_auth,
+    "bench":          cmd_bench,
     "clean":          cmd_clean,
     "convert":        cmd_convert,
     "transpile":      cmd_transpile,
 }
 
 
-_REPO_ONLY = {"build", "test", "clean"}
+_REPO_ONLY = {"bench", "build", "test", "clean"}
 
 
 def main():