diff --git a/cactus-engine/tests/CMakeLists.txt b/cactus-engine/tests/CMakeLists.txt index 804fa319c..91fdb9bb6 100644 --- a/cactus-engine/tests/CMakeLists.txt +++ b/cactus-engine/tests/CMakeLists.txt @@ -46,3 +46,6 @@ endif() add_executable(transcribe transcribe.cpp) target_link_libraries(transcribe PRIVATE cactus_engine) + +add_executable(llm_bench llm_bench.cpp) +target_link_libraries(llm_bench PRIVATE cactus_engine) diff --git a/cactus-engine/tests/android/CMakeLists.txt b/cactus-engine/tests/android/CMakeLists.txt index 9ff1fc7c2..8d6c7cbd6 100644 --- a/cactus-engine/tests/android/CMakeLists.txt +++ b/cactus-engine/tests/android/CMakeLists.txt @@ -18,6 +18,9 @@ foreach(TEST_FILE ${TEST_SOURCES}) target_link_libraries(${TEST_NAME} PRIVATE cactus_engine) endforeach() +add_executable(llm_bench ${TESTS_DIR}/llm_bench.cpp) +target_link_libraries(llm_bench PRIVATE cactus_engine) + if(TARGET test_curl AND EXISTS "${CACTUS_CURL_ROOT}/include/curl/curl.h") target_include_directories(test_curl PRIVATE "${CACTUS_CURL_ROOT}/include") endif() diff --git a/cactus-engine/tests/llm_bench.cpp b/cactus-engine/tests/llm_bench.cpp new file mode 100644 index 000000000..4c169e88f --- /dev/null +++ b/cactus-engine/tests/llm_bench.cpp @@ -0,0 +1,121 @@ +#include "cactus_engine.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +std::vector load_tokens(const std::string& path) { + std::ifstream in(path); + if (!in) { + throw std::runtime_error("failed to open token input: " + path); + } + std::string text((std::istreambuf_iterator(in)), std::istreambuf_iterator()); + std::vector tokens; + std::stringstream ss(text); + std::string item; + while (std::getline(ss, item, ',')) { + if (item.empty()) continue; + tokens.push_back(static_cast(std::stoul(item))); + } + if (tokens.empty()) { + throw std::runtime_error("token input is empty: " + path); + } + return tokens; +} + +std::string escape_json(const std::string& s) { + std::string out; + out.reserve(s.size() + 16); + for (char c : s) { + switch (c) { + case '"': out += "\\\""; break; + case '\\': out += "\\\\"; break; + case '\n': out += "\\n"; break; + case '\r': out += "\\r"; break; + case '\t': out += "\\t"; break; + default: out += c; break; + } + } + return out; +} + +std::string complete_options(size_t max_tokens) { + return R"({"max_tokens": )" + std::to_string(max_tokens) + + R"(, "temperature": 0.0, "top_p": 0.0, "top_k": 1, "telemetry_enabled": false,)" + R"( "stop_sequences": ["<|im_end|>", ""]})"; +} + +int usage(const char* prog) { + std::cerr << "Usage: " << prog << " --tokens --decode \n" + << " " << prog << " --prompt --max-tokens [--warmup]\n"; + return 2; +} + +} // namespace + +int main(int argc, char** argv) { + if (argc < 2) return usage(argv[0]); + std::string model_dir = argv[1]; + std::string tokens_path; + std::string prompt; + size_t decode_tokens = 32; + size_t max_tokens = 32; + bool warmup = false; + + for (int i = 2; i < argc; i++) { + std::string arg = argv[i]; + if (arg == "--tokens" && i + 1 < argc) tokens_path = argv[++i]; + else if (arg == "--decode" && i + 1 < argc) decode_tokens = std::stoul(argv[++i]); + else if (arg == "--prompt" && i + 1 < argc) prompt = argv[++i]; + else if (arg == "--max-tokens" && i + 1 < argc) max_tokens = std::stoul(argv[++i]); + else if (arg == "--warmup") warmup = true; + else return usage(argv[0]); + } + if (tokens_path.empty() == prompt.empty()) return usage(argv[0]); + + try { + cactus_log_set_level(2); + cactus_model_t model = cactus_init(model_dir.c_str(), nullptr, false); + if (!model) { + const char* error = cactus_get_last_error(); + std::cerr << (error ? error : "cactus_init failed") << "\n"; + return 1; + } + + std::vector buffer(65536); + bool ok; + if (!tokens_path.empty()) { + auto tokens = load_tokens(tokens_path); + int rc = cactus_benchmark_tokens(model, tokens.data(), tokens.size(), + decode_tokens, buffer.data(), buffer.size()); + ok = rc >= 0; + } else { + std::string messages = R"([{"role": "user", "content": ")" + escape_json(prompt) + R"("}])"; + if (warmup) { + cactus_complete(model, messages.c_str(), buffer.data(), buffer.size(), + complete_options(16).c_str(), nullptr, nullptr, nullptr, nullptr, 0); + cactus_reset(model); + } + int rc = cactus_complete(model, messages.c_str(), buffer.data(), buffer.size(), + complete_options(max_tokens).c_str(), nullptr, nullptr, nullptr, nullptr, 0); + ok = rc > 0; + } + cactus_destroy(model); + if (!ok) { + const char* error = cactus_get_last_error(); + std::cerr << (error ? error : buffer.data()) << "\n"; + return 1; + } + std::cout << buffer.data() << "\n"; + return 0; + } catch (const std::exception& exc) { + std::cerr << "Error: " << exc.what() << "\n"; + return 1; + } +} diff --git a/docs/benchmarking.md b/docs/benchmarking.md new file mode 100644 index 000000000..6a696d213 --- /dev/null +++ b/docs/benchmarking.md @@ -0,0 +1,97 @@ +# Benchmarking + +Reproducible CPU inference benchmarks via `cactus bench`. Runs locally by +default; `--android` cross-builds for arm64-v8a and runs on a connected +device over adb. The bundle is downloaded or converted automatically if +missing. + +``` +cactus bench # gemma-4-e2b-it, local +cactus bench --android # same workload on a device +cactus bench Qwen/Qwen3-0.6B --android +cactus bench --prompt "Explain relativity:" # chat-template path +``` + +Two workloads: + +- **Token mode** (default) — prefill exactly `--prefill-tokens` raw token + IDs (default 512 = 4 full 128-token prefill chunks, zero padding) + + `--decode-tokens` decode (default 32), via `cactus_benchmark_tokens`. + No tokenizer, template, or padding variance: quote these numbers. + Each round runs in a fresh process; the first (warmup) round is + discarded. Token IDs are synthetic (1000, 1001, …); keep + `--prefill-tokens` well below the model's vocabulary size. +- **Prompt mode** (`--prompt`) — a full `cactus_complete` through the + chat-template path, greedy decoding, `--max-tokens` decode budget. + Each round runs a discarded in-process warmup generation first. + +Both runs always set `CACTUS_DISABLE_CLOUD_HANDOFF=1` (no routing probe or +network fallback mid-generation) and `CACTUS_NO_CLOUD_TELE=1` (no telemetry +uploads during rounds). Results are not valid without them. + +Reference numbers, token mode 512+32, default configuration +(verified 2026-06-09): + +| Device | gemma-4-e2b-it (decode / prefill tps) | +|---|---| +| Pixel 10a (Tensor G4) | 10.8 / 75 | +| Samsung S25 (8 Elite) | 22–24 / 200+ | + +## Device preparation (Android) + +1. **Charging.** `adb shell dumpsys battery` must show `AC/USB powered: + true` (the CLI warns if not). A discharging pack current-limits the + SoC and silently costs 15–45% decode. On Pixels, also check the + "Charge connected device" setting: it puts the USB port in source + role, so the phone discharges even with a cable attached. +2. **Cool device, short batches.** Run 1 warmup + 3 measured rounds from + rest; back-to-back hot batches read 20–30% lower. Rest a few minutes + between batches. +3. **Quiet device.** One benchmark per device at a time; no other adb + workloads during a run. This applies equally to local runs: a + concurrent compile costs ~4× on compute-bound prefill while leaving + memory-bound decode plausible-looking. + +## Core configuration + +Cactus's multi-core scheduling is deliberately **balanced, not +maximum-throughput**: workers are pinned to performance cores and work is +distributed so the device stays responsive and inside its sustained power +envelope. Do not expect (or force) all-core saturation — on phones that +trades thermals and UX for little gain, and on asymmetric SoCs it can lose +outright. + +- **Default (mixed-core)** — no extra setup; this is the configuration + behind the reference table. Best for ~2B-class models (gemma-4-e2b). +- **Single-core (prime only)** — `--cpu-mask ` (Android only), e.g. + `--cpu-mask 80` for cpu7 on a 1-prime + 3-mid + 4-little SoC. Small + models whose decode is bound by single-stream memory bandwidth can run + *faster* this way: on Pixel 10a at the 512+32 spec, qwen3-0.6b decodes + ~26 tps single-prime vs ~18 mixed-core, while gemma-4-e2b is better + mixed (10.8 vs ~5.6). Report which configuration a number came from. + +## Interpreting prefill numbers + +The transpiled prefill graph processes fixed 128-token chunks; prompts are +padded up to a chunk multiple. Prompt-mode runs with short prompts (e.g. a +~30-token prompt) therefore understate true prefill throughput by ≈ +chunk/prompt (~4.3×). Token mode uses exact multiples of full chunks with +zero padding — quote its prefill numbers. + +## If your numbers come out low: check bundle vintage + +Bundles converted with the **current** converter need none of this. If you +are benchmarking artifacts produced by an **older release tag or converter +snapshot**, two known issues silently depress results: + +- **Row-major LM head.** Older converters stored orthogonal CQ4 output + heads row-major (header flags=2) instead of 4-row-interleaved (flags=6); + that path costs ~2.4× more per byte — measured −35% decode on Pixel, + −30% on Samsung. Check the flags word (bytes 4–7 of the `.weights` + file); fix without reconversion: + `cd python && python3 -m cactus.convert.interleave_orthogonal_cq4 IN.weights OUT.weights` + Interleaved artifacts require an engine from 2026-05-26 or later. +- **Pre-chunked-prefill text bundles.** Text-model bundles converted + before chunked prefill became the default show low long-prompt prefill + despite healthy short-prompt rates (seen with a June-2026 qwen3 bundle: + ~21–38 tps at 512 tokens). Re-convert with the current converter. diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index e937c41ca..c11dd2868 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -27,6 +27,7 @@ from .list import cmd_list from .auth import cmd_auth +from .bench import cmd_bench from .clean import cmd_clean @@ -177,6 +178,15 @@ def create_parser(): --android run on connected Android --enable-telemetry send cloud telemetry (off by default) + cactus bench [model] benchmark inference throughput + --android run on connected Android (default: local) + --prefill-tokens exact prefill length (default: 512) + --decode-tokens decode budget per round (default: 32) + --rounds measured rounds (default: 3) + --prompt benchmark the chat path instead + --cpu-mask taskset core mask (Android only) + --output write per-round results + cactus clean delete build artifacts, weights, venv cactus --help show this help @@ -294,6 +304,30 @@ def create_parser(): test_parser.add_argument("--enable-telemetry", action="store_true", help="Enable cloud telemetry (disabled by default in tests)") + bench_parser = subparsers.add_parser("bench", + help="Benchmark inference throughput (local by default, --android for a device)") + bench_parser.add_argument("model_id", nargs="?", default=DEFAULT_MODEL_ID, + type=_hf_id_or_path, + help=f"HF model ID or bundle path (default: {DEFAULT_MODEL_ID})") + bench_parser.add_argument("--android", action="store_true", + help="Build for arm64-v8a and run on a connected Android device") + bench_parser.add_argument("--serial", default=None, + help="adb device serial (required with multiple devices)") + bench_parser.add_argument("--prefill-tokens", type=_positive_int, default=512, + help="Exact prefill length in tokens (default: 512; use multiples of 128)") + bench_parser.add_argument("--decode-tokens", type=_positive_int, default=32, + help="Decode budget per round (default: 32)") + bench_parser.add_argument("--rounds", type=_positive_int, default=3, + help="Measured rounds after 1 warmup (default: 3)") + bench_parser.add_argument("--prompt", default=None, + help="Benchmark the chat-template path with this prompt instead of raw token IDs") + bench_parser.add_argument("--max-tokens", type=_positive_int, default=32, + help="Decode budget per round in prompt mode (default: 32)") + bench_parser.add_argument("--cpu-mask", default=None, + help="taskset hex mask for core-restricted runs (Android only)") + bench_parser.add_argument("--output", default=None, + help="Write per-round results to a CSV file") + auth_parser = subparsers.add_parser("auth", help="Manage cloud API key") auth_parser.add_argument("--clear", action="store_true", help="Remove the saved API key") @@ -404,13 +438,14 @@ def create_parser(): "list": cmd_list, "auth": cmd_auth, + "bench": cmd_bench, "clean": cmd_clean, "convert": cmd_convert, "transpile": cmd_transpile, } -_REPO_ONLY = {"build", "test", "clean"} +_REPO_ONLY = {"bench", "build", "test", "clean"} def main(): diff --git a/python/cactus/cli/bench.py b/python/cactus/cli/bench.py new file mode 100644 index 000000000..e2f575ef7 --- /dev/null +++ b/python/cactus/cli/bench.py @@ -0,0 +1,243 @@ +import csv +import glob +import json +import os +import shlex +import subprocess +import sys +import tempfile +from pathlib import Path + +from .common import BLUE, PROJECT_ROOT, RED, YELLOW, print_color +from .test import _bundle_dir, _ensure_bundle + +DEVICE_DIR = "/data/local/tmp/cactus-bench" +BENCH_ENV = {"CACTUS_DISABLE_CLOUD_HANDOFF": "1", "CACTUS_NO_CLOUD_TELE": "1"} + + +def _cmake_build(src, *defines): + build_dir = src / "build" + subprocess.run( + ["cmake", "-S", str(src), "-B", str(build_dir), *defines, + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_RULE_MESSAGES=OFF", "-DCMAKE_VERBOSE_MAKEFILE=OFF"], + check=True, stdout=subprocess.DEVNULL) + subprocess.run( + ["cmake", "--build", str(build_dir), "--target", "llm_bench", + "-j", str(os.cpu_count() or 4)], + check=True) + return build_dir / "llm_bench" + + +def _build_local(): + return _cmake_build(PROJECT_ROOT / "cactus-engine" / "tests") + + +def _find_ndk(): + candidates = [] + if os.environ.get("ANDROID_NDK_HOME"): + candidates.append(os.environ["ANDROID_NDK_HOME"]) + roots = [] + if os.environ.get("ANDROID_HOME"): + roots.append(f"{os.environ['ANDROID_HOME']}/ndk") + roots.append(os.path.expanduser("~/Library/Android/sdk/ndk")) + for root in roots: + if os.path.isdir(root): + candidates.extend(f"{root}/{v}" for v in sorted(os.listdir(root), reverse=True)) + candidates.extend(sorted(glob.glob( + "/opt/homebrew/Caskroom/android-ndk/*/AndroidNDK*.app/Contents/NDK"), reverse=True)) + candidates.append("/opt/homebrew/share/android-ndk") + for ndk in candidates: + if os.path.isfile(f"{ndk}/build/cmake/android.toolchain.cmake"): + return ndk + return None + + +def _build_android(): + ndk = _find_ndk() + if not ndk: + print_color(RED, "Android NDK not found. Set ANDROID_NDK_HOME.") + raise SystemExit(1) + curl_root = os.environ.get("CACTUS_CURL_ROOT", + str(PROJECT_ROOT / "cactus-engine" / "libs" / "curl")) + return _cmake_build( + PROJECT_ROOT / "cactus-engine" / "tests" / "android", + f"-DCMAKE_TOOLCHAIN_FILE={ndk}/build/cmake/android.toolchain.cmake", + "-DANDROID_ABI=arm64-v8a", + f"-DANDROID_PLATFORM={os.environ.get('ANDROID_PLATFORM', 'android-21')}", + f"-DCACTUS_CURL_ROOT={curl_root}") + + +def _adb(serial): + return ["adb", "-s", serial] if serial else ["adb"] + + +def _select_device(serial): + out = subprocess.run(["adb", "devices"], check=True, capture_output=True, text=True).stdout + devices = [line.split("\t")[0] for line in out.splitlines() if line.endswith("\tdevice")] + if serial: + if serial not in devices: + print_color(RED, f"Device {serial} not found. Connected: {', '.join(devices) or 'none'}") + raise SystemExit(1) + return serial + if len(devices) == 1: + return devices[0] + if not devices: + print_color(RED, "No Android device connected.") + else: + print_color(RED, f"Multiple devices connected, pass --serial: {', '.join(devices)}") + raise SystemExit(1) + + +def _warn_if_not_charging(adb): + out = subprocess.run(adb + ["shell", "dumpsys", "battery"], + capture_output=True, text=True).stdout + if not any(f"{kind} powered: true" in out for kind in ("AC", "USB", "Wireless")): + print_color(YELLOW, + "WARNING: device not charging - numbers will be power-limited " + "(see docs/benchmarking.md)") + + +def _push_weights(adb, name, local_dir): + if subprocess.run(adb + ["shell", "test", "-d", f"{DEVICE_DIR}/weights/{name}"], + capture_output=True).returncode == 0: + return + print_color(BLUE, f"Pushing weights: {name}") + subprocess.run(adb + ["push", str(local_dir), f"{DEVICE_DIR}/weights/"], + check=True, capture_output=True) + + +def _driver_args(args, device_tokens_path=None): + if args.prompt is not None: + return ["--prompt", args.prompt, "--max-tokens", str(args.max_tokens), "--warmup"] + return ["--tokens", device_tokens_path, "--decode", str(args.decode_tokens)] + + +def _parse_result(stdout): + try: + result = json.loads(stdout.strip()) + except json.JSONDecodeError: + print_color(RED, f"Unparseable benchmark output:\n{stdout.strip()}") + raise SystemExit(1) + return result + + +def _write_tokens_csv(prefill_tokens): + f = tempfile.NamedTemporaryFile("w", suffix=".csv", delete=False) + f.write(",".join(str(1000 + i) for i in range(prefill_tokens))) + f.close() + return f.name + + +def _exec_round(cmd, env=None): + proc = subprocess.run(cmd, env=env, capture_output=True, text=True) + if proc.returncode != 0: + print_color(RED, proc.stderr.strip() or proc.stdout.strip() or "benchmark run failed") + raise SystemExit(1) + if proc.stderr.strip(): + print(proc.stderr.strip(), file=sys.stderr) + return proc.stdout + + +def _run_rounds(args, run_round): + results = [] + if args.prompt is None: + run_round() + for i in range(args.rounds): + result = _parse_result(run_round()) + ttft = f", ttft {result['time_to_first_token_ms']:.0f}ms" if args.prompt is not None else "" + print(f"round {i + 1}: prefill {result['prefill_tps']:.1f} tps, " + f"decode {result['decode_tps']:.1f} tps{ttft}") + results.append(result) + return results + + +def _report(args, model_name, results): + if not results: + return + mean_prefill = sum(r["prefill_tps"] for r in results) / len(results) + mean_decode = sum(r["decode_tps"] for r in results) / len(results) + print_color(BLUE, f"mean over {len(results)} rounds: " + f"prefill {mean_prefill:.1f} tps, decode {mean_decode:.1f} tps") + if args.output: + with open(args.output, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(["model", "mode", "round", "prefill_tps", "decode_tps", + "time_to_first_token_ms"]) + mode = "prompt" if args.prompt is not None else "tokens" + for i, r in enumerate(results): + writer.writerow([model_name, mode, i + 1, r["prefill_tps"], + r["decode_tps"], r.get("time_to_first_token_ms", "")]) + print(f"Results written to: {args.output}") + + +def _bench_local(args, bundle): + binary = _build_local() + tokens_csv = None if args.prompt is not None else _write_tokens_csv(args.prefill_tokens) + cmd = [str(binary), str(bundle)] + _driver_args(args, tokens_csv) + env = {**os.environ, **BENCH_ENV} + try: + return _run_rounds(args, lambda: _exec_round(cmd, env)) + finally: + if tokens_csv: + os.unlink(tokens_csv) + + +def _bench_android(args, bundle): + binary = _build_android() + adb = _adb(_select_device(args.serial)) + _warn_if_not_charging(adb) + subprocess.run(adb + ["shell", "mkdir", "-p", f"{DEVICE_DIR}/weights"], check=True) + subprocess.run(adb + ["push", str(binary), f"{DEVICE_DIR}/llm_bench"], + check=True, capture_output=True) + subprocess.run(adb + ["shell", "chmod", "+x", f"{DEVICE_DIR}/llm_bench"], check=True) + _push_weights(adb, bundle.name, bundle) + + device_tokens = None + if args.prompt is None: + tokens_csv = _write_tokens_csv(args.prefill_tokens) + device_tokens = f"{DEVICE_DIR}/tokens.csv" + subprocess.run(adb + ["push", tokens_csv, device_tokens], + check=True, capture_output=True) + os.unlink(tokens_csv) + + runner = (f"taskset {shlex.quote(args.cpu_mask)} ./llm_bench" if args.cpu_mask + else "./llm_bench") + driver = " ".join(shlex.quote(a) for a in _driver_args(args, device_tokens)) + env = " ".join(f"{k}={v}" for k, v in BENCH_ENV.items()) + shell_cmd = (f"cd {DEVICE_DIR} && {env} {runner} " + f"{shlex.quote(f'weights/{bundle.name}')} {driver}") + return _run_rounds(args, lambda: _exec_round(adb + ["shell", shell_cmd])) + + +def cmd_bench(args): + if args.cpu_mask and not args.android: + print_color(RED, "--cpu-mask requires --android (taskset runs on the device)") + return 2 + if args.prompt is not None and not args.prompt.strip(): + print_color(RED, "--prompt must not be empty") + return 2 + + candidate = Path(args.model_id) + if (candidate / "components" / "manifest.json").exists(): + bundle = candidate + else: + if not _ensure_bundle(args.model_id): + return 1 + bundle = _bundle_dir(args.model_id) + + if args.prompt is not None: + workload = f"prompt + {args.max_tokens} decode, {args.rounds} rounds (per-round warmup)" + else: + workload = (f"{args.prefill_tokens}-token prefill + {args.decode_tokens} decode, " + f"1 warmup + {args.rounds} rounds") + target = "android" if args.android else "local" + print_color(BLUE, f"==> {bundle.name}: {workload} ({target})") + + try: + results = _bench_android(args, bundle) if args.android else _bench_local(args, bundle) + except subprocess.CalledProcessError as exc: + print_color(RED, f"command failed ({exc.returncode}): {' '.join(map(str, exc.cmd))}") + return 1 + _report(args, bundle.name, results) + return 0