diff --git a/python/README.md b/python/README.md index 76e6da126..722a14a25 100644 --- a/python/README.md +++ b/python/README.md @@ -123,6 +123,87 @@ if result["cloud_handoff"]: } ``` +## Benchmarking + +Use `cactus benchmark` when comparing inference changes across devices, +quantization settings, or long-context profiles. It runs warmups, resets the KV +cache between runs by default, records every trial as JSONL, and writes a grouped +summary with mean, p50, p95, min, and max values for TTFT, total latency, +prefill throughput, decode throughput, RAM, and token counts. + +```bash +cactus benchmark LiquidAI/LFM2-VL-450M \ + --profile short_chat \ + --profile long_prefill \ + --sweep-token-counts 512,2048,4096 \ + --iterations 5 \ + --warmup 2 \ + --max-tokens 128 \ + --output runs/lfm2_vl.jsonl \ + --summary-json runs/lfm2_vl_summary.json \ + --markdown-report runs/lfm2_vl_report.md +``` + +Custom profiles can be checked into a PR so reviewers can rerun the exact same +prompt shapes: + +```json +{ + "profiles": [ + { + "name": "long_context_decode", + "messages": [ + {"role": "user", "content": "Repeat your benchmark context here..."} + ], + "options": {"max_tokens": 128, "temperature": 0.0} + } + ] +} +``` + +```bash +cactus benchmark ./weights/lfm2-vl --profiles-file profiles.json --output bench.jsonl +``` + +Compare two summary files to catch regressions before sending a runtime change +for review: + +```bash +cactus benchmark \ + --compare runs/main_summary.json runs/pr_summary.json \ + --compare-metric time_to_first_token_ms \ + --compare-metric decode_tps \ + --regression-threshold 5 \ + --fail-on-regression \ + --markdown-report runs/regression_report.md +``` + +Use a budget file when CI or device-lab runs need different tolerances per +metric or profile. For example, a long-context sweep can use p95 decode +throughput while RAM gets a tighter global budget: + +```json +{ + "threshold_pct": 5, + "metrics": { + "ram_usage_mb": {"threshold_pct": 2} + }, + "profiles": { + "context_sweep_4096": { + "decode_tps": {"stat": "p95", "threshold_pct": 3}, + "time_to_first_token_ms": {"threshold_pct": 8} + } + } +} +``` + +```bash +cactus benchmark \ + --compare runs/main_summary.json runs/pr_summary.json \ + --budget-json runs/mobile_budgets.json \ + --fail-on-regression +``` + ### Prefill Pre-processes input text and populates the KV cache without generating output tokens. This reduces latency for subsequent calls to `cactus_complete`. diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index ac8708fc6..aa3866f5e 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -7,14 +7,6 @@ DEFAULT_TRANSCRIPTION_MODEL_ID, SUPPORTED_PLATFORMS, ) -from .download import cmd_download - -_PLATFORM_CHOICES = ("cpu", *SUPPORTED_PLATFORMS) -_PLATFORM_HELP = ( - f"target accelerator: cpu = generic ARM (default); " - f"or one of: {', '.join(SUPPORTED_PLATFORMS) if SUPPORTED_PLATFORMS else '(none yet)'}" -) -_PLATFORM_PIPE = "|".join(_PLATFORM_CHOICES) from .compile import cmd_build from .serve import cmd_serve from .transcribe import cmd_transcribe @@ -22,16 +14,28 @@ from .convert import cmd_convert, cmd_transpile from .run import cmd_run from .list import cmd_list +from .benchmark import cmd_benchmark +from .download import cmd_download from .auth import cmd_auth from .clean import cmd_clean +_PLATFORM_CHOICES = ("cpu", *SUPPORTED_PLATFORMS) +_PLATFORM_HELP = ( + f"target accelerator: cpu = generic ARM (default); " + f"or one of: {', '.join(SUPPORTED_PLATFORMS) if SUPPORTED_PLATFORMS else '(none yet)'}" +) +_PLATFORM_PIPE = "|".join(_PLATFORM_CHOICES) + def _telemetry_parent(): """Args shared by commands that support telemetry toggle.""" p = argparse.ArgumentParser(add_help=False) - p.add_argument("--no-cloud-tele", action="store_true", - help="Disable cloud telemetry (write to cache only)") + p.add_argument( + "--no-cloud-tele", + action="store_true", + help="Disable cloud telemetry (write to cache only)", + ) return p @@ -73,8 +77,6 @@ def _hf_id_or_path(value): return v - - def create_parser(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, @@ -174,16 +176,38 @@ def create_parser(): --android run on connected Android --enable-telemetry send cloud telemetry (off by default) + ----------------------------------------------------------------- + + cactus benchmark run repeatable local inference profiles + writes per-run JSONL and summary JSON + + Optional flags: + --profile run one built-in or file profile + --profiles-file JSON list of benchmark profiles + --iterations measured runs per profile + --warmup warmup runs per profile + --output write raw JSONL measurements + --summary-json write grouped p50/p95 summary + --markdown-report write a human-readable report + --sweep-token-counts 512,2048 add synthetic context profiles + --compare base.json cand.json compare summaries for regressions + --budget-json per-profile regression thresholds + + ----------------------------------------------------------------- + cactus clean delete build artifacts + + ----------------------------------------------------------------- + cactus --help show this help ----------------------------------------------------------------- -""" +""", ) parser.add_argument("--version", action="version", version=f"cactus {__version__}") - subparsers = parser.add_subparsers(dest='command') + subparsers = parser.add_subparsers(dest="command") subparsers.required = False for action in parser._actions: @@ -192,217 +216,525 @@ def create_parser(): parser._action_groups = [] - download_parser = subparsers.add_parser("download", - help="Download a pre-built bundle from huggingface.co/Cactus-Compute") - download_parser.add_argument("model_id", nargs="?", default=DEFAULT_MODEL_ID, - type=_hf_id_or_path, - help=f"HuggingFace model id (default: {DEFAULT_MODEL_ID})") - download_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, - help="CQ quantization bits (default: 4)") - download_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="cpu", - help=_PLATFORM_HELP) + download_parser = subparsers.add_parser( + "download", + help="Download a pre-built bundle from huggingface.co/Cactus-Compute", + ) + download_parser.add_argument( + "model_id", + nargs="?", + default=DEFAULT_MODEL_ID, + type=_hf_id_or_path, + help=f"HuggingFace model id (default: {DEFAULT_MODEL_ID})", + ) + download_parser.add_argument( + "--bits", + type=int, + choices=[1, 2, 3, 4], + default=4, + help="CQ quantization bits (default: 4)", + ) + download_parser.add_argument( + "--platform", choices=_PLATFORM_CHOICES, default="cpu", help=_PLATFORM_HELP + ) download_parser.add_argument("--token", help="HuggingFace token") build_parser = subparsers.add_parser("build", help="Build cactus libraries") build_group = build_parser.add_mutually_exclusive_group() - build_group.add_argument("--apple", action="store_true", - help="Build for Apple (iOS/macOS)") - build_group.add_argument("--android", action="store_true", - help="Build for Android") - build_group.add_argument("--python", action="store_true", - help="Build shared library for Python FFI") - - run_parser = subparsers.add_parser("run", help="Run a model (downloads bundle if needed)", - parents=[_telemetry_parent()]) - run_parser.add_argument("model_id", nargs="?", default=DEFAULT_MODEL_ID, - type=_hf_id_or_path, - help=f"HuggingFace model id or local bundle path (default: {DEFAULT_MODEL_ID})") - run_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, - help="CQ quantization bits (default: 4)") - run_parser.add_argument("--platform", choices=_PLATFORM_CHOICES, default="cpu", - help=_PLATFORM_HELP) + build_group.add_argument( + "--apple", action="store_true", help="Build for Apple (iOS/macOS)" + ) + build_group.add_argument("--android", action="store_true", help="Build for Android") + build_group.add_argument( + "--python", action="store_true", help="Build shared library for Python FFI" + ) + + run_parser = subparsers.add_parser( + "run", + help="Run a model (downloads bundle if needed)", + parents=[_telemetry_parent()], + ) + run_parser.add_argument( + "model_id", + nargs="?", + default=DEFAULT_MODEL_ID, + type=_hf_id_or_path, + help=f"HuggingFace model id or local bundle path (default: {DEFAULT_MODEL_ID})", + ) + run_parser.add_argument( + "--bits", + type=int, + choices=[1, 2, 3, 4], + default=4, + help="CQ quantization bits (default: 4)", + ) + run_parser.add_argument( + "--platform", choices=_PLATFORM_CHOICES, default="cpu", help=_PLATFORM_HELP + ) run_parser.add_argument("--token", help="HuggingFace token") - run_parser.add_argument("--reconvert", action="store_true", - help="Force local convert+transpile fallback") - run_parser.add_argument("--image", - help="Path to image file for VLM inference (attached to first message)") - run_parser.add_argument("--audio", - help="Path to audio file (WAV) for audio chat (attached to first message)") - run_parser.add_argument("--system", - help="System prompt to prepend to all messages") - run_parser.add_argument("--prompt", - help="Initial prompt to send immediately") - run_parser.add_argument("--input-ids", default=None, - help="Comma-separated token ids for transpiled causal-LM bundles") - run_parser.add_argument("--input-ids-file", default=None, - help="File containing token ids for transpiled causal-LM bundles") - run_parser.add_argument("--max-new-tokens", type=_positive_int, default=None, - help="Maximum tokens to generate for transpiled causal-LM bundles") - run_parser.add_argument("--result-json", default=None, - help="Optional path to save transpiled bundle results as JSON") - run_parser.add_argument("--thinking", action="store_true", - help="Enable thinking/reasoning for models that support it") - run_parser.add_argument("--no-cloud-handoff", action="store_true", - help="Disable automatic cloud handoff for this run") - run_parser.add_argument("--confidence-threshold", type=_unit_float, default=None, - help="Confidence threshold below which local completions may hand off to cloud") - run_parser.add_argument("--cloud-timeout-ms", type=_non_negative_int, default=None, - help="Maximum time to wait for cloud handoff before falling back locally") - - transcribe_parser = subparsers.add_parser("transcribe", help="Transcribe audio with a model", - parents=[_telemetry_parent()]) - transcribe_parser.add_argument("model_id", nargs="?", default=DEFAULT_TRANSCRIPTION_MODEL_ID, - type=_hf_id_or_path, - help=f"HuggingFace model id (default: {DEFAULT_TRANSCRIPTION_MODEL_ID})") - transcribe_parser.add_argument("--file", dest="audio_file", required=True, - help="Audio file to transcribe (WAV)") - transcribe_parser.add_argument("--language", default="en", - help="Language code (default: en)") + run_parser.add_argument( + "--reconvert", + action="store_true", + help="Force local convert+transpile fallback", + ) + run_parser.add_argument( + "--image", + help="Path to image file for VLM inference (attached to first message)", + ) + run_parser.add_argument( + "--audio", + help="Path to audio file (WAV) for audio chat (attached to first message)", + ) + run_parser.add_argument("--system", help="System prompt to prepend to all messages") + run_parser.add_argument("--prompt", help="Initial prompt to send immediately") + run_parser.add_argument( + "--input-ids", + default=None, + help="Comma-separated token ids for transpiled causal-LM bundles", + ) + run_parser.add_argument( + "--input-ids-file", + default=None, + help="File containing token ids for transpiled causal-LM bundles", + ) + run_parser.add_argument( + "--max-new-tokens", + type=_positive_int, + default=None, + help="Maximum tokens to generate for transpiled causal-LM bundles", + ) + run_parser.add_argument( + "--result-json", + default=None, + help="Optional path to save transpiled bundle results as JSON", + ) + run_parser.add_argument( + "--thinking", + action="store_true", + help="Enable thinking/reasoning for models that support it", + ) + run_parser.add_argument( + "--no-cloud-handoff", + action="store_true", + help="Disable automatic cloud handoff for this run", + ) + run_parser.add_argument( + "--confidence-threshold", + type=_unit_float, + default=None, + help="Confidence threshold below which local completions may hand off to cloud", + ) + run_parser.add_argument( + "--cloud-timeout-ms", + type=_non_negative_int, + default=None, + help="Maximum time to wait for cloud handoff before falling back locally", + ) + + transcribe_parser = subparsers.add_parser( + "transcribe", + help="Transcribe audio with a model", + parents=[_telemetry_parent()], + ) + transcribe_parser.add_argument( + "model_id", + nargs="?", + default=DEFAULT_TRANSCRIPTION_MODEL_ID, + type=_hf_id_or_path, + help=f"HuggingFace model id (default: {DEFAULT_TRANSCRIPTION_MODEL_ID})", + ) + transcribe_parser.add_argument( + "--file", + dest="audio_file", + required=True, + help="Audio file to transcribe (WAV)", + ) + transcribe_parser.add_argument( + "--language", default="en", help="Language code (default: en)" + ) transcribe_parser.add_argument("--token", help="HuggingFace token") - transcribe_parser.add_argument("--force-handoff", action="store_true", - help=argparse.SUPPRESS) - transcribe_parser.add_argument("--reconvert", action="store_true", - help="Force reconversion from source") - - serve_parser = subparsers.add_parser("serve", help="OpenAI-compatible local HTTP server") - serve_parser.add_argument("model", nargs="?", default=None, - type=_hf_id_or_path, - help="HuggingFace model id (e.g. openai/whisper-base) or bundle path") - serve_parser.add_argument("--host", default="127.0.0.1", - help="Bind address (default: 127.0.0.1)") - serve_parser.add_argument("--port", type=_port_int, default=8080, - help="Port (default: 8080)") + transcribe_parser.add_argument( + "--force-handoff", action="store_true", help=argparse.SUPPRESS + ) + transcribe_parser.add_argument( + "--reconvert", action="store_true", help="Force reconversion from source" + ) + + serve_parser = subparsers.add_parser( + "serve", help="OpenAI-compatible local HTTP server" + ) + serve_parser.add_argument( + "model", + nargs="?", + default=None, + type=_hf_id_or_path, + help="HuggingFace model id (e.g. openai/whisper-base) or bundle path", + ) + serve_parser.add_argument( + "--host", default="127.0.0.1", help="Bind address (default: 127.0.0.1)" + ) + serve_parser.add_argument( + "--port", type=_port_int, default=8080, help="Port (default: 8080)" + ) test_parser = subparsers.add_parser("test", help="Run the test suite") - test_parser.add_argument("--component", choices=COMPONENTS, default="all", - help="Component to test (default: all)") - test_parser.add_argument("--model", dest="model_id", default=None, - type=_hf_id_or_path, - help=f"HF model ID under test (default: {DEFAULT_MODEL_ID})") - test_parser.add_argument("--transcription-model", dest="transcription_model_id", default=None, - type=_hf_id_or_path, - help=f"HF transcription model ID under test (default: {DEFAULT_TRANSCRIPTION_MODEL_ID})") - test_parser.add_argument("--suite", default=None, - help="Run a single test suite by name; resolved across all components (e.g. llm → engine, performance → kernels + graph)") - test_parser.add_argument("--list", action="store_true", - help="List available components and engine tests, then exit") - test_parser.add_argument("--android", action="store_true", help="Run tests on Android") + test_parser.add_argument( + "--component", + choices=COMPONENTS, + default="all", + help="Component to test (default: all)", + ) + test_parser.add_argument( + "--model", + dest="model_id", + default=None, + type=_hf_id_or_path, + help=f"HF model ID under test (default: {DEFAULT_MODEL_ID})", + ) + test_parser.add_argument( + "--transcription-model", + dest="transcription_model_id", + default=None, + type=_hf_id_or_path, + help=f"HF transcription model ID under test (default: {DEFAULT_TRANSCRIPTION_MODEL_ID})", + ) + test_parser.add_argument( + "--suite", + default=None, + help="Run a single test suite by name; resolved across all components (e.g. llm → engine, performance → kernels + graph)", + ) + test_parser.add_argument( + "--list", + action="store_true", + help="List available components and engine tests, then exit", + ) + test_parser.add_argument( + "--android", action="store_true", help="Run tests on Android" + ) test_parser.add_argument("--ios", action="store_true", help="Run tests on iOS") - test_parser.add_argument("--enable-telemetry", action="store_true", - help="Enable cloud telemetry (disabled by default in tests)") + test_parser.add_argument( + "--enable-telemetry", + action="store_true", + help="Enable cloud telemetry (disabled by default in tests)", + ) + + benchmark_parser = subparsers.add_parser( + "benchmark", help="Run repeatable local inference benchmark profiles" + ) + benchmark_parser.add_argument( + "model_id", + nargs="?", + default=DEFAULT_MODEL_ID, + help=f"Model or prepared bundle to benchmark (default: {DEFAULT_MODEL_ID})", + ) + benchmark_parser.add_argument( + "--profile", + action="append", + help="Profile name to run. Repeat to select multiple profiles.", + ) + benchmark_parser.add_argument( + "--profiles-file", help="JSON file with benchmark profiles" + ) + benchmark_parser.add_argument( + "--iterations", + type=int, + default=3, + help="Measured runs per profile (default: 3)", + ) + benchmark_parser.add_argument( + "--warmup", type=int, default=1, help="Warmup runs per profile (default: 1)" + ) + benchmark_parser.add_argument( + "--max-tokens", + type=int, + default=64, + help="Generation token limit for benchmark runs (default: 64)", + ) + benchmark_parser.add_argument( + "--temperature", + type=float, + default=0.0, + help="Sampling temperature for benchmark runs (default: 0.0)", + ) + benchmark_parser.add_argument( + "--keep-cache", + action="store_true", + help="Do not reset the KV cache between runs", + ) + benchmark_parser.add_argument( + "--output", help="Path to write per-run JSONL measurements" + ) + benchmark_parser.add_argument( + "--summary-json", help="Path to write grouped summary JSON" + ) + benchmark_parser.add_argument( + "--markdown-report", help="Path to write a Markdown benchmark report" + ) + benchmark_parser.add_argument( + "--sweep-token-counts", + help="Comma-separated synthetic context lengths to add as profiles", + ) + benchmark_parser.add_argument( + "--compare", + nargs=2, + metavar=("BASELINE", "CANDIDATE"), + help="Compare two summary JSON files instead of running a benchmark", + ) + benchmark_parser.add_argument( + "--compare-metric", + action="append", + help="Metric to compare. Repeat to compare multiple metrics.", + ) + benchmark_parser.add_argument( + "--compare-stat", + default="p50", + choices=["mean", "p50", "p95", "min", "max"], + help="Summary statistic to compare (default: p50)", + ) + benchmark_parser.add_argument( + "--regression-threshold", + type=float, + default=5.0, + help="Allowed percent regression before flagging (default: 5.0)", + ) + benchmark_parser.add_argument( + "--budget-json", + help="JSON file with per-metric or per-profile regression budgets", + ) + benchmark_parser.add_argument( + "--fail-on-regression", + action="store_true", + help="Exit non-zero when --compare finds a regression", + ) + benchmark_parser.add_argument("--token", help="HuggingFace API token") + benchmark_parser.add_argument( + "--reconvert", action="store_true", help="Force conversion from source" + ) auth_parser = subparsers.add_parser("auth", help="Manage cloud API key") - auth_parser.add_argument("--clear", action="store_true", - help="Remove the saved API key") - auth_parser.add_argument("--status", action="store_true", - help="Show current key status") + auth_parser.add_argument( + "--clear", action="store_true", help="Remove the saved API key" + ) + auth_parser.add_argument( + "--status", action="store_true", help="Show current key status" + ) subparsers.add_parser("clean", help="Delete build artifacts") subparsers.add_parser("list", help="List local converted weights and bundles") - convert_parser = subparsers.add_parser("convert", - help="Convert HuggingFace weights to CQ format") - convert_parser.add_argument("model_id", type=_hf_id_or_path, - help="HuggingFace model id (e.g. openai/whisper-base)") - convert_parser.add_argument("output_dir", nargs="?", default=None, - help="Output directory (default: weights/)") - convert_parser.add_argument("--bits", type=int, choices=[1, 2, 3, 4], default=4, - help="CQ quantization bits (default: 4)") + convert_parser = subparsers.add_parser( + "convert", help="Convert HuggingFace weights to CQ format" + ) + convert_parser.add_argument( + "model_id", + type=_hf_id_or_path, + help="HuggingFace model id (e.g. openai/whisper-base)", + ) + convert_parser.add_argument( + "output_dir", + nargs="?", + default=None, + help="Output directory (default: weights/)", + ) + convert_parser.add_argument( + "--bits", + type=int, + choices=[1, 2, 3, 4], + default=4, + help="CQ quantization bits (default: 4)", + ) convert_parser.add_argument("--token", help="HuggingFace token") - convert_parser.add_argument("--lora", - help="Path or HF id of a LoRA adapter to merge before converting (requires `peft`)") - convert_parser.add_argument("--reconvert", action="store_true", - help="Force conversion from source") - - transpile_parser = subparsers.add_parser("transpile", - help="Build a runnable bundle from CQ weights") - transpile_parser.add_argument("model_id", type=_hf_id_or_path, - help="HuggingFace model id or local checkpoint path") - transpile_parser.add_argument("--weights-dir", - help="CQ weights directory (default: weights/)") - transpile_parser.add_argument("--task", default="auto", - choices=["auto", "causal_lm_logits", "multimodal_causal_lm_logits", - "ctc_logits", "encoder_hidden_states", - "seq2seq_transcription", "tdt_transcription"], - help="Transpile task (default: auto, from model config)") - transpile_parser.add_argument("--prompt", - help="Prompt for causal/multimodal shape capture") - transpile_parser.add_argument("--system-prompt", default=None, - help="System prompt for multimodal chat formats") - transpile_parser.add_argument("--enable-thinking", action="store_true", - help="Enable thinking markers when the prompt supports them") - transpile_parser.add_argument("--input-ids", default=None, - help="Comma-separated token ids for causal-LM shape capture") - transpile_parser.add_argument("--image-file", action="append", default=[], - help="Image for multimodal shape capture (repeatable)") - transpile_parser.add_argument("--audio-file", - help="Audio file (WAV) for audio/multimodal shape capture") - transpile_parser.add_argument("--max-new-tokens", type=_positive_int, default=None, - help="Decode context to preallocate for causal LM (default: 32)") - transpile_parser.add_argument("--component-pipeline", default="auto", choices=["auto", "on", "off"], - help="Split-component transpilation when supported (default: auto)") - transpile_parser.add_argument("--components", - help="Comma-separated component subset (e.g. vision_encoder,decoder)") - transpile_parser.add_argument("--torch-dtype", default=None, - choices=["float16", "float32", "bfloat16"], - help="Torch dtype for HF loading (default: float16)") - transpile_parser.add_argument("--token", default=None, - help="HuggingFace token for gated models (default: $HF_TOKEN)") - transpile_parser.add_argument("--trust-remote-code", action="store_true", - help="Pass trust_remote_code=True to HF loaders") - transpile_parser.add_argument("--local-files-only", action="store_true", - help="Require model/processor to already be local") - transpile_parser.add_argument("--allow-unconverted-weights", action="store_true", - help="Debug: transpile without CQ weights") - transpile_parser.add_argument("--execute-after-transpile", action="store_true", - help="Run a reference execution after lowering") - transpile_parser.add_argument("--artifact-dir", - help="Output directory (default: weights/)") - transpile_parser.add_argument("--graph-filename", default=None, - help="Saved graph filename (default: graph.cactus)") - transpile_parser.add_argument("--skip-reference-compare", action="store_true", - help="Skip PyTorch comparison (requires --execute-after-transpile)") - transpile_parser.add_argument("--no-fuse-rms-norm", action="store_true", - help="Disable RMSNorm fusion") - transpile_parser.add_argument("--no-fuse-rope", action="store_true", - help="Disable RoPE fusion") - transpile_parser.add_argument("--no-fuse-attention", action="store_true", - help="Disable attention fusion") - transpile_parser.add_argument("--no-fuse-attention-block", action="store_true", - help="Disable attention-block fusion") - transpile_parser.add_argument("--no-fuse-add-clipped", action="store_true", - help="Disable add-clipped fusion") - transpile_parser.add_argument("--no-fuse-gated-deltanet", action="store_true", - help="Disable gated DeltaNet fusion") - transpile_parser.add_argument("--npu", action="store_true", - help="Also emit CoreML .mlpackage(s) for Apple NPU encoders") - transpile_parser.add_argument("--npu-quantize", type=int, choices=[0, 4, 8], default=None, - help="Legacy: force both NPU encoders to same quant (0=fp16, 4=int4, 8=int8)") - transpile_parser.add_argument("--npu-audio-quantize", type=int, choices=[0, 4, 8], default=None, - help="NPU audio encoder quant: 0=fp16, 4=int4, 8=int8 (default: 8)") - transpile_parser.add_argument("--npu-vision-quantize", type=int, choices=[0, 4, 8], default=None, - help="NPU vision encoder quant: 0=fp16, 4=int4, 8=int8 (default: 0; int4 degrades Gemma4 vision)") - transpile_parser.add_argument("--cache-context-length", default=None, - help="KV cache context length for cached decode graphs (default: model config)") + convert_parser.add_argument( + "--lora", + help="Path or HF id of a LoRA adapter to merge before converting (requires `peft`)", + ) + convert_parser.add_argument( + "--reconvert", action="store_true", help="Force conversion from source" + ) - return parser + transpile_parser = subparsers.add_parser( + "transpile", help="Build a runnable bundle from CQ weights" + ) + transpile_parser.add_argument( + "model_id", + type=_hf_id_or_path, + help="HuggingFace model id or local checkpoint path", + ) + transpile_parser.add_argument( + "--weights-dir", help="CQ weights directory (default: weights/)" + ) + transpile_parser.add_argument( + "--task", + default="auto", + choices=[ + "auto", + "causal_lm_logits", + "multimodal_causal_lm_logits", + "ctc_logits", + "encoder_hidden_states", + "seq2seq_transcription", + "tdt_transcription", + ], + help="Transpile task (default: auto, from model config)", + ) + transpile_parser.add_argument( + "--prompt", help="Prompt for causal/multimodal shape capture" + ) + transpile_parser.add_argument( + "--system-prompt", + default=None, + help="System prompt for multimodal chat formats", + ) + transpile_parser.add_argument( + "--enable-thinking", + action="store_true", + help="Enable thinking markers when the prompt supports them", + ) + transpile_parser.add_argument( + "--input-ids", + default=None, + help="Comma-separated token ids for causal-LM shape capture", + ) + transpile_parser.add_argument( + "--image-file", + action="append", + default=[], + help="Image for multimodal shape capture (repeatable)", + ) + transpile_parser.add_argument( + "--audio-file", help="Audio file (WAV) for audio/multimodal shape capture" + ) + transpile_parser.add_argument( + "--max-new-tokens", + type=_positive_int, + default=None, + help="Decode context to preallocate for causal LM (default: 32)", + ) + transpile_parser.add_argument( + "--component-pipeline", + default="auto", + choices=["auto", "on", "off"], + help="Split-component transpilation when supported (default: auto)", + ) + transpile_parser.add_argument( + "--components", + help="Comma-separated component subset (e.g. vision_encoder,decoder)", + ) + transpile_parser.add_argument( + "--torch-dtype", + default=None, + choices=["float16", "float32", "bfloat16"], + help="Torch dtype for HF loading (default: float16)", + ) + transpile_parser.add_argument( + "--token", + default=None, + help="HuggingFace token for gated models (default: $HF_TOKEN)", + ) + transpile_parser.add_argument( + "--trust-remote-code", + action="store_true", + help="Pass trust_remote_code=True to HF loaders", + ) + transpile_parser.add_argument( + "--local-files-only", + action="store_true", + help="Require model/processor to already be local", + ) + transpile_parser.add_argument( + "--allow-unconverted-weights", + action="store_true", + help="Debug: transpile without CQ weights", + ) + transpile_parser.add_argument( + "--execute-after-transpile", + action="store_true", + help="Run a reference execution after lowering", + ) + transpile_parser.add_argument( + "--artifact-dir", help="Output directory (default: weights/)" + ) + transpile_parser.add_argument( + "--graph-filename", + default=None, + help="Saved graph filename (default: graph.cactus)", + ) + transpile_parser.add_argument( + "--skip-reference-compare", + action="store_true", + help="Skip PyTorch comparison (requires --execute-after-transpile)", + ) + transpile_parser.add_argument( + "--no-fuse-rms-norm", action="store_true", help="Disable RMSNorm fusion" + ) + transpile_parser.add_argument( + "--no-fuse-rope", action="store_true", help="Disable RoPE fusion" + ) + transpile_parser.add_argument( + "--no-fuse-attention", action="store_true", help="Disable attention fusion" + ) + transpile_parser.add_argument( + "--no-fuse-attention-block", + action="store_true", + help="Disable attention-block fusion", + ) + transpile_parser.add_argument( + "--no-fuse-add-clipped", action="store_true", help="Disable add-clipped fusion" + ) + transpile_parser.add_argument( + "--no-fuse-gated-deltanet", + action="store_true", + help="Disable gated DeltaNet fusion", + ) + transpile_parser.add_argument( + "--npu", + action="store_true", + help="Also emit CoreML .mlpackage(s) for Apple NPU encoders", + ) + transpile_parser.add_argument( + "--npu-quantize", + type=int, + choices=[0, 4, 8], + default=None, + help="Legacy: force both NPU encoders to same quant (0=fp16, 4=int4, 8=int8)", + ) + transpile_parser.add_argument( + "--npu-audio-quantize", + type=int, + choices=[0, 4, 8], + default=None, + help="NPU audio encoder quant: 0=fp16, 4=int4, 8=int8 (default: 8)", + ) + transpile_parser.add_argument( + "--npu-vision-quantize", + type=int, + choices=[0, 4, 8], + default=None, + help="NPU vision encoder quant: 0=fp16, 4=int4, 8=int8 (default: 0; int4 degrades Gemma4 vision)", + ) + transpile_parser.add_argument( + "--cache-context-length", + default=None, + help="KV cache context length for cached decode graphs (default: model config)", + ) + return parser _COMMANDS = { - "download": cmd_download, - "build": cmd_build, - "run": cmd_run, - "serve": cmd_serve, + "download": cmd_download, + "build": cmd_build, + "run": cmd_run, + "serve": cmd_serve, "transcribe": cmd_transcribe, - "test": cmd_test, - "list": cmd_list, - - "auth": cmd_auth, - "clean": cmd_clean, - "convert": cmd_convert, - "transpile": cmd_transpile, + "test": cmd_test, + "list": cmd_list, + "benchmark": cmd_benchmark, + "auth": cmd_auth, + "clean": cmd_clean, + "convert": cmd_convert, + "transpile": cmd_transpile, } @@ -416,7 +748,9 @@ def main(): args = parser.parse_args() if args.command in _REPO_ONLY and not is_repo_checkout(): - print(f"Error: `cactus {args.command}` requires a git clone of the cactus repository.") + print( + f"Error: `cactus {args.command}` requires a git clone of the cactus repository." + ) print("See: https://github.com/cactus-compute/cactus") sys.exit(1) @@ -428,5 +762,5 @@ def main(): sys.exit(1) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/python/cactus/cli/benchmark.py b/python/cactus/cli/benchmark.py new file mode 100644 index 000000000..e1178ad23 --- /dev/null +++ b/python/cactus/cli/benchmark.py @@ -0,0 +1,599 @@ +from __future__ import annotations + +import json +import platform +import statistics +import sys +import time +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +from .common import GREEN, RED, print_color + + +DEFAULT_PROFILES = [ + { + "name": "short_chat", + "messages": [ + { + "role": "user", + "content": "Summarize on-device inference in one sentence.", + } + ], + }, + { + "name": "long_prefill", + "messages": [ + { + "role": "user", + "content": " ".join( + [ + "Mobile inference benchmarks need stable prompt shapes, warmup runs,", + "and separate reporting for prefill latency and decode throughput.", + ] + * 24 + ), + } + ], + }, + { + "name": "tool_json", + "messages": [ + { + "role": "user", + "content": "Return JSON with device, prompt_tokens, decode_tokens, and one bottleneck.", + } + ], + "options": {"response_format": {"type": "json_object"}}, + }, +] + +METRIC_FIELDS = ( + "time_to_first_token_ms", + "total_time_ms", + "prefill_tps", + "decode_tps", + "ram_usage_mb", + "prefill_tokens", + "decode_tokens", + "total_tokens", +) + +LOWER_IS_BETTER = {"time_to_first_token_ms", "total_time_ms", "ram_usage_mb"} +DEFAULT_COMPARE_METRICS = ( + "time_to_first_token_ms", + "total_time_ms", + "prefill_tps", + "decode_tps", +) +DEFAULT_COMPARE_STAT = "p50" + + +@dataclass(frozen=True) +class BenchmarkProfile: + name: str + messages: list[dict[str, Any]] + options: dict[str, Any] + tools: list[dict[str, Any]] | None = None + + +def _load_profile_data(path: str | None) -> list[dict[str, Any]]: + if path is None: + return list(DEFAULT_PROFILES) + data = json.loads(Path(path).read_text(encoding="utf-8")) + if isinstance(data, dict): + data = data.get("profiles", []) + if not isinstance(data, list): + raise ValueError( + "Benchmark profile file must contain a list or a {'profiles': [...]} object" + ) + return data + + +def load_profiles( + path: str | None, names: set[str] | None, base_options: dict[str, Any] +) -> list[BenchmarkProfile]: + profiles = [] + for item in _load_profile_data(path): + if not isinstance(item, dict): + raise ValueError("Each benchmark profile must be an object") + name = str(item.get("name") or "").strip() + messages = item.get("messages") + if not name or not isinstance(messages, list): + raise ValueError("Each benchmark profile needs a name and messages list") + if names and name not in names: + continue + options = dict(base_options) + options.update(item.get("options") or {}) + profiles.append( + BenchmarkProfile( + name=name, + messages=messages, + options=options, + tools=item.get("tools"), + ) + ) + if not profiles: + selected = ", ".join(sorted(names or [])) or "all" + raise ValueError(f"No benchmark profiles selected: {selected}") + return profiles + + +def _parse_int_list(raw: str | None) -> list[int]: + if not raw: + return [] + values = [] + for part in raw.split(","): + part = part.strip() + if not part: + continue + value = int(part) + if value <= 0: + raise ValueError("Sweep token counts must be positive") + values.append(value) + return values + + +def build_sweep_profiles( + token_counts: list[int], base_options: dict[str, Any] +) -> list[BenchmarkProfile]: + profiles = [] + seed = "Cactus local inference regression profile. " + for count in token_counts: + words = (seed.split() * ((count // len(seed.split())) + 1))[:count] + profiles.append( + BenchmarkProfile( + name=f"context_sweep_{count}", + messages=[ + { + "role": "user", + "content": " ".join(words) + + "\n\nReturn one short sentence about the bottleneck.", + } + ], + options=dict(base_options), + ) + ) + return profiles + + +def collect_environment(model_id: str, bundle_dir: Path | str | None) -> dict[str, Any]: + try: + from cactus import __version__ as cactus_version + except Exception: + cactus_version = None + return { + "created_at": datetime.now(timezone.utc).isoformat(), + "model_id": model_id, + "bundle_dir": str(bundle_dir) if bundle_dir is not None else None, + "cactus_version": cactus_version, + "python": sys.version.split()[0], + "platform": platform.platform(), + "machine": platform.machine(), + "processor": platform.processor(), + } + + +def _percentile(values: list[float], pct: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = (len(ordered) - 1) * pct + lower = int(index) + upper = min(lower + 1, len(ordered) - 1) + if lower == upper: + return ordered[lower] + weight = index - lower + return ordered[lower] * (1 - weight) + ordered[upper] * weight + + +def summarize_results( + results: Iterable[dict[str, Any]], metadata: dict[str, Any] | None = None +) -> dict[str, Any]: + grouped: dict[str, list[dict[str, Any]]] = {} + for row in results: + if row.get("phase") == "measure" and row.get("success"): + grouped.setdefault(row["profile"], []).append(row) + + summaries = [] + for profile, rows in sorted(grouped.items()): + metrics: dict[str, dict[str, float]] = {} + for field in METRIC_FIELDS: + values = [ + float(row[field]) + for row in rows + if isinstance(row.get(field), (int, float)) + ] + if values: + metrics[field] = { + "mean": statistics.fmean(values), + "p50": _percentile(values, 0.50), + "p95": _percentile(values, 0.95), + "min": min(values), + "max": max(values), + } + summaries.append({"profile": profile, "runs": len(rows), "metrics": metrics}) + summary = {"profiles": summaries} + if metadata is not None: + summary["environment"] = metadata + return summary + + +def _run_once( + runtime, model, profile: BenchmarkProfile, *, phase: str, iteration: int +) -> dict[str, Any]: + started = time.perf_counter() + try: + result = runtime.cactus_complete( + model, profile.messages, profile.options, profile.tools, None + ) + wall_ms = (time.perf_counter() - started) * 1000.0 + row = { + "profile": profile.name, + "phase": phase, + "iteration": iteration, + "success": bool(result.get("success", True)), + "wall_time_ms": wall_ms, + "response_chars": len(str(result.get("response", ""))), + } + for field in METRIC_FIELDS: + if field in result: + row[field] = result[field] + if result.get("error"): + row["error"] = result["error"] + return row + except Exception as exc: + return { + "profile": profile.name, + "phase": phase, + "iteration": iteration, + "success": False, + "wall_time_ms": (time.perf_counter() - started) * 1000.0, + "error": str(exc), + } + + +def run_benchmark( + runtime, + model, + profiles: list[BenchmarkProfile], + *, + warmup: int, + iterations: int, + reset_between: bool, + metadata: dict[str, Any] | None = None, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + rows = [] + for profile in profiles: + for i in range(warmup): + if reset_between: + runtime.cactus_reset(model) + rows.append(_run_once(runtime, model, profile, phase="warmup", iteration=i)) + for i in range(iterations): + if reset_between: + runtime.cactus_reset(model) + rows.append( + _run_once(runtime, model, profile, phase="measure", iteration=i) + ) + return rows, summarize_results(rows, metadata=metadata) + + +def _profile_metric( + summary: dict[str, Any], profile: str, metric: str, stat: str +) -> float | None: + for item in summary.get("profiles", []): + if item.get("profile") != profile: + continue + value = item.get("metrics", {}).get(metric, {}).get(stat) + return float(value) if isinstance(value, (int, float)) else None + return None + + +def load_regression_budget(path: str | None) -> dict[str, Any]: + if path is None: + return {} + data = json.loads(Path(path).read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError("Regression budget must be a JSON object") + return data + + +def _budget_number(value: Any) -> float | None: + return float(value) if isinstance(value, (int, float)) else None + + +def _metric_budget(entry: Any, metric: str) -> dict[str, Any]: + if not isinstance(entry, dict): + return {} + value = entry.get(metric) + return value if isinstance(value, dict) else {} + + +def _comparison_budget( + budget: dict[str, Any], + profile: str, + metric: str, + default_stat: str, + default_threshold_pct: float, +) -> tuple[str, float]: + stat = str(budget.get("stat") or default_stat) + threshold_pct = _budget_number(budget.get("threshold_pct")) or default_threshold_pct + + metric_budget = _metric_budget(budget.get("metrics"), metric) + stat = str(metric_budget.get("stat") or stat) + threshold_pct = _budget_number(metric_budget.get("threshold_pct")) or threshold_pct + + profiles_budget = budget.get("profiles") + if isinstance(profiles_budget, dict): + profile_budget = profiles_budget.get(profile) + if isinstance(profile_budget, dict): + profile_metric_budget = _metric_budget(profile_budget, metric) + stat = str( + profile_metric_budget.get("stat") or profile_budget.get("stat") or stat + ) + threshold_pct = ( + _budget_number(profile_metric_budget.get("threshold_pct")) + or _budget_number(profile_budget.get("threshold_pct")) + or threshold_pct + ) + + return stat, threshold_pct + + +def _regression_severity(change_pct: float, threshold_pct: float) -> str: + over_budget = abs(change_pct) - threshold_pct + if over_budget >= 15: + return "critical" + if over_budget >= 5: + return "major" + return "minor" + + +def compare_summaries( + baseline: dict[str, Any], + candidate: dict[str, Any], + *, + metrics: list[str], + stat: str, + threshold_pct: float, + budget: dict[str, Any] | None = None, +) -> dict[str, Any]: + budget = budget or {} + profiles = sorted( + { + item.get("profile") + for source in (baseline, candidate) + for item in source.get("profiles", []) + if item.get("profile") + } + ) + rows = [] + for profile in profiles: + for metric in metrics: + metric_stat, metric_threshold_pct = _comparison_budget( + budget, + profile, + metric, + stat, + threshold_pct, + ) + base_value = _profile_metric(baseline, profile, metric, metric_stat) + cand_value = _profile_metric(candidate, profile, metric, metric_stat) + if base_value is None or cand_value is None or base_value == 0: + continue + change_pct = ((cand_value - base_value) / abs(base_value)) * 100.0 + lower_is_better = metric in LOWER_IS_BETTER + regression = ( + change_pct > metric_threshold_pct + if lower_is_better + else change_pct < -metric_threshold_pct + ) + rows.append( + { + "profile": profile, + "metric": metric, + "stat": metric_stat, + "baseline": base_value, + "candidate": cand_value, + "change_pct": change_pct, + "threshold_pct": metric_threshold_pct, + "lower_is_better": lower_is_better, + "regression": regression, + "severity": _regression_severity(change_pct, metric_threshold_pct) + if regression + else "ok", + } + ) + return { + "threshold_pct": threshold_pct, + "budget": budget, + "regressions": [row for row in rows if row["regression"]], + "comparisons": rows, + } + + +def render_markdown( + summary: dict[str, Any], comparison: dict[str, Any] | None = None +) -> str: + lines = ["# Cactus benchmark report", ""] + env = summary.get("environment") or {} + if env: + lines.extend( + [ + "## Environment", + "", + "| Field | Value |", + "| --- | --- |", + ] + ) + for key in ( + "created_at", + "model_id", + "bundle_dir", + "cactus_version", + "python", + "platform", + "machine", + "processor", + ): + if env.get(key) is not None: + lines.append(f"| {key} | {env[key]} |") + lines.append("") + + lines.extend( + [ + "## Summary", + "", + "| Profile | Runs | TTFT p50 ms | Decode p50 tok/s | Prefill p50 tok/s | RAM p50 MB |", + "| --- | ---: | ---: | ---: | ---: | ---: |", + ] + ) + for item in summary.get("profiles", []): + metrics = item.get("metrics", {}) + + def p50(name: str) -> float: + return float(metrics.get(name, {}).get("p50", 0.0)) + + lines.append( + f"| {item.get('profile')} | {item.get('runs', 0)} | " + f"{p50('time_to_first_token_ms'):.2f} | {p50('decode_tps'):.2f} | " + f"{p50('prefill_tps'):.2f} | {p50('ram_usage_mb'):.2f} |" + ) + lines.append("") + + if comparison is not None: + lines.extend( + [ + "## Regression comparison", + "", + "| Profile | Metric | Baseline | Candidate | Change | Allowed | Result |", + "| --- | --- | ---: | ---: | ---: | ---: | --- |", + ] + ) + for row in comparison.get("comparisons", []): + result = row["severity"] if row["regression"] else "ok" + lines.append( + f"| {row['profile']} | {row['metric']} {row['stat']} | " + f"{row['baseline']:.3f} | {row['candidate']:.3f} | " + f"{row['change_pct']:.2f}% | {row['threshold_pct']:.2f}% | {result} |" + ) + lines.append("") + return "\n".join(lines) + + +def _write_jsonl(path: str | None, rows: list[dict[str, Any]]) -> None: + if path is None: + return + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + with target.open("w", encoding="utf-8") as fh: + for row in rows: + fh.write(json.dumps(row, sort_keys=True) + "\n") + + +def _write_summary(path: str | None, summary: dict[str, Any]) -> None: + if path is None: + return + target = Path(path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text( + json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + +def _load_summary(path: str) -> dict[str, Any]: + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def cmd_benchmark(args): + if args.compare: + baseline = _load_summary(args.compare[0]) + candidate = _load_summary(args.compare[1]) + metrics = args.compare_metric or list(DEFAULT_COMPARE_METRICS) + try: + comparison = compare_summaries( + baseline, + candidate, + metrics=metrics, + stat=args.compare_stat, + threshold_pct=args.regression_threshold, + budget=load_regression_budget(args.budget_json), + ) + except ValueError as exc: + print_color(RED, str(exc)) + return 1 + report = render_markdown(candidate, comparison) + if args.markdown_report: + Path(args.markdown_report).write_text(report, encoding="utf-8") + print(report) + return 1 if args.fail_on_regression and comparison["regressions"] else 0 + + from .model import TranspileOptions, ensure_bundle, resolve_bundle_dir + from ..bindings import cactus as runtime + + base_options = {} + if args.max_tokens is not None: + base_options["max_tokens"] = args.max_tokens + if args.temperature is not None: + base_options["temperature"] = args.temperature + + try: + profiles = load_profiles( + args.profiles_file, set(args.profile or []), base_options + ) + profiles.extend( + build_sweep_profiles(_parse_int_list(args.sweep_token_counts), base_options) + ) + except ValueError as exc: + print_color(RED, str(exc)) + return 1 + + bundle_dir = resolve_bundle_dir(args.model_id) + if bundle_dir is None: + try: + bundle_dir = ensure_bundle( + args.model_id, + token=args.token, + reconvert=args.reconvert, + transpile=TranspileOptions(max_new_tokens=args.max_tokens), + ) + except RuntimeError as exc: + print_color(RED, f"Model setup failed: {exc}") + return 1 + + model = None + try: + model = runtime.cactus_init(str(bundle_dir)) + rows, summary = run_benchmark( + runtime, + model, + profiles, + warmup=args.warmup, + iterations=args.iterations, + reset_between=not args.keep_cache, + metadata=collect_environment(args.model_id, bundle_dir), + ) + finally: + if model is not None: + runtime.cactus_destroy(model) + + _write_jsonl(args.output, rows) + _write_summary(args.summary_json, summary) + if args.markdown_report: + Path(args.markdown_report).write_text( + render_markdown(summary), encoding="utf-8" + ) + + print_color( + GREEN, + f"Benchmarked {len(profiles)} profile(s), {args.iterations} measured run(s) each", + ) + for item in summary["profiles"]: + decode = item["metrics"].get("decode_tps", {}) + ttft = item["metrics"].get("time_to_first_token_ms", {}) + print( + f"{item['profile']}: ttft_p50={ttft.get('p50', 0.0):.2f}ms decode_p50={decode.get('p50', 0.0):.2f} tok/s" + ) + return 0 diff --git a/python/tests/test_cli_benchmark.py b/python/tests/test_cli_benchmark.py new file mode 100644 index 000000000..979e6c58b --- /dev/null +++ b/python/tests/test_cli_benchmark.py @@ -0,0 +1,313 @@ +import json +import sys +from argparse import Namespace + +import cactus.cli.benchmark as benchmark + + +class FakeRuntime: + def __init__(self): + self.calls = [] + self.resets = 0 + + def cactus_complete(self, model, messages, options, tools, callback): + self.calls.append((model, messages, options, tools, callback)) + idx = len(self.calls) + return { + "success": True, + "response": "ok", + "time_to_first_token_ms": 10.0 + idx, + "total_time_ms": 20.0 + idx, + "prefill_tps": 100.0, + "decode_tps": 40.0 + idx, + "prefill_tokens": 8, + "decode_tokens": 4, + "total_tokens": 12, + } + + def cactus_reset(self, model): + self.resets += 1 + + +def test_load_profiles_filters_and_applies_base_options(tmp_path): + profiles = tmp_path / "profiles.json" + profiles.write_text( + json.dumps( + { + "profiles": [ + {"name": "short", "messages": [{"role": "user", "content": "hi"}]}, + { + "name": "long", + "messages": [{"role": "user", "content": "hello"}], + "options": {"temperature": 0.2}, + }, + ] + } + ), + encoding="utf-8", + ) + + selected = benchmark.load_profiles( + str(profiles), {"long"}, {"max_tokens": 32, "temperature": 0.0} + ) + + assert [p.name for p in selected] == ["long"] + assert selected[0].options == {"max_tokens": 32, "temperature": 0.2} + + +def test_run_benchmark_records_json_ready_rows_and_summary(): + runtime = FakeRuntime() + profiles = [ + benchmark.BenchmarkProfile( + name="short", + messages=[{"role": "user", "content": "hi"}], + options={"max_tokens": 8}, + ) + ] + + rows, summary = benchmark.run_benchmark( + runtime, + "model", + profiles, + warmup=1, + iterations=3, + reset_between=True, + metadata={"model_id": "fake"}, + ) + + assert runtime.resets == 4 + assert [row["phase"] for row in rows] == ["warmup", "measure", "measure", "measure"] + assert rows[1]["decode_tps"] == 42.0 + assert summary["profiles"][0]["profile"] == "short" + assert summary["environment"]["model_id"] == "fake" + assert summary["profiles"][0]["runs"] == 3 + assert summary["profiles"][0]["metrics"]["decode_tps"]["p50"] == 43.0 + + +def test_build_sweep_profiles_creates_named_context_profiles(): + profiles = benchmark.build_sweep_profiles([8, 16], {"max_tokens": 4}) + + assert [profile.name for profile in profiles] == [ + "context_sweep_8", + "context_sweep_16", + ] + assert profiles[0].options == {"max_tokens": 4} + assert "Return one short sentence" in profiles[0].messages[0]["content"] + + +def test_compare_summaries_flags_latency_and_throughput_regressions(): + baseline = { + "profiles": [ + { + "profile": "short", + "runs": 2, + "metrics": { + "time_to_first_token_ms": {"p50": 100.0}, + "decode_tps": {"p50": 50.0}, + }, + } + ] + } + candidate = { + "profiles": [ + { + "profile": "short", + "runs": 2, + "metrics": { + "time_to_first_token_ms": {"p50": 112.0}, + "decode_tps": {"p50": 44.0}, + }, + } + ] + } + + comparison = benchmark.compare_summaries( + baseline, + candidate, + metrics=["time_to_first_token_ms", "decode_tps"], + stat="p50", + threshold_pct=5.0, + ) + + assert len(comparison["regressions"]) == 2 + assert {row["metric"] for row in comparison["regressions"]} == { + "time_to_first_token_ms", + "decode_tps", + } + + +def test_compare_summaries_applies_profile_metric_budget(): + baseline = { + "profiles": [ + { + "profile": "long_prefill", + "runs": 2, + "metrics": {"decode_tps": {"p50": 50.0, "p95": 45.0}}, + } + ] + } + candidate = { + "profiles": [ + { + "profile": "long_prefill", + "runs": 2, + "metrics": {"decode_tps": {"p50": 48.0, "p95": 40.0}}, + } + ] + } + + comparison = benchmark.compare_summaries( + baseline, + candidate, + metrics=["decode_tps"], + stat="p50", + threshold_pct=10.0, + budget={ + "profiles": { + "long_prefill": { + "decode_tps": {"stat": "p95", "threshold_pct": 5.0}, + } + } + }, + ) + + row = comparison["comparisons"][0] + assert row["stat"] == "p95" + assert row["threshold_pct"] == 5.0 + assert row["regression"] is True + assert row["severity"] == "major" + + +def test_render_markdown_includes_environment_and_comparison(): + summary = { + "environment": {"model_id": "m", "python": "3.11"}, + "profiles": [ + { + "profile": "short", + "runs": 1, + "metrics": { + "time_to_first_token_ms": {"p50": 1.0}, + "decode_tps": {"p50": 2.0}, + "prefill_tps": {"p50": 3.0}, + "ram_usage_mb": {"p50": 4.0}, + }, + } + ], + } + comparison = { + "comparisons": [ + { + "profile": "short", + "metric": "decode_tps", + "stat": "p50", + "baseline": 3.0, + "candidate": 2.0, + "change_pct": -33.3, + "threshold_pct": 5.0, + "regression": True, + "severity": "critical", + } + ] + } + + report = benchmark.render_markdown(summary, comparison) + + assert "# Cactus benchmark report" in report + assert "| model_id | m |" in report + assert "critical" in report + + +def test_cmd_benchmark_uses_bundle_and_writes_outputs(monkeypatch, tmp_path): + bundle = tmp_path / "bundle" + bundle.mkdir() + output = tmp_path / "bench.jsonl" + summary = tmp_path / "summary.json" + fake_runtime = FakeRuntime() + fake_runtime.cactus_init = lambda path: "model" + fake_runtime.cactus_destroy = lambda model: None + + monkeypatch.setattr(benchmark, "print_color", lambda *args, **kwargs: None) + monkeypatch.setattr("cactus.cli.model.resolve_bundle_dir", lambda model_id: bundle) + monkeypatch.setattr( + "cactus.cli.model.ensure_bundle", lambda *args, **kwargs: bundle + ) + monkeypatch.setitem(sys.modules, "cactus.bindings.cactus", fake_runtime) + + args = Namespace( + model_id="bundle", + profiles_file=None, + profile=["short_chat"], + max_tokens=16, + temperature=0.0, + token=None, + reconvert=False, + warmup=0, + iterations=2, + keep_cache=False, + output=str(output), + summary_json=str(summary), + markdown_report=None, + sweep_token_counts=None, + compare=None, + compare_metric=None, + compare_stat="p50", + regression_threshold=5.0, + budget_json=None, + fail_on_regression=False, + ) + + assert benchmark.cmd_benchmark(args) == 0 + assert len(output.read_text(encoding="utf-8").splitlines()) == 2 + body = json.loads(summary.read_text(encoding="utf-8")) + assert body["profiles"][0]["runs"] == 2 + + +def test_parser_accepts_benchmark_command(): + from cactus.cli import create_parser + + args = create_parser().parse_args( + [ + "benchmark", + "LiquidAI/LFM2-1.2B", + "--profile", + "short_chat", + "--iterations", + "5", + "--output", + "bench.jsonl", + "--sweep-token-counts", + "512,2048", + "--markdown-report", + "report.md", + ] + ) + + assert args.command == "benchmark" + assert args.profile == ["short_chat"] + assert args.iterations == 5 + assert args.sweep_token_counts == "512,2048" + assert args.markdown_report == "report.md" + + +def test_parser_accepts_benchmark_compare_mode(): + from cactus.cli import create_parser + + args = create_parser().parse_args( + [ + "benchmark", + "--compare", + "base.json", + "candidate.json", + "--compare-metric", + "decode_tps", + "--budget-json", + "budgets.json", + "--fail-on-regression", + ] + ) + + assert args.command == "benchmark" + assert args.compare == ["base.json", "candidate.json"] + assert args.compare_metric == ["decode_tps"] + assert args.budget_json == "budgets.json" + assert args.fail_on_regression is True