From ce8cccb27a760e616115744f442a2a9a137613cd Mon Sep 17 00:00:00 2001 From: Yuewei Na Date: Thu, 27 Aug 2026 09:59:46 -0700 Subject: [PATCH] test(benchmark): unit + golden-file tests for the agentperf porter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unit tests cover the porter's utility layer: path-prefix rewriting (single value + nested trees), absolute-path collection, worker-env translation (drop/rename/DYN_LOG pinning), quoted env-token-list extraction from srun lines, frontend-arg reconstruction from env, and settle-time recovery from output filename stems. The golden e2e test materializes a real harness run directory as a stable fixture (tests/fixtures/agentperf_porter/c1010 — the DSV4 c1010 GB300 baseline, usernames anonymized, content otherwise verbatim: real engine configs, the real 35-token worker env, the real client banner, real srun line shapes) plus checked-in expected recipe/workload YAMLs. The test runs the porter on the fixture and byte-compares normalized output against the goldens, so any behavior drift over time fails loudly; regeneration instructions are in the module docstring. A second test loads the golden recipe through SrtConfig's schema. Signed-off-by: Yuewei Na --- .../agentperf_porter/c1010/3_output_CTX_0.log | 2 + ...__phase0__dur2400s__settle240s__traj3.json | 0 .../agentperf_porter/c1010/client.log | 28 +++ .../c1010/client_cmds_base.sh | 1 + .../agentperf_porter/c1010/ctx_config.yaml | 44 ++++ ...ample-user-trajectory-assignments-20k.json | 1 + ...ing-agent-scenario_tuning_v2_500traj.jsonl | 1 + .../agentperf_porter/c1010/gen_config.yaml | 53 ++++ tests/fixtures/agentperf_porter/c1010/job.log | 8 + .../agentperf_porter/c1010/job_params.env | 14 ++ .../agentperf_porter/expected-recipe.yaml | 230 ++++++++++++++++++ .../agentperf_porter/expected-workload.yaml | 20 ++ tests/test_agentperf_porter_golden.py | 139 +++++++++++ 13 files changed, 541 insertions(+) create mode 100644 tests/fixtures/agentperf_porter/c1010/3_output_CTX_0.log create mode 100644 tests/fixtures/agentperf_porter/c1010/agentperf/_lustre_fsw_portfolios_coreai_projects_model-ci_artifacts_model_nvidia_deepseek-v4-pro-nvfp4-dspark_hf_hf-318bf60_orig__1010u__phase0__dur2400s__settle240s__traj3.json create mode 100644 tests/fixtures/agentperf_porter/c1010/client.log create mode 100644 tests/fixtures/agentperf_porter/c1010/client_cmds_base.sh create mode 100644 tests/fixtures/agentperf_porter/c1010/ctx_config.yaml create mode 100644 tests/fixtures/agentperf_porter/c1010/datasets/500-sample-user-trajectory-assignments-20k.json create mode 100644 tests/fixtures/agentperf_porter/c1010/datasets/aa-rwlt_coding-agent-scenario_tuning_v2_500traj.jsonl create mode 100644 tests/fixtures/agentperf_porter/c1010/gen_config.yaml create mode 100644 tests/fixtures/agentperf_porter/c1010/job.log create mode 100644 tests/fixtures/agentperf_porter/c1010/job_params.env create mode 100644 tests/fixtures/agentperf_porter/expected-recipe.yaml create mode 100644 tests/fixtures/agentperf_porter/expected-workload.yaml create mode 100644 tests/test_agentperf_porter_golden.py diff --git a/tests/fixtures/agentperf_porter/c1010/3_output_CTX_0.log b/tests/fixtures/agentperf_porter/c1010/3_output_CTX_0.log new file mode 100644 index 000000000..5569f5275 --- /dev/null +++ b/tests/fixtures/agentperf_porter/c1010/3_output_CTX_0.log @@ -0,0 +1,2 @@ +clean worker startup +no pinning in this baseline diff --git a/tests/fixtures/agentperf_porter/c1010/agentperf/_lustre_fsw_portfolios_coreai_projects_model-ci_artifacts_model_nvidia_deepseek-v4-pro-nvfp4-dspark_hf_hf-318bf60_orig__1010u__phase0__dur2400s__settle240s__traj3.json b/tests/fixtures/agentperf_porter/c1010/agentperf/_lustre_fsw_portfolios_coreai_projects_model-ci_artifacts_model_nvidia_deepseek-v4-pro-nvfp4-dspark_hf_hf-318bf60_orig__1010u__phase0__dur2400s__settle240s__traj3.json new file mode 100644 index 000000000..e69de29bb diff --git a/tests/fixtures/agentperf_porter/c1010/client.log b/tests/fixtures/agentperf_porter/c1010/client.log new file mode 100644 index 000000000..deec9b62c --- /dev/null +++ b/tests/fixtures/agentperf_porter/c1010/client.log @@ -0,0 +1,28 @@ +0: warning: `VIRTUAL_ENV=/opt/dynamo/venv` does not match the project environment path `/tmp/agentperf-570849/venv` and will be ignored; use `--active` to target the active environment instead +0: ============================================================ +0: AA-AGENTPERF: Deterministic Load Test +0: ============================================================ +0: Base URL: http://nvl72d102-T10:8333/v1 +0: Model: /lustre/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig +0: Server type: trtllm +0: Concurrencies: [1010] +0: Max workers: 8 +0: Connection pool: 8192 (auto-sized, keepalive expiry disabled) +0: Phase Timeout: 2400.0s +0: Max ISL offset: 10 +0: User assignments: /tmp/agentperf-570849/data/500-sample-user-trajectory-assignments-20k.json +0: Max tokens: 2000 +0: Reasoning effort: high +0: Tool-call delays: enabled +0: Conversation routing headers: enabled +0: Dynamo conv-aware routing (X-Dynamo-Session-ID): enabled +0: Power sampling: disabled +0: Results dir: /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/agentperf +0: Request log: /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/agentperf/requests.jsonl +0: Seed: 42 +0: ============================================================ +0: Loading trajectories from /tmp/agentperf-570849/data/aa-rwlt_coding-agent-scenario_tuning_v2_500traj.jsonl... +0: Loaded 500 trajectories +0: Total requests across all trajectories: 18997 +0: Loaded user assignments from /tmp/agentperf-570849/data/500-sample-user-trajectory-assignments-20k.json: 20000 users, 3 trajectories each +0: Running 1 phase(s): concurrencies=[1010] diff --git a/tests/fixtures/agentperf_porter/c1010/client_cmds_base.sh b/tests/fixtures/agentperf_porter/c1010/client_cmds_base.sh new file mode 100644 index 000000000..4b8389f0e --- /dev/null +++ b/tests/fixtures/agentperf_porter/c1010/client_cmds_base.sh @@ -0,0 +1 @@ +srun -l --export=ALL --nodelist --container-name=agentperf-${SLURM_JOB_ID} --container-image=/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/images/dynamo_ff959852b740_trtllm_5b427d9d5d.sqsh --container-mounts=/lustre:/lustre,/scratch:/scratch --no-container-mount-home --mpi=pmix --overlap -N 1 -n 1 bash -c ' set -euo pipefail; export RUNTIME=/tmp/agentperf-${SLURM_JOB_ID}; test -f ${RUNTIME}/READY; export HOME=${RUNTIME}/home CARGO_HOME=${RUNTIME}/cargo RUSTUP_HOME=${RUNTIME}/rustup CARGO_TARGET_DIR=${RUNTIME}/target UV_PROJECT_ENVIRONMENT=${RUNTIME}/venv UV_CACHE_DIR=${RUNTIME}/uv-cache; export PATH=${HOME}/.local/bin:${CARGO_HOME}/bin:/usr/local/bin:/usr/bin:/bin; mkdir -p /logs/agentperf; cd /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/agentperf-client-worktrees/9e4aae134986eee8fdb44264c1d918a2ca3f18f7; uv run python agentperf/run.py --config ${RUNTIME}/benchmark_config.yaml --concurrencies 1010 --request-log-path /logs/agentperf/requests.jsonl --results-dir /logs/agentperf' &> /logs/client.log diff --git a/tests/fixtures/agentperf_porter/c1010/ctx_config.yaml b/tests/fixtures/agentperf_porter/c1010/ctx_config.yaml new file mode 100644 index 000000000..b83fda71b --- /dev/null +++ b/tests/fixtures/agentperf_porter/c1010/ctx_config.yaml @@ -0,0 +1,44 @@ +num_serve_frontends: 4 +max_batch_size: 128 +max_num_tokens: 8192 +max_seq_len: 147456 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +enable_attention_dp: true +pipeline_parallel_size: 1 +enable_chunked_prefill: true +print_iter_log: true +cuda_graph_config: null +enable_autotuner: true +disable_overlap_scheduler: false +moe_config: + backend: MEGAMOE_CUTEDSL +kv_cache_config: + enable_block_reuse: true + free_gpu_memory_fraction: 0.4 + dtype: fp8 + host_cache_size: 107374182400 + pool_ratio: + - 0.45 + - 0.3 + - 0.25 +cache_transceiver_config: + max_tokens_in_buffer: 147456 + backend: NIXL + transceiver_runtime: PYTHON + kv_transfer_timeout_ms: 600000 +scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION +attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 +return_perf_metrics: false +enable_iter_perf_stats: false +enable_iter_req_stats: false +nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core diff --git a/tests/fixtures/agentperf_porter/c1010/datasets/500-sample-user-trajectory-assignments-20k.json b/tests/fixtures/agentperf_porter/c1010/datasets/500-sample-user-trajectory-assignments-20k.json new file mode 100644 index 000000000..732c9afa9 --- /dev/null +++ b/tests/fixtures/agentperf_porter/c1010/datasets/500-sample-user-trajectory-assignments-20k.json @@ -0,0 +1 @@ +{"metadata":{"per_user":3,"n_users_total":20000},"assignments":{}} diff --git a/tests/fixtures/agentperf_porter/c1010/datasets/aa-rwlt_coding-agent-scenario_tuning_v2_500traj.jsonl b/tests/fixtures/agentperf_porter/c1010/datasets/aa-rwlt_coding-agent-scenario_tuning_v2_500traj.jsonl new file mode 100644 index 000000000..98c62fa08 --- /dev/null +++ b/tests/fixtures/agentperf_porter/c1010/datasets/aa-rwlt_coding-agent-scenario_tuning_v2_500traj.jsonl @@ -0,0 +1 @@ +{"conversation_id":"aa-rwlt-coding-agent-001","messages":[]} diff --git a/tests/fixtures/agentperf_porter/c1010/gen_config.yaml b/tests/fixtures/agentperf_porter/c1010/gen_config.yaml new file mode 100644 index 000000000..00ddc2d2e --- /dev/null +++ b/tests/fixtures/agentperf_porter/c1010/gen_config.yaml @@ -0,0 +1,53 @@ +tensor_parallel_size: 32 +moe_expert_parallel_size: 32 +enable_attention_dp: true +enable_lm_head_tp_in_adp: true +pipeline_parallel_size: 1 +max_batch_size: 16 +max_num_tokens: 128 +max_seq_len: 147456 +print_iter_log: true +enable_autotuner: false +kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + dtype: fp8 + host_cache_size: 0 + avg_seq_len: 39040 +moe_config: + backend: MEGAMOE_CUTEDSL + use_low_precision_moe_combine: true + load_balancer: /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/eplb/moe_load_balancer_gen_ep32_slots384_dspark.yaml +cache_transceiver_config: + max_tokens_in_buffer: 147456 + backend: NIXL + transceiver_runtime: PYTHON + kv_transfer_timeout_ms: 600000 +stream_interval: 100 +num_postprocess_workers: 8 +cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 +speculative_config: + decoding_type: DSpark + speculative_model: /lustre/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig + max_draft_len: 6 + block_size: 6 +nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core +sparse_attention_config: + algorithm: deepseek_v4 +return_perf_metrics: false +enable_iter_perf_stats: false +enable_iter_req_stats: false diff --git a/tests/fixtures/agentperf_porter/c1010/job.log b/tests/fixtures/agentperf_porter/c1010/job.log new file mode 100644 index 000000000..65d819d04 --- /dev/null +++ b/tests/fixtures/agentperf_porter/c1010/job.log @@ -0,0 +1,8 @@ +[server] Executing: srun -l --export=ALL,ETCD_ENDPOINTS=http://nvl72d102-T10:2379,ROUTER_MODE=kv,DYN_KV_BLOCK_SIZE=128,DEBUG_ROUTER=0,PYTHONUNBUFFERED=1,DYN_ROUTER_SESSION_AFFINITY_TTL_SECS=14400,DYN_ROUTER_TEMPERATURE=0,DYN_ROUTER_QUEUE_THRESHOLD=None,DYN_TOKENIZER_CACHE=1,DYN_TOKENIZER_CACHE_BYTES=8000000000,DYN_TOKENIZER=fastokens,DYN_TCP_REQUEST_TIMEOUT=30,DYN_REQUEST_PLANE=tcp,DYN_FRONTEND_ENABLE_KV_EVENTS=0,DYN_LOGGING_JSONL=false,OTEL_EXPORT_ENABLED=false --nodelist nvl72d102-T10 --container-name=disagg-${SLURM_JOB_ID} --container-image=/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/images/dynamo_ff959852b740_trtllm_5b427d9d5d.sqsh --container-mounts=/lustre:/lustre,/scratch:/scratch --no-container-mount-home --mpi=pmix --overlap -N 1 -n 1 bash /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-harness/38808e274dd8e18e/start_dynamo_infra.sh etcd nvl72d102-T10 2379 /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/4_output_etcd.log & +[server] Executing: srun -l --export=ALL,ETCD_ENDPOINTS=http://nvl72d102-T10:2379,ROUTER_MODE=kv,DYN_KV_BLOCK_SIZE=128,DEBUG_ROUTER=0,PYTHONUNBUFFERED=1,DYN_ROUTER_SESSION_AFFINITY_TTL_SECS=14400,DYN_ROUTER_TEMPERATURE=0,DYN_ROUTER_QUEUE_THRESHOLD=None,DYN_TOKENIZER_CACHE=1,DYN_TOKENIZER_CACHE_BYTES=8000000000,DYN_TOKENIZER=fastokens,DYN_TCP_REQUEST_TIMEOUT=30,DYN_REQUEST_PLANE=tcp,DYN_FRONTEND_ENABLE_KV_EVENTS=0,DYN_LOGGING_JSONL=false,OTEL_EXPORT_ENABLED=false --nodelist nvl72d102-T10 --container-name=disagg-${SLURM_JOB_ID} --container-image=/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/images/dynamo_ff959852b740_trtllm_5b427d9d5d.sqsh --container-mounts=/lustre:/lustre,/scratch:/scratch --no-container-mount-home --mpi=pmix --overlap -N 1 -n 1 bash /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-harness/38808e274dd8e18e/start_dynamo_infra.sh frontend nvl72d102-T10 8333 /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/4_output_frontend.log & +[server] Executing: srun -l --export=ALL,HOME=/home/bench-user --nodelist nvl72d102-T01,nvl72d102-T02,nvl72d102-T03,nvl72d102-T04,nvl72d102-T05,nvl72d102-T06,nvl72d102-T07,nvl72d102-T08 --container-name=disagg-${SLURM_JOB_ID} --container-image=/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/images/dynamo_ff959852b740_trtllm_5b427d9d5d.sqsh --container-mounts=/lustre:/lustre,/scratch:/scratch --no-container-mount-home --mpi=pmix --overlap -N 8 -n 32 bash /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-harness/38808e274dd8e18e/start_worker_dynamo.sh GEN 0 /lustre/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig 8101 e2e '1010' true /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010 false '0-0' /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/gen_config.yaml 'TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 TRTLLM_DISABLE_KV_CACHE_RATIO_UPDATE=1 TRTLLM_MHC_ENABLE_FUSED_HC=1 TRTLLM_MOE_A2A_DISABLE_CFT_COUNTED_WRITES=1 TRTLLM_MLA_EXTRA_OVERLAP=1 TRTLLM_FUSED_FP8_QUANT_PACK=1 TRTLLM_DSV4_MEM_OPTS=1 TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE=1 HF_HUB_OFFLINE=1 TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN=1 CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 TLLM_NUMA_AWARE_WORKER_AFFINITY=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True VIRTUAL_ENV=/opt/dynamo/venv PATH=/opt/dynamo/venv/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin LD_PRELOAD=/opt/dynamo/libstdc++.so.6:/usr/local/lib/python3.12/dist-packages/tensorrt_llm/libs/nixl/libnixl.so NIXL_PLUGIN_DIR=/usr/local/lib/python3.12/dist-packages/tensorrt_llm/libs/nixl/plugins DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30 DYN_UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp DYN_REQUEST_PLANE=tcp DYN_WORKER_PUBLISH_KV_EVENTS=0 DYN_LOGGING_JSONL=false OTEL_EXPORT_ENABLED=false TRTLLM_SERVE_ENABLE_ORJSON=1 CUDA_VISIBLE_DEVICES=0,1,2,3 ETCD_ENDPOINTS=http://nvl72d102-T10:2379 DYN_KV_BLOCK_SIZE=128 PYTHONUNBUFFERED=1 DYN_TOOL_CALL_PARSER=deepseek_v4 DYN_REASONING_PARSER=deepseek_v4' none none &> /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/3_output_GEN_0.log & +[server] Executing: srun -l --export=ALL,HOME=/home/bench-user --nodelist nvl72d102-T10 --container-name=disagg-${SLURM_JOB_ID} --container-image=/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/images/dynamo_ff959852b740_trtllm_5b427d9d5d.sqsh --container-mounts=/lustre:/lustre,/scratch:/scratch --no-container-mount-home --mpi=pmix --overlap -N 1 -n 4 bash /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-harness/38808e274dd8e18e/start_worker_dynamo.sh CTX 0 /lustre/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig 8102 e2e '1010' true /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010 false '0-0' /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/ctx_config.yaml 'TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 TRTLLM_DISABLE_KV_CACHE_RATIO_UPDATE=1 TRTLLM_MHC_ENABLE_FUSED_HC=1 TRTLLM_MOE_A2A_DISABLE_CFT_COUNTED_WRITES=1 TRTLLM_MLA_EXTRA_OVERLAP=1 TRTLLM_FUSED_FP8_QUANT_PACK=1 TRTLLM_DSV4_MEM_OPTS=1 TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE=1 HF_HUB_OFFLINE=1 TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN=1 CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 TLLM_NUMA_AWARE_WORKER_AFFINITY=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True VIRTUAL_ENV=/opt/dynamo/venv PATH=/opt/dynamo/venv/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin LD_PRELOAD=/opt/dynamo/libstdc++.so.6:/usr/local/lib/python3.12/dist-packages/tensorrt_llm/libs/nixl/libnixl.so NIXL_PLUGIN_DIR=/usr/local/lib/python3.12/dist-packages/tensorrt_llm/libs/nixl/plugins DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30 DYN_UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp DYN_REQUEST_PLANE=tcp DYN_WORKER_PUBLISH_KV_EVENTS=0 DYN_LOGGING_JSONL=false OTEL_EXPORT_ENABLED=false TRTLLM_SERVE_ENABLE_ORJSON=1 CUDA_VISIBLE_DEVICES=0,1,2,3 ETCD_ENDPOINTS=http://nvl72d102-T10:2379 DYN_KV_BLOCK_SIZE=128 PYTHONUNBUFFERED=1 DYN_TOOL_CALL_PARSER=deepseek_v4 DYN_REASONING_PARSER=deepseek_v4 TRTLLM_SKIP_KV_CACHE_ESTIMATION=1' none none &> /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/3_output_CTX_0.log & +[server] Executing: srun -l --export=ALL,HOME=/home/bench-user --nodelist nvl72d102-T11 --container-name=disagg-${SLURM_JOB_ID} --container-image=/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/images/dynamo_ff959852b740_trtllm_5b427d9d5d.sqsh --container-mounts=/lustre:/lustre,/scratch:/scratch --no-container-mount-home --mpi=pmix --overlap -N 1 -n 4 bash /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-harness/38808e274dd8e18e/start_worker_dynamo.sh CTX 1 /lustre/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig 8103 e2e '1010' true /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010 false '0-0' /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/ctx_config.yaml 'TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 TRTLLM_DISABLE_KV_CACHE_RATIO_UPDATE=1 TRTLLM_MHC_ENABLE_FUSED_HC=1 TRTLLM_MOE_A2A_DISABLE_CFT_COUNTED_WRITES=1 TRTLLM_MLA_EXTRA_OVERLAP=1 TRTLLM_FUSED_FP8_QUANT_PACK=1 TRTLLM_DSV4_MEM_OPTS=1 TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE=1 HF_HUB_OFFLINE=1 TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN=1 CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 TLLM_NUMA_AWARE_WORKER_AFFINITY=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True VIRTUAL_ENV=/opt/dynamo/venv PATH=/opt/dynamo/venv/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin LD_PRELOAD=/opt/dynamo/libstdc++.so.6:/usr/local/lib/python3.12/dist-packages/tensorrt_llm/libs/nixl/libnixl.so NIXL_PLUGIN_DIR=/usr/local/lib/python3.12/dist-packages/tensorrt_llm/libs/nixl/plugins DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30 DYN_UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp DYN_REQUEST_PLANE=tcp DYN_WORKER_PUBLISH_KV_EVENTS=0 DYN_LOGGING_JSONL=false OTEL_EXPORT_ENABLED=false TRTLLM_SERVE_ENABLE_ORJSON=1 CUDA_VISIBLE_DEVICES=0,1,2,3 ETCD_ENDPOINTS=http://nvl72d102-T10:2379 DYN_KV_BLOCK_SIZE=128 PYTHONUNBUFFERED=1 DYN_TOOL_CALL_PARSER=deepseek_v4 DYN_REASONING_PARSER=deepseek_v4 TRTLLM_SKIP_KV_CACHE_ESTIMATION=1' none none &> /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/3_output_CTX_1.log & +[server] Executing: srun -l --export=ALL,HOME=/home/bench-user --nodelist nvl72d102-T12 --container-name=disagg-${SLURM_JOB_ID} --container-image=/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/images/dynamo_ff959852b740_trtllm_5b427d9d5d.sqsh --container-mounts=/lustre:/lustre,/scratch:/scratch --no-container-mount-home --mpi=pmix --overlap -N 1 -n 4 bash /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-harness/38808e274dd8e18e/start_worker_dynamo.sh CTX 2 /lustre/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig 8104 e2e '1010' true /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010 false '0-0' /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/ctx_config.yaml 'TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 TRTLLM_DISABLE_KV_CACHE_RATIO_UPDATE=1 TRTLLM_MHC_ENABLE_FUSED_HC=1 TRTLLM_MOE_A2A_DISABLE_CFT_COUNTED_WRITES=1 TRTLLM_MLA_EXTRA_OVERLAP=1 TRTLLM_FUSED_FP8_QUANT_PACK=1 TRTLLM_DSV4_MEM_OPTS=1 TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE=1 HF_HUB_OFFLINE=1 TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN=1 CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 TLLM_NUMA_AWARE_WORKER_AFFINITY=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True VIRTUAL_ENV=/opt/dynamo/venv PATH=/opt/dynamo/venv/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin LD_PRELOAD=/opt/dynamo/libstdc++.so.6:/usr/local/lib/python3.12/dist-packages/tensorrt_llm/libs/nixl/libnixl.so NIXL_PLUGIN_DIR=/usr/local/lib/python3.12/dist-packages/tensorrt_llm/libs/nixl/plugins DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30 DYN_UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp DYN_REQUEST_PLANE=tcp DYN_WORKER_PUBLISH_KV_EVENTS=0 DYN_LOGGING_JSONL=false OTEL_EXPORT_ENABLED=false TRTLLM_SERVE_ENABLE_ORJSON=1 CUDA_VISIBLE_DEVICES=0,1,2,3 ETCD_ENDPOINTS=http://nvl72d102-T10:2379 DYN_KV_BLOCK_SIZE=128 PYTHONUNBUFFERED=1 DYN_TOOL_CALL_PARSER=deepseek_v4 DYN_REASONING_PARSER=deepseek_v4 TRTLLM_SKIP_KV_CACHE_ESTIMATION=1' none none &> /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/3_output_CTX_2.log & +[server] Executing: srun -l --export=ALL,HOME=/home/bench-user --nodelist nvl72d102-T13 --container-name=disagg-${SLURM_JOB_ID} --container-image=/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/images/dynamo_ff959852b740_trtllm_5b427d9d5d.sqsh --container-mounts=/lustre:/lustre,/scratch:/scratch --no-container-mount-home --mpi=pmix --overlap -N 1 -n 4 bash /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-harness/38808e274dd8e18e/start_worker_dynamo.sh CTX 3 /lustre/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig 8105 e2e '1010' true /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010 false '0-0' /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/ctx_config.yaml 'TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 TRTLLM_DISABLE_KV_CACHE_RATIO_UPDATE=1 TRTLLM_MHC_ENABLE_FUSED_HC=1 TRTLLM_MOE_A2A_DISABLE_CFT_COUNTED_WRITES=1 TRTLLM_MLA_EXTRA_OVERLAP=1 TRTLLM_FUSED_FP8_QUANT_PACK=1 TRTLLM_DSV4_MEM_OPTS=1 TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE=1 HF_HUB_OFFLINE=1 TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN=1 CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 TLLM_NUMA_AWARE_WORKER_AFFINITY=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True VIRTUAL_ENV=/opt/dynamo/venv PATH=/opt/dynamo/venv/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin LD_PRELOAD=/opt/dynamo/libstdc++.so.6:/usr/local/lib/python3.12/dist-packages/tensorrt_llm/libs/nixl/libnixl.so NIXL_PLUGIN_DIR=/usr/local/lib/python3.12/dist-packages/tensorrt_llm/libs/nixl/plugins DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30 DYN_UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp DYN_REQUEST_PLANE=tcp DYN_WORKER_PUBLISH_KV_EVENTS=0 DYN_LOGGING_JSONL=false OTEL_EXPORT_ENABLED=false TRTLLM_SERVE_ENABLE_ORJSON=1 CUDA_VISIBLE_DEVICES=0,1,2,3 ETCD_ENDPOINTS=http://nvl72d102-T10:2379 DYN_KV_BLOCK_SIZE=128 PYTHONUNBUFFERED=1 DYN_TOOL_CALL_PARSER=deepseek_v4 DYN_REASONING_PARSER=deepseek_v4 TRTLLM_SKIP_KV_CACHE_ESTIMATION=1' none none &> /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/3_output_CTX_3.log & +[server] Executing: srun -l --export=ALL,HOME=/home/bench-user --nodelist nvl72d102-T14 --container-name=disagg-${SLURM_JOB_ID} --container-image=/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/images/dynamo_ff959852b740_trtllm_5b427d9d5d.sqsh --container-mounts=/lustre:/lustre,/scratch:/scratch --no-container-mount-home --mpi=pmix --overlap -N 1 -n 4 bash /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-harness/38808e274dd8e18e/start_worker_dynamo.sh CTX 4 /lustre/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig 8106 e2e '1010' true /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010 false '0-0' /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/ctx_config.yaml 'TLLM_LOG_LEVEL=INFO TRTLLM_SERVER_DISABLE_GC=1 TRTLLM_WORKER_DISABLE_GC=1 TRTLLM_ENABLE_PDL=1 NCCL_GRAPH_MIXING_SUPPORT=0 MIMALLOC_PURGE_DELAY=0 TRTLLM_DISABLE_KV_CACHE_RATIO_UPDATE=1 TRTLLM_MHC_ENABLE_FUSED_HC=1 TRTLLM_MOE_A2A_DISABLE_CFT_COUNTED_WRITES=1 TRTLLM_MLA_EXTRA_OVERLAP=1 TRTLLM_FUSED_FP8_QUANT_PACK=1 TRTLLM_DSV4_MEM_OPTS=1 TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE=1 HF_HUB_OFFLINE=1 TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN=1 CUDA_SCALE_LAUNCH_QUEUES=4x TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD=0.10 TLLM_NUMA_AWARE_WORKER_AFFINITY=0 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True VIRTUAL_ENV=/opt/dynamo/venv PATH=/opt/dynamo/venv/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/bin LD_PRELOAD=/opt/dynamo/libstdc++.so.6:/usr/local/lib/python3.12/dist-packages/tensorrt_llm/libs/nixl/libnixl.so NIXL_PLUGIN_DIR=/usr/local/lib/python3.12/dist-packages/tensorrt_llm/libs/nixl/plugins DYN_HEALTH_CHECK_ENABLED=false DYN_ENGINE_CONV_AFFINITY=1 DYN_TOKENIZER=fastokens DYN_TCP_REQUEST_TIMEOUT=30 DYN_UCX_TLS=cuda_ipc,cuda_copy,sm,self,tcp DYN_REQUEST_PLANE=tcp DYN_WORKER_PUBLISH_KV_EVENTS=0 DYN_LOGGING_JSONL=false OTEL_EXPORT_ENABLED=false TRTLLM_SERVE_ENABLE_ORJSON=1 CUDA_VISIBLE_DEVICES=0,1,2,3 ETCD_ENDPOINTS=http://nvl72d102-T10:2379 DYN_KV_BLOCK_SIZE=128 PYTHONUNBUFFERED=1 DYN_TOOL_CALL_PARSER=deepseek_v4 DYN_REASONING_PARSER=deepseek_v4 TRTLLM_SKIP_KV_CACHE_ESTIMATION=1' none none &> /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/3_output_CTX_4.log & diff --git a/tests/fixtures/agentperf_porter/c1010/job_params.env b/tests/fixtures/agentperf_porter/c1010/job_params.env new file mode 100644 index 000000000..63c7ed45a --- /dev/null +++ b/tests/fixtures/agentperf_porter/c1010/job_params.env @@ -0,0 +1,14 @@ +PROJECT_ROOT="/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs" +MODEL_PATH="/lustre/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig" +PORT=8100 +CONCURRENCIES="1010" +BENCHMARK_CONFIG="/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/benchmark_config.yaml" +SERVER_LOG_LEVEL="verbose" +CONTAINER_IMAGE="/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/images/dynamo_ff959852b740_trtllm_5b427d9d5d.sqsh" +CONTAINER_MOUNTS="/lustre:/lustre,/scratch:/scratch" +HARNESS_DIR="/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-harness/38808e274dd8e18e" +ENABLE_METRICS_POLL=0 +COLLECT_POWER=1 +SERVER_BACKEND="dynamo" +DYNAMO_JOB_ENV='DYN_LOGGING_JSONL=false OTEL_EXPORT_ENABLED=false' +NSYS_ENABLED=0 diff --git a/tests/fixtures/agentperf_porter/expected-recipe.yaml b/tests/fixtures/agentperf_porter/expected-recipe.yaml new file mode 100644 index 000000000..368129291 --- /dev/null +++ b/tests/fixtures/agentperf_porter/expected-recipe.yaml @@ -0,0 +1,230 @@ +name: agentperf-porter +model: + path: /scratch/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig + container: /scratch/fsw/portfolios/coreai/projects/team/users/bench-user/images/dynamo_ff959852b740_trtllm_5b427d9d5d.sqsh + precision: fp4 +resources: + gpu_type: gb300 + gpus_per_node: 4 + prefill_nodes: 5 + prefill_workers: 5 + decode_nodes: 8 + decode_workers: 1 +dynamo: + install: false + request_plane: tcp + event_plane: zmq +backend: + type: trtllm + numa_memory_bind: true + numa_cpu_bind: false + prefill_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRTLLM_DISABLE_KV_CACHE_RATIO_UPDATE: '1' + TRTLLM_MHC_ENABLE_FUSED_HC: '1' + TRTLLM_MOE_A2A_DISABLE_CFT_COUNTED_WRITES: '1' + TRTLLM_MLA_EXTRA_OVERLAP: '1' + TRTLLM_FUSED_FP8_QUANT_PACK: '1' + TRTLLM_DSV4_MEM_OPTS: '1' + TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE: '1' + HF_HUB_OFFLINE: '1' + TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_NUMA_AWARE_WORKER_AFFINITY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + DYN_HEALTH_CHECK_ENABLED: 'false' + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_REQUEST_PLANE: tcp + DYN_WORKER_PUBLISH_KV_EVENTS: '0' + DYN_LOGGING_JSONL: 'false' + OTEL_EXPORT_ENABLED: 'false' + TRTLLM_SERVE_ENABLE_ORJSON: '1' + DYN_TRTLLM_KV_BLOCK_SIZE: '128' + PYTHONUNBUFFERED: '1' + DYN_TOOL_CALL_PARSER: deepseek_v4 + DYN_REASONING_PARSER: deepseek_v4 + TRTLLM_SKIP_KV_CACHE_ESTIMATION: '1' + DYN_LOG: info + decode_environment: + TLLM_LOG_LEVEL: INFO + TRTLLM_SERVER_DISABLE_GC: '1' + TRTLLM_WORKER_DISABLE_GC: '1' + TRTLLM_ENABLE_PDL: '1' + NCCL_GRAPH_MIXING_SUPPORT: '0' + MIMALLOC_PURGE_DELAY: '0' + TRTLLM_DISABLE_KV_CACHE_RATIO_UPDATE: '1' + TRTLLM_MHC_ENABLE_FUSED_HC: '1' + TRTLLM_MOE_A2A_DISABLE_CFT_COUNTED_WRITES: '1' + TRTLLM_MLA_EXTRA_OVERLAP: '1' + TRTLLM_FUSED_FP8_QUANT_PACK: '1' + TRTLLM_DSV4_MEM_OPTS: '1' + TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE: '1' + HF_HUB_OFFLINE: '1' + TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN: '1' + CUDA_SCALE_LAUNCH_QUEUES: 4x + TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10' + TLLM_NUMA_AWARE_WORKER_AFFINITY: '0' + PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True + DYN_HEALTH_CHECK_ENABLED: 'false' + DYN_ENGINE_CONV_AFFINITY: '1' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp + DYN_REQUEST_PLANE: tcp + DYN_WORKER_PUBLISH_KV_EVENTS: '0' + DYN_LOGGING_JSONL: 'false' + OTEL_EXPORT_ENABLED: 'false' + TRTLLM_SERVE_ENABLE_ORJSON: '1' + DYN_TRTLLM_KV_BLOCK_SIZE: '128' + PYTHONUNBUFFERED: '1' + DYN_TOOL_CALL_PARSER: deepseek_v4 + DYN_REASONING_PARSER: deepseek_v4 + DYN_LOG: info + trtllm_config: + prefill: + num_serve_frontends: 4 + max_batch_size: 128 + max_num_tokens: 8192 + max_seq_len: 147456 + tensor_parallel_size: 4 + moe_expert_parallel_size: 4 + enable_attention_dp: true + pipeline_parallel_size: 1 + enable_chunked_prefill: true + print_iter_log: true + cuda_graph_config: null + enable_autotuner: true + disable_overlap_scheduler: false + moe_config: + backend: MEGAMOE_CUTEDSL + kv_cache_config: + enable_block_reuse: true + free_gpu_memory_fraction: 0.4 + dtype: fp8 + host_cache_size: 107374182400 + pool_ratio: + - 0.45 + - 0.3 + - 0.25 + cache_transceiver_config: + max_tokens_in_buffer: 147456 + backend: NIXL + transceiver_runtime: PYTHON + kv_transfer_timeout_ms: 600000 + scheduler_config: + capacity_scheduler_policy: MAX_UTILIZATION + attention_dp_config: + enable_kv_cache_aware_routing: false + kv_cache_routing_conversation_affinity: true + kv_cache_routing_max_sessions: 65536 + return_perf_metrics: false + enable_iter_perf_stats: false + enable_iter_req_stats: false + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + decode: + tensor_parallel_size: 32 + moe_expert_parallel_size: 32 + enable_attention_dp: true + enable_lm_head_tp_in_adp: true + pipeline_parallel_size: 1 + max_batch_size: 16 + max_num_tokens: 128 + max_seq_len: 147456 + print_iter_log: true + enable_autotuner: false + kv_cache_config: + enable_block_reuse: false + free_gpu_memory_fraction: 0.7 + dtype: fp8 + host_cache_size: 0 + avg_seq_len: 39040 + moe_config: + backend: MEGAMOE_CUTEDSL + use_low_precision_moe_combine: true + load_balancer: /scratch/fsw/portfolios/coreai/projects/team/users/bench-user/eplb/moe_load_balancer_gen_ep32_slots384_dspark.yaml + cache_transceiver_config: + max_tokens_in_buffer: 147456 + backend: NIXL + transceiver_runtime: PYTHON + kv_transfer_timeout_ms: 600000 + stream_interval: 100 + num_postprocess_workers: 8 + cuda_graph_config: + enable_padding: true + batch_sizes: + - 1 + - 2 + - 4 + - 8 + - 16 + - 32 + - 64 + speculative_config: + decoding_type: DSpark + speculative_model: /scratch/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig + max_draft_len: 6 + block_size: 6 + nvfp4_gemm_config: + allowed_backends: + - cutlass + - cublaslt + - cutedsl + - cuda_core + sparse_attention_config: + algorithm: deepseek_v4 + return_perf_metrics: false + enable_iter_perf_stats: false + enable_iter_req_stats: false +frontend: + type: dynamo + enable_multiple_frontends: false + args: + router-mode: kv + no-router-kv-events: true + kv-cache-block-size: 128 + enforce-disagg: true + request-plane: tcp + event-plane: zmq + env: + ROUTER_MODE: kv + DYN_KV_BLOCK_SIZE: '128' + DEBUG_ROUTER: '0' + PYTHONUNBUFFERED: '1' + DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400' + DYN_ROUTER_TEMPERATURE: '0' + DYN_ROUTER_QUEUE_THRESHOLD: None + DYN_TOKENIZER_CACHE: '1' + DYN_TOKENIZER_CACHE_BYTES: '8000000000' + DYN_TOKENIZER: fastokens + DYN_TCP_REQUEST_TIMEOUT: '30' + DYN_REQUEST_PLANE: tcp + DYN_FRONTEND_ENABLE_KV_EVENTS: '0' + DYN_LOGGING_JSONL: 'false' + OTEL_EXPORT_ENABLED: 'false' + HOME: /tmp +infra: + etcd_nats_dedicated_node: false +observability: + enabled: false +benchmark: + type: agentperf + client_placement: last_decode + concurrency: 1010 + agentperf_client_dir: /scratch/fsw/portfolios/coreai/projects/team/users/bench-user/agentperf-client-worktrees/9e4aae134986eee8fdb44264c1d918a2ca3f18f7 + agentperf_config: +extra_mount: +- /scratch:/scratch diff --git a/tests/fixtures/agentperf_porter/expected-workload.yaml b/tests/fixtures/agentperf_porter/expected-workload.yaml new file mode 100644 index 000000000..7e748cef4 --- /dev/null +++ b/tests/fixtures/agentperf_porter/expected-workload.yaml @@ -0,0 +1,20 @@ +base_url: http://placeholder:8000/v1 +api_key: dummy +model: placeholder +server_type: trtllm +concurrencies: +- 1010 +phase_timeout_seconds: 2400.0 +settling_time_seconds: 240.0 +trajectory_path: /c1010/datasets/aa-rwlt_coding-agent-scenario_tuning_v2_500traj.jsonl +user_assignments_path: /c1010/datasets/500-sample-user-trajectory-assignments-20k.json +max_starting_line_offset: 10 +seed: 42 +timeout_seconds: 300.0 +max_workers: 8 +max_tokens: 2000 +reasoning_effort: high +client_impl: rust +send_conversation_routing_headers: true +use_dynamo_conv_aware_routing: true +power_dcgm_url: null diff --git a/tests/test_agentperf_porter_golden.py b/tests/test_agentperf_porter_golden.py new file mode 100644 index 000000000..e41692891 --- /dev/null +++ b/tests/test_agentperf_porter_golden.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit + golden-file tests for the AgentPerf harness-run porter. + +The golden e2e test pins the porter's behavior over time against a +materialized copy of a real harness run directory +(tests/fixtures/agentperf_porter/c1010 — the DSV4 c1010 GB300 baseline with +usernames anonymized, content otherwise verbatim: real engine configs, the +real 35-token worker env, the real client banner, real srun line shapes). + +To regenerate the goldens after an INTENTIONAL behavior change: + + python3 src/srtctl/benchmarks/scripts/agentperf/port_harness_run.py \ + tests/fixtures/agentperf_porter/c1010 \ + --out /tmp/recipe.yaml --workload-out /tmp/workload.yaml \ + --dataset-root tests/fixtures/agentperf_porter/c1010/datasets + +then replace the fixture-root prefix with and the workload output +path with and copy over expected-{recipe,workload}.yaml. +""" + +import importlib.util +from pathlib import Path + +import yaml + +from srtctl.benchmarks.base import SCRIPTS_DIR + +_spec = importlib.util.spec_from_file_location( + "port_harness_run_golden", SCRIPTS_DIR / "agentperf" / "port_harness_run.py" +) +porter = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(porter) + +FIXTURE = Path(__file__).parent / "fixtures" / "agentperf_porter" + + +class TestRewriteUtils: + def test_rewrite_prefix_match(self): + assert porter.rewrite("/lustre/a/b", [("/lustre/", "/scratch/")]) == "/scratch/a/b" + + def test_rewrite_no_match_untouched(self): + assert porter.rewrite("/data/lustre/a", [("/lustre/", "/scratch/")]) == "/data/lustre/a" + + def test_rewrite_first_matching_rule_wins(self): + rules = [("/a/", "/x/"), ("/a/b/", "/y/")] + assert porter.rewrite("/a/b/c", rules) == "/x/b/c" + + def test_rewrite_tree_nested(self): + obj = {"k": "/lustre/x", "l": ["/lustre/y", {"m": "/lustre/z"}], "n": 3, "o": None} + out = porter.rewrite_tree(obj, [("/lustre/", "/scratch/")]) + assert out == {"k": "/scratch/x", "l": ["/scratch/y", {"m": "/scratch/z"}], "n": 3, "o": None} + + def test_absolute_paths_collects_nested_strings(self): + obj = {"a": "/one", "b": ["x", "/two", {"c": "/three", "d": 5}]} + assert sorted(porter.absolute_paths(obj)) == ["/one", "/three", "/two"] + + +class TestEnvTranslation: + def test_drop_rename_passthrough(self): + env = porter.translate_env( + ["CUDA_VISIBLE_DEVICES=0,1", "PATH=/x", "DYN_KV_BLOCK_SIZE=128", + "DYN_UCX_TLS=sm,tcp", "TLLM_LOG_LEVEL=INFO"] + ) + assert "CUDA_VISIBLE_DEVICES" not in env and "PATH" not in env + assert env["DYN_TRTLLM_KV_BLOCK_SIZE"] == "128" + assert env["UCX_TLS"] == "sm,tcp" + assert env["TLLM_LOG_LEVEL"] == "INFO" + + def test_dyn_log_pinned_but_not_overridden(self): + assert porter.translate_env([])["DYN_LOG"] == "info" + assert porter.translate_env(["DYN_LOG=debug"])["DYN_LOG"] == "debug" + + def test_quoted_env_tokens_picks_env_list_not_scalar_args(self): + line = "srun x '1010' y '0-0' z 'A_B=1 C_D=2 E_F=3' w 'single'" + assert porter._quoted_env_tokens(line) == ["A_B=1", "C_D=2", "E_F=3"] + + def test_quoted_env_tokens_none_when_no_env_list(self): + assert porter._quoted_env_tokens("srun '1010' 'plain words here'") == [] + + +class TestFrontendArgs: + def test_defaults_and_kv_events_off(self): + args = porter.build_frontend_args({}) + assert args["router-mode"] == "kv" + assert args["no-router-kv-events"] is True + assert args["kv-cache-block-size"] == 128 + assert args["enforce-disagg"] is True + assert "dyn-tool-call-parser" not in args + + def test_env_driven_values(self): + args = porter.build_frontend_args( + {"ROUTER_MODE": "round-robin", "DYN_FRONTEND_ENABLE_KV_EVENTS": "1", + "DYN_KV_BLOCK_SIZE": "64", "DYN_REQUEST_PLANE": "nats"} + ) + assert args["router-mode"] == "round-robin" + assert args["no-router-kv-events"] is False + assert args["kv-cache-block-size"] == 64 + assert args["request-plane"] == "nats" + + +class TestSettleStem: + def test_settle_regex_int_and_float(self): + assert porter.SETTLE_RE.search("m__1010u__phase0__dur2400s__settle240s__traj3.json").group(1) == "240" + assert porter.SETTLE_RE.search("m__8u__phase1__dur300.5s__settle2.5s__traj3.txt").group(1) == "2.5" + + +class TestGoldenEndToEnd: + """Run the porter on the materialized real run dir; outputs must be byte-stable.""" + + def _normalize(self, text: str, workload_out: Path) -> str: + return text.replace(str(FIXTURE), "").replace(str(workload_out.resolve()), "") + + def test_golden_outputs(self, tmp_path): + out, wl = tmp_path / "recipe.yaml", tmp_path / "workload.yaml" + rc = porter.main([ + str(FIXTURE / "c1010"), "--out", str(out), "--workload-out", str(wl), + "--dataset-root", str(FIXTURE / "c1010" / "datasets"), + ]) + assert rc == 0 + for generated, golden in ((out, "expected-recipe.yaml"), (wl, "expected-workload.yaml")): + got = self._normalize(generated.read_text(), wl) + want = (FIXTURE / golden).read_text() + assert got == want, ( + f"{golden} drifted from the golden output.\n" + "If the change is intentional, regenerate the goldens (see module docstring)." + ) + + def test_golden_recipe_loads_into_schema(self, tmp_path): + """The golden recipe (with placeholders filled) must satisfy SrtConfig's schema.""" + from srtctl.core.schema import SrtConfig + + raw = (FIXTURE / "expected-recipe.yaml").read_text() + raw = raw.replace("", str(FIXTURE)).replace("", "/workloads/w.yaml") + cfg = SrtConfig.Schema().load(yaml.safe_load(raw)) + assert cfg.benchmark.type == "agentperf" + assert cfg.resources.prefill_workers == 5 + assert cfg.backend.numa_cpu_bind is False