test(benchmark): unit + golden-file tests for the agentperf porter

tests/fixtures/agentperf_porter/c1010/3_output_CTX_0.log

@@ -0,0 +1,2 @@
+clean worker startup
+no pinning in this baseline

tests/fixtures/agentperf_porter/c1010/client.log

@@ -0,0 +1,28 @@
+0: warning: `VIRTUAL_ENV=/opt/dynamo/venv` does not match the project environment path `/tmp/agentperf-570849/venv` and will be ignored; use `--active` to target the active environment instead
+0: ============================================================
+0: AA-AGENTPERF: Deterministic Load Test
+0: ============================================================
+0:   Base URL: http://nvl72d102-T10:8333/v1
+0:   Model: /lustre/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig
+0:   Server type: trtllm
+0:   Concurrencies: [1010]
+0:   Max workers: 8
+0:   Connection pool: 8192 (auto-sized, keepalive expiry disabled)
+0:   Phase Timeout: 2400.0s
+0:   Max ISL offset: 10
+0:   User assignments: /tmp/agentperf-570849/data/500-sample-user-trajectory-assignments-20k.json
+0:   Max tokens: 2000
+0:   Reasoning effort: high
+0:   Tool-call delays: enabled
+0:   Conversation routing headers: enabled
+0:   Dynamo conv-aware routing (X-Dynamo-Session-ID): enabled
+0:   Power sampling: disabled
+0:   Results dir: /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/agentperf
+0:   Request log: /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/agentperf/requests.jsonl
+0:   Seed: 42
+0: ============================================================
+0: Loading trajectories from /tmp/agentperf-570849/data/aa-rwlt_coding-agent-scenario_tuning_v2_500traj.jsonl...
+0: Loaded 500 trajectories
+0: Total requests across all trajectories: 18997
+0: Loaded user assignments from /tmp/agentperf-570849/data/500-sample-user-trajectory-assignments-20k.json: 20000 users, 3 trajectories each
+0: Running 1 phase(s): concurrencies=[1010]

tests/fixtures/agentperf_porter/c1010/client_cmds_base.sh

@@ -0,0 +1 @@
+srun -l --export=ALL --nodelist <node0_placeholder> --container-name=agentperf-${SLURM_JOB_ID} --container-image=/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/images/dynamo_ff959852b740_trtllm_5b427d9d5d.sqsh --container-mounts=/lustre:/lustre,/scratch:/scratch --no-container-mount-home --mpi=pmix --overlap -N 1 -n 1 bash -c ' set -euo pipefail; export RUNTIME=/tmp/agentperf-${SLURM_JOB_ID}; test -f ${RUNTIME}/READY; export HOME=${RUNTIME}/home CARGO_HOME=${RUNTIME}/cargo RUSTUP_HOME=${RUNTIME}/rustup CARGO_TARGET_DIR=${RUNTIME}/target UV_PROJECT_ENVIRONMENT=${RUNTIME}/venv UV_CACHE_DIR=${RUNTIME}/uv-cache; export PATH=${HOME}/.local/bin:${CARGO_HOME}/bin:/usr/local/bin:/usr/bin:/bin; mkdir -p /logs/agentperf; cd /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/agentperf-client-worktrees/9e4aae134986eee8fdb44264c1d918a2ca3f18f7; uv run python agentperf/run.py --config ${RUNTIME}/benchmark_config.yaml --concurrencies 1010 --request-log-path /logs/agentperf/requests.jsonl --results-dir /logs/agentperf' &> /logs/client.log

tests/fixtures/agentperf_porter/c1010/ctx_config.yaml

@@ -0,0 +1,44 @@
+num_serve_frontends: 4
+max_batch_size: 128
+max_num_tokens: 8192
+max_seq_len: 147456
+tensor_parallel_size: 4
+moe_expert_parallel_size: 4
+enable_attention_dp: true
+pipeline_parallel_size: 1
+enable_chunked_prefill: true
+print_iter_log: true
+cuda_graph_config: null
+enable_autotuner: true
+disable_overlap_scheduler: false
+moe_config:
+  backend: MEGAMOE_CUTEDSL
+kv_cache_config:
+  enable_block_reuse: true
+  free_gpu_memory_fraction: 0.4
+  dtype: fp8
+  host_cache_size: 107374182400
+  pool_ratio:
+  - 0.45
+  - 0.3
+  - 0.25
+cache_transceiver_config:
+  max_tokens_in_buffer: 147456
+  backend: NIXL
+  transceiver_runtime: PYTHON
+  kv_transfer_timeout_ms: 600000
+scheduler_config:
+  capacity_scheduler_policy: MAX_UTILIZATION
+attention_dp_config:
+  enable_kv_cache_aware_routing: false
+  kv_cache_routing_conversation_affinity: true
+  kv_cache_routing_max_sessions: 65536
+return_perf_metrics: false
+enable_iter_perf_stats: false
+enable_iter_req_stats: false
+nvfp4_gemm_config:
+  allowed_backends:
+  - cutlass
+  - cublaslt
+  - cutedsl
+  - cuda_core

tests/fixtures/agentperf_porter/c1010/datasets/500-sample-user-trajectory-assignments-20k.json

@@ -0,0 +1 @@
+{"metadata":{"per_user":3,"n_users_total":20000},"assignments":{}}

tests/fixtures/agentperf_porter/c1010/datasets/aa-rwlt_coding-agent-scenario_tuning_v2_500traj.jsonl

@@ -0,0 +1 @@
+{"conversation_id":"aa-rwlt-coding-agent-001","messages":[]}

tests/fixtures/agentperf_porter/c1010/gen_config.yaml

@@ -0,0 +1,53 @@
+tensor_parallel_size: 32
+moe_expert_parallel_size: 32
+enable_attention_dp: true
+enable_lm_head_tp_in_adp: true
+pipeline_parallel_size: 1
+max_batch_size: 16
+max_num_tokens: 128
+max_seq_len: 147456
+print_iter_log: true
+enable_autotuner: false
+kv_cache_config:
+  enable_block_reuse: false
+  free_gpu_memory_fraction: 0.7
+  dtype: fp8
+  host_cache_size: 0
+  avg_seq_len: 39040
+moe_config:
+  backend: MEGAMOE_CUTEDSL
+  use_low_precision_moe_combine: true
+  load_balancer: /lustre/fsw/portfolios/coreai/projects/team/users/bench-user/eplb/moe_load_balancer_gen_ep32_slots384_dspark.yaml
+cache_transceiver_config:
+  max_tokens_in_buffer: 147456
+  backend: NIXL
+  transceiver_runtime: PYTHON
+  kv_transfer_timeout_ms: 600000
+stream_interval: 100
+num_postprocess_workers: 8
+cuda_graph_config:
+  enable_padding: true
+  batch_sizes:
+  - 1
+  - 2
+  - 4
+  - 8
+  - 16
+  - 32
+  - 64
+speculative_config:
+  decoding_type: DSpark
+  speculative_model: /lustre/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig
+  max_draft_len: 6
+  block_size: 6
+nvfp4_gemm_config:
+  allowed_backends:
+  - cutlass
+  - cublaslt
+  - cutedsl
+  - cuda_core
+sparse_attention_config:
+  algorithm: deepseek_v4
+return_perf_metrics: false
+enable_iter_perf_stats: false
+enable_iter_req_stats: false

tests/fixtures/agentperf_porter/c1010/job_params.env

@@ -0,0 +1,14 @@
+PROJECT_ROOT="/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs"
+MODEL_PATH="/lustre/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig"
+PORT=8100
+CONCURRENCIES="1010"
+BENCHMARK_CONFIG="/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-runs/results/bm_agentperf_c1010/benchmark_config.yaml"
+SERVER_LOG_LEVEL="verbose"
+CONTAINER_IMAGE="/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/images/dynamo_ff959852b740_trtllm_5b427d9d5d.sqsh"
+CONTAINER_MOUNTS="/lustre:/lustre,/scratch:/scratch"
+HARNESS_DIR="/lustre/fsw/portfolios/coreai/projects/team/users/bench-user/disagg-harness/38808e274dd8e18e"
+ENABLE_METRICS_POLL=0
+COLLECT_POWER=1
+SERVER_BACKEND="dynamo"
+DYNAMO_JOB_ENV='DYN_LOGGING_JSONL=false OTEL_EXPORT_ENABLED=false'
+NSYS_ENABLED=0

tests/fixtures/agentperf_porter/expected-recipe.yaml

@@ -0,0 +1,230 @@
+name: agentperf-porter
+model:
+  path: /scratch/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig
+  container: /scratch/fsw/portfolios/coreai/projects/team/users/bench-user/images/dynamo_ff959852b740_trtllm_5b427d9d5d.sqsh
+  precision: fp4
+resources:
+  gpu_type: gb300
+  gpus_per_node: 4
+  prefill_nodes: 5
+  prefill_workers: 5
+  decode_nodes: 8
+  decode_workers: 1
+dynamo:
+  install: false
+  request_plane: tcp
+  event_plane: zmq
+backend:
+  type: trtllm
+  numa_memory_bind: true
+  numa_cpu_bind: false
+  prefill_environment:
+    TLLM_LOG_LEVEL: INFO
+    TRTLLM_SERVER_DISABLE_GC: '1'
+    TRTLLM_WORKER_DISABLE_GC: '1'
+    TRTLLM_ENABLE_PDL: '1'
+    NCCL_GRAPH_MIXING_SUPPORT: '0'
+    MIMALLOC_PURGE_DELAY: '0'
+    TRTLLM_DISABLE_KV_CACHE_RATIO_UPDATE: '1'
+    TRTLLM_MHC_ENABLE_FUSED_HC: '1'
+    TRTLLM_MOE_A2A_DISABLE_CFT_COUNTED_WRITES: '1'
+    TRTLLM_MLA_EXTRA_OVERLAP: '1'
+    TRTLLM_FUSED_FP8_QUANT_PACK: '1'
+    TRTLLM_DSV4_MEM_OPTS: '1'
+    TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE: '1'
+    HF_HUB_OFFLINE: '1'
+    TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN: '1'
+    CUDA_SCALE_LAUNCH_QUEUES: 4x
+    TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10'
+    TLLM_NUMA_AWARE_WORKER_AFFINITY: '0'
+    PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+    DYN_HEALTH_CHECK_ENABLED: 'false'
+    DYN_ENGINE_CONV_AFFINITY: '1'
+    DYN_TOKENIZER: fastokens
+    DYN_TCP_REQUEST_TIMEOUT: '30'
+    UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp
+    DYN_REQUEST_PLANE: tcp
+    DYN_WORKER_PUBLISH_KV_EVENTS: '0'
+    DYN_LOGGING_JSONL: 'false'
+    OTEL_EXPORT_ENABLED: 'false'
+    TRTLLM_SERVE_ENABLE_ORJSON: '1'
+    DYN_TRTLLM_KV_BLOCK_SIZE: '128'
+    PYTHONUNBUFFERED: '1'
+    DYN_TOOL_CALL_PARSER: deepseek_v4
+    DYN_REASONING_PARSER: deepseek_v4
+    TRTLLM_SKIP_KV_CACHE_ESTIMATION: '1'
+    DYN_LOG: info
+  decode_environment:
+    TLLM_LOG_LEVEL: INFO
+    TRTLLM_SERVER_DISABLE_GC: '1'
+    TRTLLM_WORKER_DISABLE_GC: '1'
+    TRTLLM_ENABLE_PDL: '1'
+    NCCL_GRAPH_MIXING_SUPPORT: '0'
+    MIMALLOC_PURGE_DELAY: '0'
+    TRTLLM_DISABLE_KV_CACHE_RATIO_UPDATE: '1'
+    TRTLLM_MHC_ENABLE_FUSED_HC: '1'
+    TRTLLM_MOE_A2A_DISABLE_CFT_COUNTED_WRITES: '1'
+    TRTLLM_MLA_EXTRA_OVERLAP: '1'
+    TRTLLM_FUSED_FP8_QUANT_PACK: '1'
+    TRTLLM_DSV4_MEM_OPTS: '1'
+    TRTLLM_DSV4_ENABLE_SWA_SCRATCH_REUSE: '1'
+    HF_HUB_OFFLINE: '1'
+    TLLM_DISAGG_GEN_PRIORITIZE_FIRST_TOKEN: '1'
+    CUDA_SCALE_LAUNCH_QUEUES: 4x
+    TLLM_ADP_ROUTER_MATCH_RATE_THRESHOLD: '0.10'
+    TLLM_NUMA_AWARE_WORKER_AFFINITY: '0'
+    PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True
+    DYN_HEALTH_CHECK_ENABLED: 'false'
+    DYN_ENGINE_CONV_AFFINITY: '1'
+    DYN_TOKENIZER: fastokens
+    DYN_TCP_REQUEST_TIMEOUT: '30'
+    UCX_TLS: cuda_ipc,cuda_copy,sm,self,tcp
+    DYN_REQUEST_PLANE: tcp
+    DYN_WORKER_PUBLISH_KV_EVENTS: '0'
+    DYN_LOGGING_JSONL: 'false'
+    OTEL_EXPORT_ENABLED: 'false'
+    TRTLLM_SERVE_ENABLE_ORJSON: '1'
+    DYN_TRTLLM_KV_BLOCK_SIZE: '128'
+    PYTHONUNBUFFERED: '1'
+    DYN_TOOL_CALL_PARSER: deepseek_v4
+    DYN_REASONING_PARSER: deepseek_v4
+    DYN_LOG: info
+  trtllm_config:
+    prefill:
+      num_serve_frontends: 4
+      max_batch_size: 128
+      max_num_tokens: 8192
+      max_seq_len: 147456
+      tensor_parallel_size: 4
+      moe_expert_parallel_size: 4
+      enable_attention_dp: true
+      pipeline_parallel_size: 1
+      enable_chunked_prefill: true
+      print_iter_log: true
+      cuda_graph_config: null
+      enable_autotuner: true
+      disable_overlap_scheduler: false
+      moe_config:
+        backend: MEGAMOE_CUTEDSL
+      kv_cache_config:
+        enable_block_reuse: true
+        free_gpu_memory_fraction: 0.4
+        dtype: fp8
+        host_cache_size: 107374182400
+        pool_ratio:
+        - 0.45
+        - 0.3
+        - 0.25
+      cache_transceiver_config:
+        max_tokens_in_buffer: 147456
+        backend: NIXL
+        transceiver_runtime: PYTHON
+        kv_transfer_timeout_ms: 600000
+      scheduler_config:
+        capacity_scheduler_policy: MAX_UTILIZATION
+      attention_dp_config:
+        enable_kv_cache_aware_routing: false
+        kv_cache_routing_conversation_affinity: true
+        kv_cache_routing_max_sessions: 65536
+      return_perf_metrics: false
+      enable_iter_perf_stats: false
+      enable_iter_req_stats: false
+      nvfp4_gemm_config:
+        allowed_backends:
+        - cutlass
+        - cublaslt
+        - cutedsl
+        - cuda_core
+    decode:
+      tensor_parallel_size: 32
+      moe_expert_parallel_size: 32
+      enable_attention_dp: true
+      enable_lm_head_tp_in_adp: true
+      pipeline_parallel_size: 1
+      max_batch_size: 16
+      max_num_tokens: 128
+      max_seq_len: 147456
+      print_iter_log: true
+      enable_autotuner: false
+      kv_cache_config:
+        enable_block_reuse: false
+        free_gpu_memory_fraction: 0.7
+        dtype: fp8
+        host_cache_size: 0
+        avg_seq_len: 39040
+      moe_config:
+        backend: MEGAMOE_CUTEDSL
+        use_low_precision_moe_combine: true
+        load_balancer: /scratch/fsw/portfolios/coreai/projects/team/users/bench-user/eplb/moe_load_balancer_gen_ep32_slots384_dspark.yaml
+      cache_transceiver_config:
+        max_tokens_in_buffer: 147456
+        backend: NIXL
+        transceiver_runtime: PYTHON
+        kv_transfer_timeout_ms: 600000
+      stream_interval: 100
+      num_postprocess_workers: 8
+      cuda_graph_config:
+        enable_padding: true
+        batch_sizes:
+        - 1
+        - 2
+        - 4
+        - 8
+        - 16
+        - 32
+        - 64
+      speculative_config:
+        decoding_type: DSpark
+        speculative_model: /scratch/fsw/portfolios/coreai/projects/model-ci/artifacts/model/nvidia_deepseek-v4-pro-nvfp4-dspark/hf/hf-318bf60_orig
+        max_draft_len: 6
+        block_size: 6
+      nvfp4_gemm_config:
+        allowed_backends:
+        - cutlass
+        - cublaslt
+        - cutedsl
+        - cuda_core
+      sparse_attention_config:
+        algorithm: deepseek_v4
+      return_perf_metrics: false
+      enable_iter_perf_stats: false
+      enable_iter_req_stats: false
+frontend:
+  type: dynamo
+  enable_multiple_frontends: false
+  args:
+    router-mode: kv
+    no-router-kv-events: true
+    kv-cache-block-size: 128
+    enforce-disagg: true
+    request-plane: tcp
+    event-plane: zmq
+  env:
+    ROUTER_MODE: kv
+    DYN_KV_BLOCK_SIZE: '128'
+    DEBUG_ROUTER: '0'
+    PYTHONUNBUFFERED: '1'
+    DYN_ROUTER_SESSION_AFFINITY_TTL_SECS: '14400'
+    DYN_ROUTER_TEMPERATURE: '0'
+    DYN_ROUTER_QUEUE_THRESHOLD: None
+    DYN_TOKENIZER_CACHE: '1'
+    DYN_TOKENIZER_CACHE_BYTES: '8000000000'
+    DYN_TOKENIZER: fastokens
+    DYN_TCP_REQUEST_TIMEOUT: '30'
+    DYN_REQUEST_PLANE: tcp
+    DYN_FRONTEND_ENABLE_KV_EVENTS: '0'
+    DYN_LOGGING_JSONL: 'false'
+    OTEL_EXPORT_ENABLED: 'false'
+    HOME: /tmp
+infra:
+  etcd_nats_dedicated_node: false
+observability:
+  enabled: false
+benchmark:
+  type: agentperf
+  client_placement: last_decode
+  concurrency: 1010
+  agentperf_client_dir: /scratch/fsw/portfolios/coreai/projects/team/users/bench-user/agentperf-client-worktrees/9e4aae134986eee8fdb44264c1d918a2ca3f18f7
+  agentperf_config: <WORKLOAD_OUT>
+extra_mount:
+- /scratch:/scratch