PipelinePerfTest - add test for idle state numbers

.github/workflows/pipeline-perf-test-continuous.yml

@@ -93,6 +93,11 @@ jobs:
               python orchestrator/run_orchestrator.py --config "$config"
             done
 
+        - name: Run idle state performance test
+          run: |
+            cd tools/pipeline_perf_test
+            python orchestrator/run_orchestrator.py --config test_suites/integration/continuous/idle-state-docker.yaml
+
         - name: Upload benchmark results for processing
           uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
           with:
@@ -105,6 +110,12 @@ jobs:
             name: benchmark-results-saturation
             path: tools/pipeline_perf_test/results/continuous_saturation*/gh-actions-benchmark/*.json
 
+        - name: Upload benchmark results for processing (Idle State)
+          uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+          with:
+            name: benchmark-results-idle-state
+            path: tools/pipeline_perf_test/results/idle_state/gh-actions-benchmark/*.json
+
         - name: Add benchmark link to job summary
           run: |
             echo "### Benchmark Results" >> $GITHUB_STEP_SUMMARY
@@ -135,6 +146,13 @@ jobs:
               merge-multiple: true
               path: results-saturation
 
+          - name: Download benchmark artifacts (Idle State)
+            uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
+            with:
+              pattern: benchmark-results-idle-state*
+              merge-multiple: true
+              path: results-idle-state
+
           - name: Consolidate pipeline benchmark data
             run: |
               bash ./.github/workflows/scripts/consolidate-benchmarks.sh results-pipeline output-pipeline.json
@@ -143,6 +161,10 @@ jobs:
             run: |
               bash ./.github/workflows/scripts/consolidate-benchmarks.sh results-saturation output-saturation.json
 
+          - name: Consolidate idle state benchmark data
+            run: |
+              bash ./.github/workflows/scripts/consolidate-benchmarks.sh results-idle-state output-idle-state.json
+
           - name: Update pipeline benchmark data and deploy to GitHub Pages
             uses: benchmark-action/github-action-benchmark@4bdcce38c94cec68da58d012ac24b7b1155efe8b # v1.20.7
             with:
@@ -167,8 +189,21 @@ jobs:
               auto-push: true
               save-data-file: true
 
+          - name: Update idle state benchmark data and deploy to GitHub Pages
+            uses: benchmark-action/github-action-benchmark@4bdcce38c94cec68da58d012ac24b7b1155efe8b # v1.20.7
+            with:
+              tool: "customSmallerIsBetter"
+              output-file-path: output-idle-state.json
+              gh-pages-branch: benchmarks
+              max-items-in-chart: 100
+              github-token: ${{ secrets.GITHUB_TOKEN }}
+              benchmark-data-dir-path: "docs/benchmarks/continuous-idle-state"
+              auto-push: true
+              save-data-file: true
+
           - name: Add benchmark link to job summary
             run: |
               echo "### Benchmark Results" >> $GITHUB_STEP_SUMMARY
               echo "[View the pipeline benchmark results here](https://open-telemetry.github.io/otel-arrow/benchmarks/continuous/)" >> $GITHUB_STEP_SUMMARY
               echo "[View the saturation benchmark results here](https://open-telemetry.github.io/otel-arrow/benchmarks/continuous-saturation/)" >> $GITHUB_STEP_SUMMARY
+              echo "[View the idle state benchmark results here](https://open-telemetry.github.io/otel-arrow/benchmarks/continuous-idle-state/)" >> $GITHUB_STEP_SUMMARY

tools/pipeline_perf_test/test_suites/integration/configs/idle_state_report.yaml

@@ -0,0 +1,121 @@
+queries:
+  - name: Calculate observation window for this test
+    sql: |
+      CREATE TABLE observation_window AS WITH event_times AS (
+        SELECT
+          "attributes.test.name" AS test_name,
+          name AS event_name,
+          TO_TIMESTAMP(timestamp / 1000000000.0) AS timestamp
+        FROM events
+        WHERE name IN ('observation_start', 'observation_stop')
+      ),
+      observation_windows AS (
+        SELECT
+          start.test_name,
+          start.timestamp AS start_ts,
+          stop.timestamp AS stop_ts
+        FROM
+          event_times start
+          JOIN event_times stop ON start.test_name = stop.test_name
+        WHERE
+          start.event_name = 'observation_start'
+          AND stop.event_name = 'observation_stop'
+          AND start.timestamp < stop.timestamp
+      )
+      SELECT w.*
+      FROM metadata_row
+      JOIN observation_windows w ON metadata_row."test.name" = w.test_name
+
+  - name: Create component resource metrics
+    sql: |
+      CREATE TABLE component_resource_metrics AS
+        WITH metrics_filtered AS (
+          SELECT *,
+          "metric_attributes.component_name" AS component_name
+          FROM metrics
+          WHERE metric_name IN ('container.cpu.usage', 'container.memory.usage')
+        ),
+        observation_only AS (
+          SELECT *
+          FROM observation_window
+        )
+        SELECT
+          m.component_name,
+          metric_name,
+          MIN(m.value) AS min_value,
+          AVG(m.value) AS avg_value,
+          MAX(m.value) AS max_value
+        FROM
+          observation_only AS ow
+        JOIN
+          metrics_filtered AS m
+          ON m.timestamp BETWEEN ow.start_ts AND ow.stop_ts
+        GROUP BY
+          m.component_name, m.metric_name
+        ORDER BY
+          m.component_name DESC;
+
+  - name: Idle State Summary
+    sql: |
+      CREATE TABLE idle_state_summary AS
+      SELECT
+        component_name,
+        ROUND(AVG(CASE WHEN metric_name = 'container.cpu.usage' THEN avg_value END), 2) AS avg_cpu_percent,
+        ROUND(MAX(CASE WHEN metric_name = 'container.cpu.usage' THEN max_value END), 2) AS max_cpu_percent,
+        ROUND(AVG(CASE WHEN metric_name = 'container.memory.usage' THEN avg_value / 1024.0 / 1024.0 END), 2) AS avg_memory_mb,
+        ROUND(MAX(CASE WHEN metric_name = 'container.memory.usage' THEN max_value / 1024.0 / 1024.0 END), 2) AS max_memory_mb
+      FROM component_resource_metrics
+      GROUP BY component_name
+
+  - name: Create Final Github Actions Format table
+    sql: |
+      CREATE TABLE gh_actions_benchmark AS
+          SELECT 'idle_cpu_percentage_avg' AS name, '%' AS unit,
+                (crm.avg_value) * 100 AS value,
+                metadata_row."test.suite" || '/' || metadata_row."test.name" || ' - Idle CPU % (Avg)' AS extra
+          FROM component_resource_metrics crm
+          CROSS JOIN metadata_row
+          WHERE crm.component_name = 'df-engine' AND crm.metric_name = 'container.cpu.usage'
+
+          UNION ALL
+
+          SELECT 'idle_cpu_percentage_max' AS name, '%' AS unit,
+                (crm.max_value) * 100 AS value,
+                metadata_row."test.suite" || '/' || metadata_row."test.name" || ' - Idle CPU % (Max)' AS extra
+          FROM component_resource_metrics crm
+          CROSS JOIN metadata_row
+          WHERE crm.component_name = 'df-engine' AND crm.metric_name = 'container.cpu.usage'
+
+          UNION ALL
+
+          SELECT 'idle_ram_mib_avg' AS name, 'MiB' AS unit, avg_value / 1024 / 1024 AS value,
+                metadata_row."test.suite" || '/' || metadata_row."test.name" || ' - Idle RAM (MiB) (Avg)' AS extra
+          FROM component_resource_metrics
+          CROSS JOIN metadata_row
+          WHERE component_name = 'df-engine' AND metric_name = 'container.memory.usage'
+
+          UNION ALL
+
+          SELECT 'idle_ram_mib_max' AS name, 'MiB' AS unit, max_value / 1024 / 1024 AS value,
+                metadata_row."test.suite" || '/' || metadata_row."test.name" || ' - Idle RAM (MiB) (Max)' AS extra
+          FROM component_resource_metrics
+          CROSS JOIN metadata_row
+          WHERE component_name = 'df-engine' AND metric_name = 'container.memory.usage'
+
+          UNION ALL
+
+          SELECT 'idle_test_duration' AS name, 'seconds' AS unit,
+                EXTRACT(EPOCH FROM (stop_ts - start_ts)) AS value,
+                metadata_row."test.suite" || '/' || metadata_row."test.name" || ' - Idle Test Duration' AS extra
+          FROM observation_window
+          CROSS JOIN metadata_row;
+
+result_tables:
+  - name: observation_window
+    description: The observation window for idle state measurement
+  - name: component_resource_metrics
+    description: CPU and Memory metrics during idle state
+  - name: idle_state_summary
+    description: Summary of idle state performance metrics
+  - name: gh_actions_benchmark
+    description: The final metrics used with gh_actions_benchmark.

tools/pipeline_perf_test/test_suites/integration/continuous/idle-state-docker.yaml

@@ -0,0 +1,114 @@
+# Idle State Performance Test
+# Measures baseline resource consumption with no active telemetry traffic
+name: Continuous - Idle State Performance
+components:
+  df-engine:
+    deployment:
+      docker:
+        image: df_engine:latest
+        network: testbed
+        ports:
+          - "8086:8080"
+        volumes:
+          - 'test_suites/integration/configs/engine/config.rendered.yaml:/home/dataflow/config.yaml:ro'
+        command:
+          - "--pipeline"
+          - "./config.yaml"
+          - "--core-id-range"
+          - "0-0"
+          - "--http-admin-bind"
+          - "0.0.0.0:8080"
+    monitoring:
+      docker_component:
+        allocated_cores: 1
+      prometheus:
+        endpoint: http://localhost:8086/telemetry/metrics?format=prometheus&reset=false
+
+tests:
+  - name: Idle State Baseline
+    steps:
+      - name: Deploy Dataflow Engine
+        action:
+          component_action:
+            phase: deploy
+            target: df-engine
+        hooks:
+          run:
+            pre:
+              - render_template:
+                  template_path: 'test_suites/integration/templates/configs/engine/continuous/otlp-attr-otlp.yaml'
+                  output_path: ./test_suites/integration/configs/engine/config.rendered.yaml
+                  variables:
+                    backend_hostname: localhost
+            post:
+              - ready_check_http:
+                  url: http://localhost:8086/telemetry/metrics?reset=false
+                  method: GET
+                  expected_status_code: 200
+
+      - name: Wait for Startup Stabilization
+        action:
+          wait:
+            delay_seconds: 5
+        hooks:
+          run:
+            pre:
+              - record_event:
+                  name: stabilization_start
+            post:
+              - record_event:
+                  name: stabilization_complete
+
+      - name: Monitor Engine
+        action:
+          component_action:
+            phase: start_monitoring
+            target: df-engine
+
+      - name: Observe Idle State
+        action:
+          wait:
+            delay_seconds: 15
+        hooks:
+          run:
+            pre:
+              - record_event:
+                  name: observation_start
+            post:
+              - record_event:
+                  name: observation_stop
+
+      - name: Stop Monitoring
+        action:
+          component_action:
+            phase: stop_monitoring
+            target: df-engine
+
+      - name: Destroy Engine
+        action:
+          component_action:
+            phase: destroy
+            target: df-engine
+
+      - name: Run Report
+        action:
+          wait:
+            delay_seconds: 0
+        hooks:
+          run:
+            post:
+              - print_container_logs: {}
+              - sql_report:
+                  name: Idle State Performance Report
+                  report_config_file: ./test_suites/integration/configs/idle_state_report.yaml
+                  output:
+                    - format:
+                        template: {}
+                      destination:
+                        console: {}
+                    - format:
+                        template:
+                          path: ./test_suites/integration/templates/reports/gh-action-sqlreport.j2
+                      destination:
+                        file:
+                          directory: results/idle_state/gh-actions-benchmark