[cuebot/docs/sandbox] Add Elasticsearch queue metric, enhance dashboard, and improve docs

- Implement cue_elasticsearch_index_queue_size Prometheus metric using @Autowired for ElasticsearchClient in PrometheusMetricsCollector
- Update Grafana dashboard panel colors and labels:
    - Frames Completed: DEAD (red), SUCCEEDED (green), WAITING (yellow)
    - Events Published: human-readable labels with consistent colors
- Add monitoring documentation screenshots for all components:
    - Grafana, Prometheus, Kafka UI, Elasticsearch, Kibana
- Update all monitoring docs (Quick Start, Concepts, User Guides, Reference, Tutorials, Developer Guide) with visual references
- Add load_test_jobs.py script for generating test monitoring data
- Update monitor_events.py consumer script

Author: ramonfigueiredo

.gitignore

@@ -32,4 +32,5 @@ docs/bin/
 sandbox/kafka-data
 sandbox/zookeeper-data
 sandbox/zookeeper-logs
+sandbox/rqd/shots/
 docs/_data/version.yml

cuebot/src/main/java/com/imageworks/spcue/PrometheusMetricsCollector.java

@@ -8,6 +8,7 @@
 import com.imageworks.spcue.dispatcher.DispatchQueue;
 import com.imageworks.spcue.dispatcher.HostReportHandler;
 import com.imageworks.spcue.dispatcher.HostReportQueue;
+import com.imageworks.spcue.monitoring.ElasticsearchClient;
 
 import io.prometheus.client.Counter;
 import io.prometheus.client.Gauge;
@@ -26,6 +27,9 @@ public class PrometheusMetricsCollector {
 
     private HostReportQueue reportQueue;
 
+    @Autowired(required = false)
+    private ElasticsearchClient elasticsearchClient;
+
     private boolean enabled;
 
     // BookingQueue bookingQueue
@@ -265,6 +269,12 @@ public void collectPrometheusMetrics() {
                     .set(reportQueue.getTaskCount());
             reportQueueRejectedTotal.labels(this.deployment_environment, this.cuebot_host)
                     .set(reportQueue.getRejectedTaskCount());
+
+            // ElasticsearchClient queue
+            if (elasticsearchClient != null) {
+                elasticsearchIndexQueueSize.labels(this.deployment_environment, this.cuebot_host)
+                        .set(elasticsearchClient.getPendingIndexCount());
+            }
         }
     }
 
@@ -425,4 +435,8 @@ public void setDispatchQueue(DispatchQueue dispatchQueue) {
     public void setReportQueue(HostReportQueue reportQueue) {
         this.reportQueue = reportQueue;
     }
+
+    public void setElasticsearchClient(ElasticsearchClient elasticsearchClient) {
+        this.elasticsearchClient = elasticsearchClient;
+    }
 }

cuebot/src/main/java/com/imageworks/spcue/dispatcher/FrameCompleteHandler.java

@@ -61,6 +61,7 @@
 import com.imageworks.spcue.grpc.monitoring.EventType;
 import com.imageworks.spcue.grpc.monitoring.FrameEvent;
 import com.imageworks.spcue.grpc.monitoring.JobEvent;
+import com.imageworks.spcue.PrometheusMetricsCollector;
 
 /**
  * The FrameCompleteHandler encapsulates all logic necessary for processing FrameComplete reports
@@ -90,6 +91,7 @@ public class FrameCompleteHandler {
     private Environment env;
     private KafkaEventPublisher kafkaEventPublisher;
     private MonitoringEventBuilder monitoringEventBuilder;
+    private PrometheusMetricsCollector prometheusMetrics;
 
     /*
      * The last time a proc was unbooked for subscription or job balancing. Since there are so many
@@ -744,12 +746,35 @@ public void setKafkaEventPublisher(KafkaEventPublisher kafkaEventPublisher) {
         }
     }
 
+    public PrometheusMetricsCollector getPrometheusMetrics() {
+        return prometheusMetrics;
+    }
+
+    public void setPrometheusMetrics(PrometheusMetricsCollector prometheusMetrics) {
+        this.prometheusMetrics = prometheusMetrics;
+    }
+
     /**
      * Publishes a frame complete event to Kafka for monitoring purposes. This method is called
      * asynchronously to avoid blocking the dispatch thread.
      */
     private void publishFrameCompleteEvent(FrameCompleteReport report, DispatchFrame frame,
             FrameDetail frameDetail, FrameState newFrameState, VirtualProc proc) {
+        // Record Prometheus metrics for frame completion
+        if (prometheusMetrics != null) {
+            try {
+                prometheusMetrics.recordFrameCompleted(newFrameState.name(), frame.show);
+                prometheusMetrics.recordFrameRuntime(report.getRunTime(), frame.show, "render");
+                if (report.getFrame().getMaxRss() > 0) {
+                    prometheusMetrics.recordFrameMemory(report.getFrame().getMaxRss() * 1024L,
+                            frame.show, "render");
+                }
+            } catch (Exception e) {
+                logger.trace("Failed to record Prometheus metrics: {}", e.getMessage());
+            }
+        }
+
+        // Publish to Kafka if enabled
         if (kafkaEventPublisher == null || !kafkaEventPublisher.isEnabled()) {
             return;
         }

cuebot/src/main/java/com/imageworks/spcue/dispatcher/HostReportHandler.java

@@ -171,6 +171,12 @@ public void handleHostReport(HostReport report, boolean isBoot) {
             // Publish host report event to Kafka for monitoring
             publishHostReportEvent(report, isBoot);
 
+            // Record Prometheus metric for host report
+            if (prometheusMetrics != null) {
+                String facility = report.getHost().getFacility();
+                prometheusMetrics.recordHostReport(facility != null ? facility : "unknown");
+            }
+
             long swapOut = 0;
             if (report.getHost().getAttributesMap().containsKey("swapout")) {
                 swapOut = Integer.parseInt(report.getHost().getAttributesMap().get("swapout"));

cuebot/src/main/java/com/imageworks/spcue/monitoring/KafkaEventPublisher.java

@@ -46,6 +46,7 @@
 import com.imageworks.spcue.grpc.monitoring.LayerEvent;
 import com.imageworks.spcue.grpc.monitoring.ProcEvent;
 import com.imageworks.spcue.util.CueExceptionUtil;
+import com.imageworks.spcue.PrometheusMetricsCollector;
 
 /**
  * KafkaEventPublisher publishes monitoring events to Kafka topics for downstream processing. Events
@@ -77,6 +78,7 @@ public class KafkaEventPublisher extends ThreadPoolExecutor {
     private JsonFormat.Printer jsonPrinter;
     private String sourceCuebot;
     private boolean enabled = false;
+    private PrometheusMetricsCollector prometheusMetrics;
 
     public KafkaEventPublisher() {
         super(THREAD_POOL_SIZE_INITIAL, THREAD_POOL_SIZE_MAX, 10, TimeUnit.SECONDS,
@@ -171,7 +173,8 @@ public EventHeader.Builder createEventHeader(EventType eventType, String correla
     public void publishJobEvent(JobEvent event) {
         if (!enabled)
             return;
-        publishEvent(TOPIC_JOB_EVENTS, event.getJobId(), event);
+        publishEvent(TOPIC_JOB_EVENTS, event.getJobId(), event,
+                event.getHeader().getEventType().name());
     }
 
     /**
@@ -180,7 +183,8 @@ public void publishJobEvent(JobEvent event) {
     public void publishLayerEvent(LayerEvent event) {
         if (!enabled)
             return;
-        publishEvent(TOPIC_LAYER_EVENTS, event.getLayerId(), event);
+        publishEvent(TOPIC_LAYER_EVENTS, event.getLayerId(), event,
+                event.getHeader().getEventType().name());
     }
 
     /**
@@ -189,7 +193,8 @@ public void publishLayerEvent(LayerEvent event) {
     public void publishFrameEvent(FrameEvent event) {
         if (!enabled)
             return;
-        publishEvent(TOPIC_FRAME_EVENTS, event.getFrameId(), event);
+        publishEvent(TOPIC_FRAME_EVENTS, event.getFrameId(), event,
+                event.getHeader().getEventType().name());
     }
 
     /**
@@ -198,7 +203,8 @@ public void publishFrameEvent(FrameEvent event) {
     public void publishHostEvent(HostEvent event) {
         if (!enabled)
             return;
-        publishEvent(TOPIC_HOST_EVENTS, event.getHostName(), event);
+        publishEvent(TOPIC_HOST_EVENTS, event.getHostName(), event,
+                event.getHeader().getEventType().name());
     }
 
     /**
@@ -207,7 +213,8 @@ public void publishHostEvent(HostEvent event) {
     public void publishHostReportEvent(HostReportEvent event) {
         if (!enabled)
             return;
-        publishEvent(TOPIC_HOST_REPORTS, event.getHostName(), event);
+        publishEvent(TOPIC_HOST_REPORTS, event.getHostName(), event,
+                event.getHeader().getEventType().name());
     }
 
     /**
@@ -216,13 +223,19 @@ public void publishHostReportEvent(HostReportEvent event) {
     public void publishProcEvent(ProcEvent event) {
         if (!enabled)
             return;
-        publishEvent(TOPIC_PROC_EVENTS, event.getProcId(), event);
+        publishEvent(TOPIC_PROC_EVENTS, event.getProcId(), event,
+                event.getHeader().getEventType().name());
     }
 
     /**
      * Internal method to publish any protobuf message to a Kafka topic.
      */
-    private void publishEvent(String topic, String key, Message event) {
+    private void publishEvent(String topic, String key, Message event, String eventType) {
+        // Update queue size metric
+        if (prometheusMetrics != null) {
+            prometheusMetrics.setMonitoringEventQueueSize(getQueue().size());
+        }
+
         try {
             execute(() -> {
                 try {
@@ -237,6 +250,10 @@ private void publishEvent(String topic, String key, Message event) {
                         } else {
                             logger.trace("Published event to {}, partition={}, offset={}", topic,
                                     metadata.partition(), metadata.offset());
+                            // Record successful publish
+                            if (prometheusMetrics != null) {
+                                prometheusMetrics.incrementMonitoringEventPublished(eventType);
+                            }
                         }
                     });
                 } catch (Exception e) {
@@ -246,6 +263,10 @@ private void publishEvent(String topic, String key, Message event) {
             });
         } catch (RejectedExecutionException e) {
             logger.warn("Event queue is full, dropping event for topic {}", topic);
+            // Record dropped event
+            if (prometheusMetrics != null) {
+                prometheusMetrics.incrementMonitoringEventDropped(eventType);
+            }
         }
     }
 
@@ -269,4 +290,11 @@ public String getSourceCuebot() {
     public int getPendingEventCount() {
         return getQueue().size();
     }
+
+    /**
+     * Sets the Prometheus metrics collector for recording monitoring metrics.
+     */
+    public void setPrometheusMetrics(PrometheusMetricsCollector prometheusMetrics) {
+        this.prometheusMetrics = prometheusMetrics;
+    }
 }

cuebot/src/main/java/com/imageworks/spcue/service/JobManagerSupport.java

@@ -49,6 +49,8 @@
 import com.imageworks.spcue.rqd.RqdClient;
 import com.imageworks.spcue.util.CueExceptionUtil;
 import com.imageworks.spcue.util.FrameSet;
+import com.imageworks.spcue.PrometheusMetricsCollector;
+import com.imageworks.spcue.dao.ShowDao;
 
 /**
  * A non-transaction support class for managing jobs.
@@ -66,6 +68,8 @@ public class JobManagerSupport {
     private RedirectManager redirectManager;
     private EmailSupport emailSupport;
     private FrameSearchFactory frameSearchFactory;
+    private PrometheusMetricsCollector prometheusMetrics;
+    private ShowDao showDao;
 
     public void queueShutdownJob(JobInterface job, Source source, boolean isManualKill) {
         manageQueue.execute(new DispatchJobComplete(job, source, isManualKill, this));
@@ -150,6 +154,17 @@ public boolean shutdownJob(JobInterface job, Source source, boolean isManualKill
                  */
                 emailSupport.sendShutdownEmail(job);
 
+                // Record job completion metric
+                if (prometheusMetrics != null && showDao != null) {
+                    try {
+                        String showName = showDao.getShowDetail(job.getShowId()).getName();
+                        String state = isManualKill ? "KILLED" : "FINISHED";
+                        prometheusMetrics.recordJobCompleted(state, showName);
+                    } catch (Exception e) {
+                        logger.warn("Failed to record job completion metric: " + e.getMessage());
+                    }
+                }
+
                 return true;
             }
         }
@@ -593,4 +608,20 @@ public FrameSearchFactory getFrameSearchFactory() {
     public void setFrameSearchFactory(FrameSearchFactory frameSearchFactory) {
         this.frameSearchFactory = frameSearchFactory;
     }
+
+    public PrometheusMetricsCollector getPrometheusMetrics() {
+        return prometheusMetrics;
+    }
+
+    public void setPrometheusMetrics(PrometheusMetricsCollector prometheusMetrics) {
+        this.prometheusMetrics = prometheusMetrics;
+    }
+
+    public ShowDao getShowDao() {
+        return showDao;
+    }
+
+    public void setShowDao(ShowDao showDao) {
+        this.showDao = showDao;
+    }
 }

cuebot/src/main/resources/conf/spring/applicationContext-monitoring.xml

@@ -32,7 +32,9 @@
   <bean id="kafkaEventPublisher"
         class="com.imageworks.spcue.monitoring.KafkaEventPublisher"
         init-method="initialize"
-        destroy-method="shutdown"/>
+        destroy-method="shutdown">
+    <property name="prometheusMetrics" ref="prometheusMetricsCollector"/>
+  </bean>
 
   <!--
     Monitoring Event Builder helper for creating event messages from domain objects.

cuebot/src/main/resources/conf/spring/applicationContext-service.xml

@@ -248,6 +248,8 @@
     <property name="redirectManager" ref="redirectManager" />
     <property name="emailSupport" ref="emailSupport"/>
     <property name="frameSearchFactory" ref="frameSearchFactory" />
+    <property name="prometheusMetrics" ref="prometheusMetricsCollector" />
+    <property name="showDao" ref="showDao" />
   </bean>
 
   <bean id="jobLauncher" class="com.imageworks.spcue.service.JobLauncher">
@@ -380,6 +382,7 @@
     <property name="serviceDao" ref="serviceDao" />
     <property name="showDao" ref="showDao" />
     <property name="kafkaEventPublisher" ref="kafkaEventPublisher" />
+    <property name="prometheusMetrics" ref="prometheusMetricsCollector" />
   </bean>
 
   <bean id="hostReportHandler" class="com.imageworks.spcue.dispatcher.HostReportHandler" destroy-method="shutdown">

docs/_docs/concepts/render-farm-monitoring.md

@@ -21,6 +21,8 @@ OpenCue provides a comprehensive monitoring system for tracking render farm oper
 
 The monitoring system is built on an event-driven architecture that captures lifecycle events from jobs, layers, frames, hosts, and processes. These events can be:
 
+![OpenCue Monitoring Grafana Dashboard](/assets/images/opencue_monitoring/opencue_monitoring_grafana_chart.png)
+
 - **Published to Kafka** for real-time streaming and integration with external systems
 - **Stored in Elasticsearch** for historical analysis and querying
 - **Exposed as Prometheus metrics** for real-time dashboards and alerting
@@ -44,6 +46,8 @@ Cuebot publishes events to Apache Kafka topics when significant state changes oc
 
 Events are published asynchronously to avoid impacting render farm performance. A bounded queue ensures the system remains responsive even under high load.
 
+![UI for Apache Kafka](/assets/images/opencue_monitoring/opencue_monitoring_ui_for_apache_kafka.png)
+
 ### Historical storage (Elasticsearch)
 
 The Kafka event consumer indexes events into Elasticsearch for long-term storage and analysis. This enables:
@@ -55,10 +59,14 @@ The Kafka event consumer indexes events into Elasticsearch for long-term storage
 
 Elasticsearch indices are organized by event type and time-based partitioning for efficient querying.
 
+![Elasticsearch](/assets/images/opencue_monitoring/opencue_monitoring_elasticsearch.png)
+
 ### Metrics collection (Prometheus)
 
 Cuebot exposes a `/metrics` endpoint compatible with Prometheus. Key metrics include:
 
+![Prometheus Metrics Interface](/assets/images/opencue_monitoring/opencue_monitoring_prometheus.png)
+
 **Job and frame metrics:**
 - `cue_frames_completed_total` - Counter of completed frames by state
 - `cue_jobs_completed_total` - Counter of completed jobs by show

docs/_docs/developer-guide/monitoring-development.md

@@ -181,7 +181,7 @@ public static void setActiveJobs(String show, String state, int count) {
 
 ### Index templates
 
-Create custom index templates for new event types:
+Create custom index templates for new event types. Note that events use snake_case field names and include a `header` object:
 
 ```json
 {
@@ -192,13 +192,20 @@ Create custom index templates for new event types:
   },
   "mappings": {
     "properties": {
-      "eventType": { "type": "keyword" },
-      "timestamp": { "type": "date" },
-      "jobId": { "type": "keyword" },
-      "jobName": { "type": "keyword" },
-      "showName": { "type": "keyword" },
-      "oldPriority": { "type": "integer" },
-      "newPriority": { "type": "integer" },
+      "header": {
+        "properties": {
+          "event_id": { "type": "keyword" },
+          "event_type": { "type": "keyword" },
+          "timestamp": { "type": "date", "format": "epoch_millis" },
+          "source_cuebot": { "type": "keyword" },
+          "correlation_id": { "type": "keyword" }
+        }
+      },
+      "job_id": { "type": "keyword" },
+      "job_name": { "type": "keyword" },
+      "show": { "type": "keyword" },
+      "old_priority": { "type": "integer" },
+      "new_priority": { "type": "integer" },
       "user": { "type": "keyword" }
     }
   }