input: add new histogram metric for size of records

Author: nuclearpidgeon

include/fluent-bit/flb_input.h

@@ -51,6 +51,7 @@
 #include <fluent-bit/flb_pthread.h>
 
 #include <cmetrics/cmetrics.h>
+#include <cmetrics/cmt_histogram.h>
 #include <monkey/mk_core.h>
 #include <msgpack.h>
 #include <inttypes.h>
@@ -392,9 +393,10 @@ struct flb_input_instance {
      *
      * All metrics available for an input plugin instance.
      */
-    struct cmt *cmt;                     /* parent context              */
-    struct cmt_counter *cmt_bytes;       /* metric: input_bytes_total   */
-    struct cmt_counter *cmt_records;     /* metric: input_records_total */
+    struct cmt *cmt;                        /* parent context              */
+    struct cmt_counter *cmt_bytes;          /* metric: input_bytes_total   */
+    struct cmt_histogram *cmt_record_sizes; /* metric: input_record_sizes  */
+    struct cmt_counter *cmt_records;        /* metric: input_records_total */
 
     /* is the input instance overlimit ?: 1 or 0 */
     struct cmt_gauge   *cmt_storage_overlimit;

src/flb_input.c

@@ -1090,6 +1090,17 @@ int flb_input_instance_init(struct flb_input_instance *ins,
                            "Number of input bytes.",
                            1, (char *[]) {"name"});
     cmt_counter_set(ins->cmt_bytes, ts, 0, 1, (char *[]) {name});
+    struct cmt_histogram_buckets *input_record_buckets = \
+        cmt_histogram_buckets_create_size((double[]){ 100, 1024, 2048, 4096,
+                                                      100 * 1024, 1024 * 1024, 4 * 1024 * 1024,
+                                                      10 * 1024 * 1024}, 8);
+    /* fluentbit_input_record_sizes */
+    ins->cmt_record_sizes = \
+        cmt_histogram_create(ins->cmt,
+                             "fluentbit", "input", "record_sizes",
+                             "Histogram of the size of input records",
+                             input_record_buckets,
+                             1, (char *[]) {"name"});
 
     /* fluentbit_input_records_total */
     ins->cmt_records = \

src/flb_input_chunk.c

@@ -38,6 +38,7 @@
 #include <fluent-bit/flb_ring_buffer.h>
 #include <chunkio/chunkio.h>
 #include <monkey/mk_core.h>
+#include <cmetrics/cmt_histogram.h>
 
 
 #ifdef FLB_HAVE_CHUNK_TRACE
@@ -1672,7 +1673,8 @@ static int input_chunk_append_raw(struct flb_input_instance *in,
         /* fluentbit_input_bytes_total */
         cmt_counter_add(in->cmt_bytes, ts, buf_size,
                         1, (char *[]) {(char *) flb_input_name(in)});
-
+        cmt_histogram_observe(in->cmt_record_sizes, ts, buf_size,
+                              1, (char *[]){(char *) flb_input_name(in)});
         /* OLD api */
         flb_metrics_sum(FLB_METRIC_N_RECORDS, ic->added_records, in->metrics);
         flb_metrics_sum(FLB_METRIC_N_BYTES, buf_size, in->metrics);