K8SPXC-1621 [POC] Implement Fluent-bit buffer settings

config/crd/bases/pxc.percona.com_perconaxtradbclusters.yaml

@@ -4249,6 +4249,15 @@ spec:
                     type: object
                   enabled:
                     type: boolean
+                  fluentBitBufferSettings:
+                    properties:
+                      bufferChunkSize:
+                        type: string
+                      bufferMaxSize:
+                        type: string
+                      memBufLimit:
+                        type: string
+                    type: object
                   hookScript:
                     type: string
                   image:

deploy/bundle.yaml

@@ -5216,6 +5216,15 @@ spec:
                     type: object
                   enabled:
                     type: boolean
+                  fluentBitBufferSettings:
+                    properties:
+                      bufferChunkSize:
+                        type: string
+                      bufferMaxSize:
+                        type: string
+                      memBufLimit:
+                        type: string
+                    type: object
                   hookScript:
                     type: string
                   image:

deploy/cr.yaml

@@ -546,12 +546,16 @@ spec:
     image: perconalab/percona-xtradb-cluster-operator:main-logcollector
 #    configuration: |
 #      [OUTPUT]
-#           Name  es
-#           Match *
-#           Host  192.168.2.3
-#           Port  9200
-#           Index my_index
-#           Type  my_type
+#          Name  es
+#          Match *
+#          Host  192.168.2.3
+#          Port  9200
+#          Index my_index
+#          Type  my_type
+#    fluentBitBufferSettings:
+#      bufferChunkSize: "120k"
+#      bufferMaxSize: "512k"
+#      memBufLimit: "20MB"
     resources:
       requests:
         memory: 100M

deploy/crd.yaml

@@ -5216,6 +5216,15 @@ spec:
                     type: object
                   enabled:
                     type: boolean
+                  fluentBitBufferSettings:
+                    properties:
+                      bufferChunkSize:
+                        type: string
+                      bufferMaxSize:
+                        type: string
+                      memBufLimit:
+                        type: string
+                    type: object
                   hookScript:
                     type: string
                   image:

deploy/cw-bundle.yaml

@@ -5216,6 +5216,15 @@ spec:
                     type: object
                   enabled:
                     type: boolean
+                  fluentBitBufferSettings:
+                    properties:
+                      bufferChunkSize:
+                        type: string
+                      bufferMaxSize:
+                        type: string
+                      memBufLimit:
+                        type: string
+                    type: object
                   hookScript:
                     type: string
                   image:

pkg/apis/pxc/v1/pxc_types.go

@@ -636,6 +636,21 @@ type LogCollectorSpec struct {
 	ImagePullPolicy          corev1.PullPolicy           `json:"imagePullPolicy,omitempty"`
 	RuntimeClassName         *string                     `json:"runtimeClassName,omitempty"`
 	HookScript               string                      `json:"hookScript,omitempty"`
+	// FluentBitBufferSettings allows configuring Fluent-bit buffer sizes to handle long log lines
+	FluentBitBufferSettings *FluentBitBufferSettings `json:"fluentBitBufferSettings,omitempty"`
+}
+
+// FluentBitBufferSettings defines buffer size settings for Fluent-bit to handle long log lines
+type FluentBitBufferSettings struct {
+	// BufferChunkSize sets the initial buffer size for reading file data (default: 64k)
+	// This should be large enough to handle the longest expected log line
+	BufferChunkSize string `json:"bufferChunkSize,omitempty"`
+	// BufferMaxSize sets the maximum buffer size per monitored file (default: 256k)
+	// This should be greater than or equal to BufferChunkSize
+	BufferMaxSize string `json:"bufferMaxSize,omitempty"`
+	// MemBufLimit sets the memory buffer limit for the input plugin (default: 10MB)
+	// This helps prevent memory issues when processing large amounts of log data
+	MemBufLimit string `json:"memBufLimit,omitempty"`
 }
 
 type PMMSpec struct {
@@ -1052,6 +1067,59 @@ func (cr *PerconaXtraDBCluster) CheckNSetDefaults(serverVersion *version.ServerV
 		if len(c.LogCollector.ImagePullPolicy) == 0 {
 			c.LogCollector.ImagePullPolicy = corev1.PullAlways
 		}
+
+		// Set default Fluent-bit buffer settings to handle long log lines
+		if c.LogCollector.FluentBitBufferSettings == nil {
+			c.LogCollector.FluentBitBufferSettings = &FluentBitBufferSettings{}
+		}
+
+		// Use enhanced defaults for version >= 1.19.0, fallback to basic defaults for older versions
+		if cr.CompareVersionWith("1.19.0") >= 0 {
+			// Enhanced defaults for newer versions to better handle long log lines
+			if c.LogCollector.FluentBitBufferSettings.BufferChunkSize == "" {
+				c.LogCollector.FluentBitBufferSettings.BufferChunkSize = "128k"
+			}
+
+			if c.LogCollector.FluentBitBufferSettings.BufferMaxSize == "" {
+				c.LogCollector.FluentBitBufferSettings.BufferMaxSize = "512k"
+			}
+
+			if c.LogCollector.FluentBitBufferSettings.MemBufLimit == "" {
+				c.LogCollector.FluentBitBufferSettings.MemBufLimit = "20MB"
+			}
+		} else {
+			// Basic defaults for older versions
+			if c.LogCollector.FluentBitBufferSettings.BufferChunkSize == "" {
+				c.LogCollector.FluentBitBufferSettings.BufferChunkSize = "64k"
+			}
+
+			if c.LogCollector.FluentBitBufferSettings.BufferMaxSize == "" {
+				c.LogCollector.FluentBitBufferSettings.BufferMaxSize = "256k"
+			}
+
+			if c.LogCollector.FluentBitBufferSettings.MemBufLimit == "" {
+				c.LogCollector.FluentBitBufferSettings.MemBufLimit = "10MB"
+			}
+		}
+
+		// Validate that Buffer_Max_Size >= Buffer_Chunk_Size (Fluent-bit requirement)
+		// If user provided invalid values, adjust Buffer_Max_Size to be at least as large as Buffer_Chunk_Size
+		if c.LogCollector.FluentBitBufferSettings.BufferChunkSize != "" && c.LogCollector.FluentBitBufferSettings.BufferMaxSize != "" {
+			chunkSize := c.LogCollector.FluentBitBufferSettings.BufferChunkSize
+			maxSize := c.LogCollector.FluentBitBufferSettings.BufferMaxSize
+
+			// Simple validation: if both are the same format (e.g., "128k"), compare them
+			if len(chunkSize) > 0 && len(maxSize) > 0 && chunkSize[len(chunkSize)-1] == maxSize[len(maxSize)-1] {
+				// Extract numeric values and compare
+				chunkNum := chunkSize[:len(chunkSize)-1]
+				maxNum := maxSize[:len(maxSize)-1]
+
+				// If chunk size is larger than max size, set max size to chunk size
+				if chunkNum > maxNum {
+					c.LogCollector.FluentBitBufferSettings.BufferMaxSize = chunkSize
+				}
+			}
+		}
 	}
 
 	if c.HAProxyEnabled() {