WIP: Buffer tuning for https://issues.redhat.com/browse/RFE-6849

This PR has the proposed API changes needed for RFE-6849.
The functionality is not implemented.

Author: alanconway

api/observability/v1/output_types.go

@@ -210,15 +210,41 @@ type BaseOutputTuningSpec struct {
 
 	// MinRetryDuration is the minimum time to wait between attempts to retry after delivery a failure.
 	//
+	// A duration string is a sequence of decimal numbers and unit suffixes,
+	// such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+	//
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Minimum Retry Duration"
 	MinRetryDuration *time.Duration `json:"minRetryDuration,omitempty"`
 
 	// MaxRetryDuration is the maximum time to wait between retry attempts after a delivery failure.
 	//
+	// A duration string is a sequence of decimal numbers and unit suffixes,
+	// such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+	//
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Maximum Retry Duration"
 	MaxRetryDuration *time.Duration `json:"maxRetryDuration,omitempty"`
+
+	// MaxBufferSize is the maximum size in bytes allowed for buffering unsent data to an output.
+	// It is written as a [Kubernetes quantity], for example: 128974848, 129e6, 129M,  123Mi
+	//
+	// NOTE: It is not recommended to set this field unless you understand the disk space
+	// requirements, and have done tests to validate performance under realistic conditions.
+	//
+	// The following are potential risks to bear in mind:
+	//
+	// - With `deliveryMode: AtLeastOnce`, buffers are created on the local node `/var` file system.
+	//   Filling up that file system can have severe consequences for the entire node.
+	//   The disk required on each nodes is the sum of all persistent output buffers, on all deployed ClusterLogForwarder instances.
+	//
+	// - Larger buffers can increase latency and recovery times from a period of overload.
+	//   If buffers fill up under heavy load, it takes longer for logs to get to the head of the buffer to be delivered.
+	//   If logs are not sent much faster than they are collected, the buffers may take a long time to drain.
+	//
+	// [Kubernetes quanitity]: https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity
+	// [Kubernetes resource limits]: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory
+	MaxBufferSize *resource.Quantity `json:"maxBufferSize,omitempty"`
 }
 
 // DeliveryMode sets the delivery mode for log forwarding.
@@ -650,6 +676,13 @@ type KafkaTuningSpec struct {
 	// +kubebuilder:validation:Enum:=none;snappy;zstd;lz4
 	// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Compression"
 	Compression string `json:"compression,omitempty"`
+
+	// MessageTimeout messages that are waiting to be sent are dropped after this timeout.
+	// Used to ensure that newer messages are not blocked indefinitely by older ones.
+	//
+	// A duration string is a sequence of decimal numbers and a unit suffix,
+	// such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+	MessageTimeout *time.Duration `json:"messageTimeout,omitempty"`
 }
 
 // KafkaAuthentication contains configuration for authenticating requests to a Kafka output.

api/observability/v1/zz_generated.deepcopy.go

@@ -82,7 +82,7 @@ metadata:
     categories: OpenShift Optional, Logging & Tracing
     certified: "false"
     containerImage: quay.io/openshift-logging/cluster-logging-operator:latest
-    createdAt: "2025-02-06T14:39:16Z"
+    createdAt: "2025-03-05T17:12:05Z"
     description: The Red Hat OpenShift Logging Operator for OCP provides a means for
       configuring and managing log collection and forwarding.
     features.operators.openshift.io/cnf: "false"
@@ -467,16 +467,24 @@ spec:
         path: outputs[0].azureMonitor.tuning
       - displayName: Delivery Mode
         path: outputs[0].azureMonitor.tuning.deliveryMode
-      - description: MaxRetryDuration is the maximum time to wait between retry attempts
-          after a delivery failure.
+      - description: |-
+          MaxRetryDuration is the maximum time to wait between retry attempts after a delivery failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Maximum Retry Duration
         path: outputs[0].azureMonitor.tuning.maxRetryDuration
       - description: MaxWrite limits the maximum payload in terms of bytes of a single
           "send" to the output.
         displayName: Batch Size
         path: outputs[0].azureMonitor.tuning.maxWrite
-      - description: MinRetryDuration is the minimum time to wait between attempts
-          to retry after delivery a failure.
+      - description: |-
+          MinRetryDuration is the minimum time to wait between attempts to retry after delivery a failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Minimum Retry Duration
         path: outputs[0].azureMonitor.tuning.minRetryDuration
       - description: Cloudwatch configures forwarding log events to AWS Cloudwatch
@@ -606,16 +614,24 @@ spec:
         path: outputs[0].cloudwatch.tuning.compression
       - displayName: Delivery Mode
         path: outputs[0].cloudwatch.tuning.deliveryMode
-      - description: MaxRetryDuration is the maximum time to wait between retry attempts
-          after a delivery failure.
+      - description: |-
+          MaxRetryDuration is the maximum time to wait between retry attempts after a delivery failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Maximum Retry Duration
         path: outputs[0].cloudwatch.tuning.maxRetryDuration
       - description: MaxWrite limits the maximum payload in terms of bytes of a single
           "send" to the output.
         displayName: Batch Size
         path: outputs[0].cloudwatch.tuning.maxWrite
-      - description: MinRetryDuration is the minimum time to wait between attempts
-          to retry after delivery a failure.
+      - description: |-
+          MinRetryDuration is the minimum time to wait between attempts to retry after delivery a failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Minimum Retry Duration
         path: outputs[0].cloudwatch.tuning.minRetryDuration
       - description: |-
@@ -724,16 +740,24 @@ spec:
         path: outputs[0].elasticsearch.tuning.compression
       - displayName: Delivery Mode
         path: outputs[0].elasticsearch.tuning.deliveryMode
-      - description: MaxRetryDuration is the maximum time to wait between retry attempts
-          after a delivery failure.
+      - description: |-
+          MaxRetryDuration is the maximum time to wait between retry attempts after a delivery failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Maximum Retry Duration
         path: outputs[0].elasticsearch.tuning.maxRetryDuration
       - description: MaxWrite limits the maximum payload in terms of bytes of a single
           "send" to the output.
         displayName: Batch Size
         path: outputs[0].elasticsearch.tuning.maxWrite
-      - description: MinRetryDuration is the minimum time to wait between attempts
-          to retry after delivery a failure.
+      - description: |-
+          MinRetryDuration is the minimum time to wait between attempts to retry after delivery a failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Minimum Retry Duration
         path: outputs[0].elasticsearch.tuning.minRetryDuration
       - description: |-
@@ -815,16 +839,24 @@ spec:
         path: outputs[0].googleCloudLogging.tuning
       - displayName: Delivery Mode
         path: outputs[0].googleCloudLogging.tuning.deliveryMode
-      - description: MaxRetryDuration is the maximum time to wait between retry attempts
-          after a delivery failure.
+      - description: |-
+          MaxRetryDuration is the maximum time to wait between retry attempts after a delivery failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Maximum Retry Duration
         path: outputs[0].googleCloudLogging.tuning.maxRetryDuration
       - description: MaxWrite limits the maximum payload in terms of bytes of a single
           "send" to the output.
         displayName: Batch Size
         path: outputs[0].googleCloudLogging.tuning.maxWrite
-      - description: MinRetryDuration is the minimum time to wait between attempts
-          to retry after delivery a failure.
+      - description: |-
+          MinRetryDuration is the minimum time to wait between attempts to retry after delivery a failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Minimum Retry Duration
         path: outputs[0].googleCloudLogging.tuning.minRetryDuration
       - description: HTTP configures forwarding log events to an HTTP server
@@ -904,16 +936,24 @@ spec:
         path: outputs[0].http.tuning.compression
       - displayName: Delivery Mode
         path: outputs[0].http.tuning.deliveryMode
-      - description: MaxRetryDuration is the maximum time to wait between retry attempts
-          after a delivery failure.
+      - description: |-
+          MaxRetryDuration is the maximum time to wait between retry attempts after a delivery failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Maximum Retry Duration
         path: outputs[0].http.tuning.maxRetryDuration
       - description: MaxWrite limits the maximum payload in terms of bytes of a single
           "send" to the output.
         displayName: Batch Size
         path: outputs[0].http.tuning.maxWrite
-      - description: MinRetryDuration is the minimum time to wait between attempts
-          to retry after delivery a failure.
+      - description: |-
+          MinRetryDuration is the minimum time to wait between attempts to retry after delivery a failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Minimum Retry Duration
         path: outputs[0].http.tuning.minRetryDuration
       - description: |-
@@ -1166,16 +1206,24 @@ spec:
         path: outputs[0].loki.tuning.compression
       - displayName: Delivery Mode
         path: outputs[0].loki.tuning.deliveryMode
-      - description: MaxRetryDuration is the maximum time to wait between retry attempts
-          after a delivery failure.
+      - description: |-
+          MaxRetryDuration is the maximum time to wait between retry attempts after a delivery failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Maximum Retry Duration
         path: outputs[0].loki.tuning.maxRetryDuration
       - description: MaxWrite limits the maximum payload in terms of bytes of a single
           "send" to the output.
         displayName: Batch Size
         path: outputs[0].loki.tuning.maxWrite
-      - description: MinRetryDuration is the minimum time to wait between attempts
-          to retry after delivery a failure.
+      - description: |-
+          MinRetryDuration is the minimum time to wait between attempts to retry after delivery a failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Minimum Retry Duration
         path: outputs[0].loki.tuning.minRetryDuration
       - description: |-
@@ -1360,16 +1408,24 @@ spec:
         path: outputs[0].lokiStack.tuning.compression
       - displayName: Delivery Mode
         path: outputs[0].lokiStack.tuning.deliveryMode
-      - description: MaxRetryDuration is the maximum time to wait between retry attempts
-          after a delivery failure.
+      - description: |-
+          MaxRetryDuration is the maximum time to wait between retry attempts after a delivery failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Maximum Retry Duration
         path: outputs[0].lokiStack.tuning.maxRetryDuration
       - description: MaxWrite limits the maximum payload in terms of bytes of a single
           "send" to the output.
         displayName: Batch Size
         path: outputs[0].lokiStack.tuning.maxWrite
-      - description: MinRetryDuration is the minimum time to wait between attempts
-          to retry after delivery a failure.
+      - description: |-
+          MinRetryDuration is the minimum time to wait between attempts to retry after delivery a failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Minimum Retry Duration
         path: outputs[0].lokiStack.tuning.minRetryDuration
       - description: Name used to refer to the output from a `pipeline`.
@@ -1444,16 +1500,24 @@ spec:
         - urn:alm:descriptor:com.tectonic.ui:text
       - displayName: Delivery Mode
         path: outputs[0].otlp.tuning.deliveryMode
-      - description: MaxRetryDuration is the maximum time to wait between retry attempts
-          after a delivery failure.
+      - description: |-
+          MaxRetryDuration is the maximum time to wait between retry attempts after a delivery failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Maximum Retry Duration
         path: outputs[0].otlp.tuning.maxRetryDuration
       - description: MaxWrite limits the maximum payload in terms of bytes of a single
           "send" to the output.
         displayName: Batch Size
         path: outputs[0].otlp.tuning.maxWrite
-      - description: MinRetryDuration is the minimum time to wait between attempts
-          to retry after delivery a failure.
+      - description: |-
+          MinRetryDuration is the minimum time to wait between attempts to retry after delivery a failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Minimum Retry Duration
         path: outputs[0].otlp.tuning.minRetryDuration
       - description: |-
@@ -1541,16 +1605,24 @@ spec:
         path: outputs[0].splunk.tuning.compression
       - displayName: Delivery Mode
         path: outputs[0].splunk.tuning.deliveryMode
-      - description: MaxRetryDuration is the maximum time to wait between retry attempts
-          after a delivery failure.
+      - description: |-
+          MaxRetryDuration is the maximum time to wait between retry attempts after a delivery failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Maximum Retry Duration
         path: outputs[0].splunk.tuning.maxRetryDuration
       - description: MaxWrite limits the maximum payload in terms of bytes of a single
           "send" to the output.
         displayName: Batch Size
         path: outputs[0].splunk.tuning.maxWrite
-      - description: MinRetryDuration is the minimum time to wait between attempts
-          to retry after delivery a failure.
+      - description: |-
+          MinRetryDuration is the minimum time to wait between attempts to retry after delivery a failure.
+
+
+          A duration string is a sequence of decimal numbers and unit suffixes,
+          such as "300ms" or "2h45m". Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
         displayName: Minimum Retry Duration
         path: outputs[0].splunk.tuning.minRetryDuration
       - description: |-