feat: Reduce memory usage threshold limits (#20)

Added threshold limits to control reduce memory usage: - reduceByteThresholdPerState - reduceByteThresholdAllStates ref: LOG-19194
mezmo · Mar 14, 2024 · 36c335c · 36c335c
1 parent 5a84af3
commit 36c335c
Show file tree

Hide file tree

Showing 4 changed files with 30 additions and 21 deletions.
diff --git a/charts/edge/Chart.yaml b/charts/edge/Chart.yaml
@@ -5,7 +5,7 @@ icon: https://app.mezmo.com/assets/img/mz-logo-square-128.png
 description: A Helm chart for deploying Mezmo Edge
 type: application
 appVersion: "3.7.3"
-version: 1.0.0
+version: 1.0.1
 maintainers:
   - name: Mezmo
     email: help@mezmo.com
diff --git a/charts/edge/README.md b/charts/edge/README.md
@@ -29,20 +29,22 @@ helm install edge mezmo/edge \
 
 ### Values
 
-| **Key**                   | **Type** | **Default**     | **Description**
-| ------------------------- | -------- | --------------- | ----------------------------------------------------------------------
-| mezmoApiAccessToken       | string   |                 | Your Mezmo API access token
-| mezmoDeploymentGroup      | string   |                 | Deployment group to apply to this Edge instance. Leaving undefined pulls all Edge pipelines for the org
-| mezmoApiScheme            | string   | "https"         | The scheme to use for the Mezmo API URL
-| mezmoApiHost              | string   | "api.mezmo.com" | The hostname(:port) of the Mezmo API
-| logLevel                  | string   | info            | Controls the logging verbosity of the deployment
-| autoscaling.enabled       | boolean  | false           | Whether or not to enable a HorizontalPodAutoscaler for this deployment
-| service.sourcePorts.start | int      | 8000            | The start of the port range (inclusive [start, end])
-| service.sourcePorts.end   | int      | 8010            | The end of the port range (set 0 or "" to disable port range)
-| service.sourcePorts.list  | array[]  | []              | Optional list of discrete ports to configure on the service
-| enableK8sLogs             | boolean  | false           | Whether or not to add ClusterRole and Volume access required for k8s logs source
-| pvcName                   | string   | ""              | Optional name for an existing PersistentVolumeClaim to use for disk buffering, aggregate persistence and k8s log checkpoints. Leaving undefined will default to ephemeral storage lasting the lifetime of the pod
-| maxDiskBufferBytes        | int      |                 | Optional max number of bytes to store in a disk-backed buffer. Leaving undefined will use default value of 268435488 (256MiB)
-| maxMemBufferEvents        | int      |                 | Optional max number of events to store in a memory buffer. Leaving undefined will use default value of 1000
+| **Key**                      | **Type** | **Default**      | **Description**
+| ---------------------------- | -------- | ---------------- | ----------------------------------------------------------------------
+| mezmoApiAccessToken          | string   |                  | Your Mezmo API access token
+| mezmoDeploymentGroup         | string   |                  | Deployment group to apply to this Edge instance. Leaving undefined pulls all Edge pipelines for the org
+| mezmoApiScheme               | string   | "https"          | The scheme to use for the Mezmo API URL
+| mezmoApiHost                 | string   | "api.mezmo.com"  | The hostname(:port) of the Mezmo API
+| logLevel                     | string   | info             | Controls the logging verbosity of the deployment
+| autoscaling.enabled          | boolean  | false            | Whether or not to enable a HorizontalPodAutoscaler for this deployment
+| service.sourcePorts.start    | int      | 8000             | The start of the port range (inclusive [start, end])
+| service.sourcePorts.end      | int      | 8010             | The end of the port range (set 0 or "" to disable port range)
+| service.sourcePorts.list     | array[]  | []               | Optional list of discrete ports to configure on the service
+| enableK8sLogs                | boolean  | false            | Whether or not to add ClusterRole and Volume access required for k8s logs source
+| pvcName                      | string   | ""               | Optional name for an existing PersistentVolumeClaim to use for disk buffering, aggregate persistence and k8s log checkpoints. Leaving undefined will default to ephemeral storage lasting the lifetime of the pod
+| maxDiskBufferBytes           | int      |268435488 (256MiB)| Optional max number of bytes to store in a disk-backed buffer.
+| maxMemBufferEvents           | int      | 1000             | Optional max number of events to store in a memory buffer.
+| reduceByteThresholdPerState  | int      | 104857600 (100Mb)| Optional max number of bytes to store in a memory for any given reduce component.
+| reduceByteThresholdAllStates | int      | 1073741824 (1Gb) | Optional max number of bytes to store in a memory for all given reduce components.
 
 \* Port 8686 is reserved for internal use, and should not be additionally configured for sourcePorts
diff --git a/charts/edge/templates/statefulset.yaml b/charts/edge/templates/statefulset.yaml
@@ -58,9 +58,9 @@ spec:
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           env:
           - name: MEZMO_API_HOST
-            value: {{ .Values.mezmoApiHost }}
+            value: "{{ .Values.mezmoApiHost }}"
           - name: MEZMO_API_SCHEME
-            value: {{ .Values.mezmoApiScheme }}
+            value: "{{ .Values.mezmoApiScheme }}"
           - name: MEZMO_LOCAL_DEPLOY_AUTH_TOKEN
             valueFrom:
               secretKeyRef:
@@ -75,7 +75,7 @@ spec:
           - name: MEZMO_EDGE_ID
             value: "{{include "edge.Id" . }}"
           - name: VECTOR_LOG
-            value: {{ .Values.logLevel }}
+            value: "{{ .Values.logLevel }}"
           - name: POD_NAME
             valueFrom:
               fieldRef:
@@ -94,12 +94,16 @@ spec:
             value: "1"
           {{- if .Values.maxDiskBufferBytes }}
           - name: MEZMO_DISK_BUFFER_MAX_BYTES
-            value: {{ .Values.maxDiskBufferBytes }}
+            value: "{{ .Values.maxDiskBufferBytes }}"
           {{- end }}
           {{- if .Values.maxMemBufferEvents }}
           - name: MEZMO_MEM_BUFFER_MAX_EVENTS
-            value: {{ .Values.maxMemBufferEvents }}
+            value: "{{ .Values.maxMemBufferEvents }}"
           {{- end }}
+          - name: REDUCE_BYTE_THRESHOLD_PER_STATE
+            value: "{{ .Values.reduceByteThresholdPerState }}"
+          - name: REDUCE_BYTE_THRESHOLD_ALL_STATES
+            value: "{{ .Values.reduceByteThresholdAllStates }}"
           args:
             - --config-dir
             - /etc/vector/

diff --git a/charts/edge/values.yaml b/charts/edge/values.yaml
@@ -78,3 +78,6 @@ pvcName: ""
 
 maxDiskBufferBytes: 0
 maxMemBufferEvents: 0
+
+reduceByteThresholdPerState: 104857600
+reduceByteThresholdAllStates: 1073741824