Add support for gathering profiles and delivering them to Pyroscope (#…

…446)

Signed-off-by: Pete Wall <pete.wall@grafana.com>
Co-authored-by: Christian Simon <simon@swine.de>

.github/configs/grafana.yaml

@@ -27,3 +27,11 @@ datasources:
       - name: Tempo
         type: tempo
         url: http://tempo.tempo.svc:3100
+
+      - name: Pyroscope
+        type: grafana-pyroscope-datasource
+        url: http://pyroscope.pyroscope.svc.cluster.local.:4040
+        jsonData:
+          httpHeaderName1: X-Scope-OrgID
+        secureJsonData:
+          httpHeaderValue1: "1"

.github/configs/pyroscope.yaml

@@ -0,0 +1,5 @@
+agent:
+  enabled: false
+pyroscope:
+  extraArgs:
+    "auth.multitenancy-enabled": true

.github/configs/updatecli.d/grafana-agent.yaml

@@ -46,3 +46,12 @@ targets:
             name: charts/k8s-monitoring
             versionincrement: none
         sourceid: grafana-agent
+    grafana-agent-profiles:
+        name: Bump Helm chart dependency "grafana-agent-profiles" for Helm chart "k8s-monitoring"
+        kind: helmchart
+        spec:
+            file: Chart.yaml
+            key: $.dependencies[3].version
+            name: charts/k8s-monitoring
+            versionincrement: none
+        sourceid: grafana-agent

.github/configs/updatecli.d/kube-state-metrics.yaml

@@ -15,7 +15,7 @@ conditions:
         kind: yaml
         spec:
             file: charts/k8s-monitoring/Chart.yaml
-            key: $.dependencies[3].name
+            key: $.dependencies[4].name
             value: kube-state-metrics
         disablesourceinput: true
 targets:
@@ -24,7 +24,7 @@ targets:
         kind: helmchart
         spec:
             file: Chart.yaml
-            key: $.dependencies[3].version
+            key: $.dependencies[4].version
             name: charts/k8s-monitoring
             versionincrement: none
         sourceid: kube-state-metrics

.github/configs/updatecli.d/node-exporter.yaml

@@ -16,7 +16,7 @@ conditions:
         kind: yaml
         spec:
             file: charts/k8s-monitoring/Chart.yaml
-            key: $.dependencies[4].name
+            key: $.dependencies[5].name
             value: prometheus-node-exporter
         disablesourceinput: true
 
@@ -26,7 +26,7 @@ targets:
         kind: helmchart
         spec:
             file: Chart.yaml
-            key: $.dependencies[4].version
+            key: $.dependencies[5].version
             name: charts/k8s-monitoring
             versionincrement: none
         sourceid: prometheus-node-exporter

.github/configs/updatecli.d/opencost.yaml

@@ -15,7 +15,7 @@ conditions:
         kind: yaml
         spec:
             file: charts/k8s-monitoring/Chart.yaml
-            key: $.dependencies[7].name
+            key: $.dependencies[8].name
             value: opencost
         disablesourceinput: true
 targets:
@@ -24,7 +24,7 @@ targets:
         kind: helmchart
         spec:
             file: Chart.yaml
-            key: $.dependencies[7].version
+            key: $.dependencies[8].version
             name: charts/k8s-monitoring
             versionincrement: none
         sourceid: opencost

.github/configs/updatecli.d/prometheus-operator-crds.yaml

@@ -15,7 +15,7 @@ conditions:
         kind: yaml
         spec:
             file: charts/k8s-monitoring/Chart.yaml
-            key: $.dependencies[5].name
+            key: $.dependencies[6].name
             value: prometheus-operator-crds
         disablesourceinput: true
 targets:
@@ -24,7 +24,7 @@ targets:
         kind: helmchart
         spec:
             file: Chart.yaml
-            key: $.dependencies[5].version
+            key: $.dependencies[6].version
             name: charts/k8s-monitoring
             versionincrement: none
         sourceid: prometheus-operator-crds

.github/configs/updatecli.d/windows-exporter.yaml

@@ -15,7 +15,7 @@ conditions:
         kind: yaml
         spec:
             file: charts/k8s-monitoring/Chart.yaml
-            key: $.dependencies[6].name
+            key: $.dependencies[7].name
             value: prometheus-windows-exporter
         disablesourceinput: true
 targets:
@@ -24,7 +24,7 @@ targets:
         kind: helmchart
         spec:
             file: Chart.yaml
-            key: $.dependencies[6].version
+            key: $.dependencies[7].version
             name: charts/k8s-monitoring
             versionincrement: none
         sourceid: prometheus-windows-exporter

.github/workflows/helm-test.yml

@@ -19,6 +19,7 @@ env:
   CREDENTIALS: "${{ github.workspace }}/.github/configs/credentials.yaml"
   LOKI_VALUES: "${{ github.workspace }}/.github/configs/loki.yaml"
   TEMPO_VALUES: ""  # No values for now
+  PYROSCOPE_VALUES: "${{ github.workspace }}/.github/configs/pyroscope.yaml"
   GRAFANA_VALUES: "${{ github.workspace }}/.github/configs/grafana.yaml"
   PODLOGS_OBJECTS: "${{ github.workspace }}/.github/configs/podlogs.yaml"
   MYSQL_VALUES: "${{ github.workspace }}/.github/configs/mysql.yaml"
@@ -195,6 +196,11 @@ jobs:
       run: |
         helm install tempo grafana/tempo -n tempo --create-namespace --wait
 
+    - name: Deploy Pyroscope
+      if: steps.list-changed.outputs.changed == 'true'
+      run: |
+        helm install pyroscope grafana/pyroscope -f "${PYROSCOPE_VALUES}" -n pyroscope --create-namespace --wait
+
     - name: Deploy Grafana
       if: steps.list-changed.outputs.changed == 'true'
       run: |

Makefile

@@ -4,9 +4,10 @@ SHELL := /bin/bash
 CHART_FILES = $(shell find charts/k8s-monitoring -type f)
 INPUT_FILES = $(wildcard examples/*/values.yaml)
 OUTPUT_FILES = $(subst values.yaml,output.yaml,$(INPUT_FILES))
-METRIC_CONFIG_FILES = $(subst values.yaml,metrics.river,$(INPUT_FILES))
-EVENT_CONFIG_FILES = $(subst values.yaml,events.river,$(INPUT_FILES))
-LOG_CONFIG_FILES = $(subst values.yaml,logs.river,$(INPUT_FILES))
+METRICS_CONFIG_FILES = $(subst values.yaml,metrics.river,$(INPUT_FILES))
+EVENTS_CONFIG_FILES = $(subst values.yaml,events.river,$(INPUT_FILES))
+LOGS_CONFIG_FILES = $(subst values.yaml,logs.river,$(INPUT_FILES))
+PROFILES_CONFIG_FILES = $(subst values.yaml,profiles.river,$(INPUT_FILES))
 
 CT_CONFIGFILE ?= .github/configs/ct.yaml
 LINT_CONFIGFILE ?= .github/configs/lintconf.yaml
@@ -36,9 +37,12 @@ install-deps: scripts/install-deps.sh
 %/logs.river: %/output.yaml
 	yq -r "select(.metadata.name==\"k8smon-grafana-agent-logs\") | .data[\"config.river\"] | select( . != null )" $< > $@
 
+%/profiles.river: %/output.yaml
+	yq -r "select(.metadata.name==\"k8smon-grafana-agent-profiles\") | .data[\"config.river\"] | select( . != null )" $< > $@
+
 clean:
 	rm -f $(OUTPUT_FILES) $(METRIC_CONFIG_FILES) $(EVENT_CONFIG_FILES) $(LOG_CONFIG_FILES)
 
-generate-example-outputs: $(OUTPUT_FILES) $(METRIC_CONFIG_FILES) $(EVENT_CONFIG_FILES) $(LOG_CONFIG_FILES)
+generate-example-outputs: $(OUTPUT_FILES) $(METRICS_CONFIG_FILES) $(EVENTS_CONFIG_FILES) $(LOGS_CONFIG_FILES) $(PROFILES_CONFIG_FILES)
 
 regenerate-example-outputs: clean generate-example-outputs

charts/k8s-monitoring/Chart.lock

@@ -8,6 +8,9 @@ dependencies:
 - name: grafana-agent
   repository: https://grafana.github.io/helm-charts
   version: 0.37.0
+- name: grafana-agent
+  repository: https://grafana.github.io/helm-charts
+  version: 0.37.0
 - name: kube-state-metrics
   repository: https://prometheus-community.github.io/helm-charts
   version: 5.18.0
@@ -23,5 +26,5 @@ dependencies:
 - name: opencost
   repository: https://opencost.github.io/opencost-helm-chart
   version: 1.33.1
-digest: sha256:829140f8e5d474204785b623af1fca296f0a7532dd4ead8ac8ee95a29f790a70
-generated: "2024-04-03T00:16:32.523019666Z"
+digest: sha256:d940392f1aa38b1367363be91d62f5fcc64b402ac075a9bd82eeb01e72cd03d7
+generated: "2024-04-04T09:43:52.93471-05:00"

charts/k8s-monitoring/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v2
 name: k8s-monitoring
-description: A Helm chart for gathering, scraping, and forwarding Kubernetes infrastructure metrics and logs to a Grafana Stack.
+description: A Helm chart for gathering, scraping, and forwarding Kubernetes telemetry data to a Grafana Stack.
 type: application
-version: 0.12.5
+version: 0.13.0
 appVersion: 2.1.2
 icon: https://raw.githubusercontent.com/grafana/grafana/main/public/img/grafana_icon.svg
 sources:
@@ -26,6 +26,11 @@ dependencies:
   version: 0.37.0
   repository: https://grafana.github.io/helm-charts
   condition: logs.pod_logs.enabled
+- alias: grafana-agent-profiles
+  name: grafana-agent
+  version: 0.37.0
+  repository: https://grafana.github.io/helm-charts
+  condition: profiles.enabled
 - name: kube-state-metrics
   version: 5.18.0
   repository: https://prometheus-community.github.io/helm-charts

charts/k8s-monitoring/Makefile

@@ -9,6 +9,7 @@ values.schema.json: values.yaml schema-mods/enums-and-types.json schema-mods/req
 		<(jq --indent 4 'del(.properties["grafana-agent"]) \
 			| del(.properties["grafana-agent-events"]) \
 			| del(.properties["grafana-agent-logs"]) \
+			| del(.properties["grafana-agent-profiles"]) \
 			| del(.properties["kube-state-metrics"].properties.autosharding) \
 			| del(.properties["kube-state-metrics"].properties.nodeSelector) \
 			| del(.properties["kube-state-metrics"].properties.prometheusScrape) \

charts/k8s-monitoring/README.md

@@ -3,9 +3,9 @@
 
 # k8s-monitoring
 
-![Version: 0.12.5](https://img.shields.io/badge/Version-0.12.5-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.1.2](https://img.shields.io/badge/AppVersion-2.1.2-informational?style=flat-square)
+![Version: 0.13.0](https://img.shields.io/badge/Version-0.13.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 2.1.2](https://img.shields.io/badge/AppVersion-2.1.2-informational?style=flat-square)
 
-A Helm chart for gathering, scraping, and forwarding Kubernetes infrastructure metrics and logs to a Grafana Stack.
+A Helm chart for gathering, scraping, and forwarding Kubernetes telemetry data to a Grafana Stack.
 
 ## Breaking change announcements
 
@@ -104,6 +104,7 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely
 | https://grafana.github.io/helm-charts | grafana-agent | 0.37.0 |
 | https://grafana.github.io/helm-charts | grafana-agent-events(grafana-agent) | 0.37.0 |
 | https://grafana.github.io/helm-charts | grafana-agent-logs(grafana-agent) | 0.37.0 |
+| https://grafana.github.io/helm-charts | grafana-agent-profiles(grafana-agent) | 0.37.0 |
 | https://opencost.github.io/opencost-helm-chart | opencost | 1.33.1 |
 | https://prometheus-community.github.io/helm-charts | kube-state-metrics | 5.18.0 |
 | https://prometheus-community.github.io/helm-charts | prometheus-node-exporter | 4.32.0 |
@@ -184,6 +185,21 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely
 | externalServices.prometheus.wal.truncateFrequency | string | `"2h"` | How frequently to clean up the WAL. |
 | externalServices.prometheus.writeEndpoint | string | `"/api/prom/push"` | Prometheus metrics write endpoint. Preset for Grafana Cloud Metrics instances. |
 | externalServices.prometheus.writeRelabelConfigRules | string | `""` | Rule blocks to be added to the [write_relabel_config block](https://grafana.com/docs/agent/latest/flow/reference/components/prometheus.remote_write/#write_relabel_config-block) of the prometheus.remote_write component. |
+| externalServices.pyroscope.authMode | string | `"basic"` | one of "none", "basic" |
+| externalServices.pyroscope.basicAuth.password | string | `""` | Pyroscope basic auth password |
+| externalServices.pyroscope.basicAuth.passwordKey | string | `"password"` | The key for the password property in the secret |
+| externalServices.pyroscope.basicAuth.username | string | `""` | Pyroscope basic auth username |
+| externalServices.pyroscope.basicAuth.usernameKey | string | `"username"` | The key for the username property in the secret |
+| externalServices.pyroscope.externalLabels | object | `{}` | Custom labels to be added to all profiles |
+| externalServices.pyroscope.host | string | `""` | Pyroscope host where profiles will be sent |
+| externalServices.pyroscope.hostKey | string | `"host"` | The key for the host property in the secret |
+| externalServices.pyroscope.proxyURL | string | `""` | HTTP proxy to proxy requests to Pyroscope through. |
+| externalServices.pyroscope.secret.create | bool | `true` | Should this Helm chart create the secret. If false, you must define the name and namespace values. |
+| externalServices.pyroscope.secret.name | string | `""` | The name of the secret. |
+| externalServices.pyroscope.secret.namespace | string | `""` | The namespace of the secret. |
+| externalServices.pyroscope.tenantId | string | `""` | Pyroscope tenant ID |
+| externalServices.pyroscope.tenantIdKey | string | `"tenantId"` | The key for the tenant ID property in the secret |
+| externalServices.pyroscope.tls | object | `{}` | [TLS settings](https://grafana.com/docs/agent/latest/flow/reference/components/pyroscope.write/#tls_config-block) to configure for the profiles service. |
 | externalServices.tempo.authMode | string | `"basic"` | one of "none", "basic" |
 | externalServices.tempo.basicAuth.password | string | `""` | Tempo basic auth password |
 | externalServices.tempo.basicAuth.passwordKey | string | `"password"` | The key for the password property in the secret |
@@ -356,6 +372,15 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely
 | opencost.opencost.prometheus.password_key | string | `"password"` | The key for the password property in the secret. |
 | opencost.opencost.prometheus.secret_name | string | `"prometheus-k8s-monitoring"` | The name of the secret containing the username and password for the metrics service. This must be in the same namespace as the OpenCost deployment. |
 | opencost.opencost.prometheus.username_key | string | `"username"` | The key for the username property in the secret. |
+| profiles.ebpf.demangle | string | `"none"` | C++ demangle mode. Available options are: none, simplified, templates, full |
+| profiles.ebpf.enabled | bool | `true` | Gather profiles using eBPF |
+| profiles.ebpf.extraRelabelingRules | string | `""` | Rule blocks to be added to the discovery.relabel component for eBPF profile sources. ([docs](https://grafana.com/docs/agent/latest/flow/reference/components/discovery.relabel/#rule-block)) |
+| profiles.ebpf.namespaces | list | `[]` | Which namespaces to look for pods with profiles. |
+| profiles.enabled | bool | `false` | Receive and forward profiles. |
+| profiles.pprof.enabled | bool | `true` | Gather profiles by scraping pprof HTTP endpoints |
+| profiles.pprof.extraRelabelingRules | string | `""` | Rule blocks to be added to the discovery.relabel component for eBPF profile sources. ([docs](https://grafana.com/docs/agent/latest/flow/reference/components/discovery.relabel/#rule-block)) |
+| profiles.pprof.namespaces | list | `[]` | Which namespaces to look for pods with profiles. |
+| profiles.pprof.types | list | `["memory","cpu","goroutine","block","mutex","fgprof"]` | Profile types to gather |
 | prometheus-node-exporter.enabled | bool | `true` | Should this helm chart deploy Node Exporter to the cluster. Set this to false if your cluster already has Node Exporter, or if you do not want to scrape metrics from Node Exporter. |
 | prometheus-operator-crds.enabled | bool | `true` | Should this helm chart deploy the Prometheus Operator CRDs to the cluster. Set this to false if your cluster already has the CRDs, or if you do not to have the Grafana Agent scrape metrics from PodMonitors, Probes, or ServiceMonitors. |
 | prometheus-windows-exporter.config | string | `"collectors:\n  enabled: cpu,cs,container,logical_disk,memory,net,os\ncollector:\n  service:\n    services-where: \"Name='containerd' or Name='kubelet'\""` | Windows Exporter configuration |
@@ -385,7 +410,7 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely
 | receivers.zipkin.port | int | `9411` | Which port to use for the Zipkin receiver. This port needs to be opened in the grafana-agent section below. |
 | test.attempts | int | `10` | How many times to attempt the test job. |
 | test.enabled | bool | `true` | Should `helm test` run the test job? |
-| test.envOverrides | object | `{"LOKI_URL":"","PROMETHEUS_URL":"","TEMPO_URL":""}` | Overrides the URLs for various data sources |
+| test.envOverrides | object | `{"LOKI_URL":"","PROFILECLI_URL":"","PROMETHEUS_URL":"","TEMPO_URL":""}` | Overrides the URLs for various data sources |
 | test.extraAnnotations | object | `{}` | Extra annotations to add to the test job. |
 | test.extraLabels | object | `{}` | Extra labels to add to the test job. |
 | test.extraQueries | list | `[]` | Additional queries to run during the test. NOTE that this uses the host, username, and password in the externalServices section. The user account must have the ability to run queries. Example: extraQueries:   - query: prometheus_metric{cluster="my-cluster-name"}     type: promql  Can optionally provide expectations: - query: "avg(count_over_time(scrape_samples_scraped{cluster=~\"ci-test-cluster-2|from-the-other-agent\"}[1m]))"   type: promql   expect:     value: 1     operator: == |

charts/k8s-monitoring/ci/ci-values.yaml

@@ -4,6 +4,7 @@
 # * External Labels
 # * Pod logs (via volumes)
 # * Cluster Events
+# * Profiles
 #
 cluster:
   name: ci-test-cluster
@@ -31,6 +32,10 @@ externalServices:
     tlsOptions: |-
       insecure             = true
       insecure_skip_verify = true
+  pyroscope:
+    host: http://pyroscope.pyroscope.svc:4040
+    authMode: none
+    tenantId: 1
 
 metrics:
   apiserver:
@@ -49,6 +54,9 @@ metrics:
 traces:
   enabled: true
 
+profiles:
+  enabled: true
+
 test:
   attempts: 20
   extraQueries:
@@ -69,6 +77,10 @@ test:
     - query: "count_over_time({cluster=\"ci-test-cluster\", job!=\"integrations/kubernetes/eventhandler\"}[1h])"
       type: logql
 
+    # Check for profiles
+    - query: '{cluster="ci-test-cluster"}'
+      type: pyroql
+
     # DPM check
     - query: "avg(count_over_time(scrape_samples_scraped{cluster=\"ci-test-cluster\"}[1m]))"
       type: promql

charts/k8s-monitoring/schema-mods/enums-and-types.json

@@ -56,6 +56,20 @@
               "type": ["string", "integer"]
             }
           }
+        },
+        "pyroscope": {
+          "properties": {
+            "basicAuth": {
+              "properties": {
+                "username": {
+                  "type": ["string", "integer"]
+                }
+              }
+            },
+            "tenantId": {
+              "type": ["string", "integer"]
+            }
+          }
         }
       }
     },
@@ -209,6 +223,17 @@
           }
         }
       }
+    },
+    "profiles": {
+      "properties": {
+        "ebpf": {
+          "properties": {
+            "demangle": {
+              "enum": ["none", "simplified", "templates", "full"]
+            }
+          }
+        }
+      }
     }
   }
 }

charts/k8s-monitoring/templates/_configs.tpl

@@ -114,3 +114,11 @@
     {{- tpl .Values.logs.extraConfig . | indent 0 }}
   {{- end }}
 {{- end -}}
+
+{{/* Grafana Agent Profiles config */}}
+{{- define "agentProfilesConfig" -}}
+  {{- include "agent.config.profilesEbpf" . }}
+  {{- include "agent.config.profilesPprof" . }}
+
+  {{- include "agent.config.profilesService" . }}
+{{- end -}}

charts/k8s-monitoring/templates/_helpers.tpl

@@ -83,6 +83,14 @@
 {{- end }}
 {{- end }}
 
+{{- define "kubernetes_monitoring.profiles_service.secret.name" -}}
+{{- if .Values.externalServices.tempo.secret.name }}
+  {{- .Values.externalServices.tempo.secret.name }}
+{{- else }}
+  {{- printf "pyroscope-%s" .Chart.Name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+
 {{- define "escape_label" -}}
 {{ . | replace "-" "_" | replace "." "_" | replace "/" "_" }}
 {{- end }}