Add the ability to set headers for all external services. (#703)

Signed-off-by: Pete Wall <pete.wall@grafana.com>

charts/k8s-monitoring/README.md

@@ -225,8 +225,10 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely
 | externalServices.loki.basicAuth.passwordKey | string | `"password"` | The key for the password property in the secret |
 | externalServices.loki.basicAuth.username | string | `""` | Loki basic auth username |
 | externalServices.loki.basicAuth.usernameKey | string | `"username"` | The key for the username property in the secret |
-| externalServices.loki.externalLabels | object | `{}` | Custom labels to be added to all logs and events, all values are treated as strings and automatically quoted. |
-| externalServices.loki.externalLabelsFrom | object | `{}` | Custom labels to be added to all logs and events through a dynamic reference, all values are treated as raw strings and not quoted. |
+| externalServices.loki.externalLabels | object | `{}` | Custom labels to be added to all logs and events. All values are treated as strings and automatically quoted. |
+| externalServices.loki.externalLabelsFrom | object | `{}` | Custom labels to be added to all logs and events through a dynamic reference. All values are treated as raw strings and not quoted. |
+| externalServices.loki.extraHeaders | object | `{}` | Extra headers to be set when sending metrics. All values are treated as strings and automatically quoted. |
+| externalServices.loki.extraHeadersFrom | object | `{}` | Extra headers to be set when sending metrics through a dynamic reference. All values are treated as raw strings and not quoted. |
 | externalServices.loki.host | string | `""` | Loki host where logs and events will be sent |
 | externalServices.loki.hostKey | string | `"host"` | The key for the host property in the secret |
 | externalServices.loki.oauth2.clientId | string | `""` | Loki OAuth2 client ID |
@@ -267,8 +269,10 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely
 | externalServices.prometheus.basicAuth.passwordKey | string | `"password"` | The key for the password property in the secret |
 | externalServices.prometheus.basicAuth.username | string | `""` | Prometheus basic auth username |
 | externalServices.prometheus.basicAuth.usernameKey | string | `"username"` | The key for the username property in the secret |
-| externalServices.prometheus.externalLabels | object | `{}` | Custom labels to be added to all time series, all values are treated as strings and automatically quoted. |
-| externalServices.prometheus.externalLabelsFrom | object | `{}` | Custom labels to be added to all time series through a dynamic reference, all values are treated as raw strings and not quoted. |
+| externalServices.prometheus.externalLabels | object | `{}` | Custom labels to be added to all time series. All values are treated as strings and automatically quoted. |
+| externalServices.prometheus.externalLabelsFrom | object | `{}` | Custom labels to be added to all time series through a dynamic reference. All values are treated as raw strings and not quoted. |
+| externalServices.prometheus.extraHeaders | object | `{}` | Extra headers to be set when sending metrics. All values are treated as strings and automatically quoted. |
+| externalServices.prometheus.extraHeadersFrom | object | `{}` | Extra headers to be set when sending metrics through a dynamic reference. All values are treated as raw strings and not quoted. |
 | externalServices.prometheus.host | string | `""` | Prometheus host where metrics will be sent |
 | externalServices.prometheus.hostKey | string | `"host"` | The key for the host property in the secret |
 | externalServices.prometheus.oauth2.clientId | string | `""` | Prometheus OAuth2 client ID |
@@ -323,8 +327,10 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely
 | externalServices.pyroscope.basicAuth.passwordKey | string | `"password"` | The key for the password property in the secret |
 | externalServices.pyroscope.basicAuth.username | string | `""` | Pyroscope basic auth username |
 | externalServices.pyroscope.basicAuth.usernameKey | string | `"username"` | The key for the username property in the secret |
-| externalServices.pyroscope.externalLabels | object | `{}` | Custom labels to be added to all profiles, all values are treated as strings and automatically quoted. |
-| externalServices.pyroscope.externalLabelsFrom | object | `{}` | Custom labels to be added to all profiles through a dynamic reference, all values are treated as raw strings and not quoted. |
+| externalServices.pyroscope.externalLabels | object | `{}` | Custom labels to be added to all profiles. All values are treated as strings and automatically quoted. |
+| externalServices.pyroscope.externalLabelsFrom | object | `{}` | Custom labels to be added to all profiles through a dynamic reference. All values are treated as raw strings and not quoted. |
+| externalServices.pyroscope.extraHeaders | object | `{}` | Extra headers to be set when sending metrics. All values are treated as strings and automatically quoted. |
+| externalServices.pyroscope.extraHeadersFrom | object | `{}` | Extra headers to be set when sending metrics through a dynamic reference. All values are treated as raw strings and not quoted. |
 | externalServices.pyroscope.host | string | `""` | Pyroscope host where profiles will be sent |
 | externalServices.pyroscope.hostKey | string | `"host"` | The key for the host property in the secret |
 | externalServices.pyroscope.proxyURL | string | `""` | HTTP proxy to proxy requests to Pyroscope through. |
@@ -344,6 +350,8 @@ The Prometheus and Loki services may be hosted on the same cluster, or remotely
 | externalServices.tempo.basicAuth.passwordKey | string | `"password"` | The key for the password property in the secret |
 | externalServices.tempo.basicAuth.username | string | `""` | Tempo basic auth username |
 | externalServices.tempo.basicAuth.usernameKey | string | `"username"` | The key for the username property in the secret |
+| externalServices.tempo.extraHeaders | object | `{}` | Extra headers to be set when sending metrics. All values are treated as strings and automatically quoted. |
+| externalServices.tempo.extraHeadersFrom | object | `{}` | Extra headers to be set when sending metrics through a dynamic reference. All values are treated as raw strings and not quoted. |
 | externalServices.tempo.host | string | `""` | Tempo host where traces will be sent |
 | externalServices.tempo.hostKey | string | `"host"` | The key for the host property in the secret |
 | externalServices.tempo.protocol | string | `"otlp"` | The type of server protocol for writing metrics Options:   * "otlp" will use OTLP   * "otlphttp" will use OTLP HTTP |

charts/k8s-monitoring/templates/alloy_config/_logs_service_loki.alloy.txt

@@ -5,6 +5,16 @@ loki.write "logs_service" {
   endpoint {
     url = nonsensitive(remote.kubernetes.secret.logs_service.data[{{ .hostKey | quote }}]) + "{{ .writeEndpoint }}"
     tenant_id = nonsensitive(remote.kubernetes.secret.logs_service.data[{{ .tenantIdKey | quote }}])
+{{- if or .extraHeaders .extraHeadersFrom }}
+    headers {
+{{- range $key, $value := .extraHeaders }}
+      {{ $key | quote }} = {{ $value | quote }},
+{{- end }}
+{{- range $key, $value := .extraHeadersFrom }}
+      {{ $key | quote }} = {{ $value }},
+{{- end }}
+    }
+{{- end }}
 {{- if .proxyURL }}
     proxy_url = {{ .proxyURL | quote }}
 {{- end }}

charts/k8s-monitoring/templates/alloy_config/_logs_service_otlp.alloy.txt

@@ -71,9 +71,15 @@ otelcol.exporter.otlphttp "logs_service" {
 {{ if or (.basicAuth.username) (.basicAuth.password) }}
     auth = otelcol.auth.basic.logs_service.handler
 {{- end }}
-{{- if .tenantId }}
-    headers = { "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.logs_service.data[{{ .tenantIdKey | quote }}]) }
+    headers = {
+      "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.logs_service.data[{{ .tenantIdKey | quote }}]),
+{{- range $key, $value := .extraHeaders }}
+      {{ $key | quote }} = {{ $value | quote }},
 {{- end }}
+{{- range $key, $value := .extraHeadersFrom }}
+      {{ $key | quote }} = {{ $value }},
+{{- end }}
+    }
 {{- if .tls }}
     tls {
     {{- range $k, $v := .tls }}

charts/k8s-monitoring/templates/alloy_config/_metrics_service_otlp.alloy.txt

@@ -71,9 +71,15 @@ otelcol.exporter.otlphttp "metrics_service" {
 {{ if or (.basicAuth.username) (.basicAuth.password) }}
     auth = otelcol.auth.basic.metrics_service.handler
 {{- end }}
-{{- if .tenantId }}
-    headers = { "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.metrics_service.data[{{ .tenantIdKey | quote }}]) }
+    headers = {
+      "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.metrics_service.data[{{ .tenantIdKey | quote }}]),
+{{- range $key, $value := .extraHeaders }}
+      {{ $key | quote }} = {{ $value | quote }},
 {{- end }}
+{{- range $key, $value := .extraHeadersFrom }}
+      {{ $key | quote }} = {{ $value }},
+{{- end }}
+    }
 {{- if .tls }}
     tls {
     {{- range $k, $v := .tls }}

charts/k8s-monitoring/templates/alloy_config/_metrics_service_remote_write.alloy.txt

@@ -3,7 +3,15 @@
 prometheus.remote_write "metrics_service" {
   endpoint {
     url = nonsensitive(remote.kubernetes.secret.metrics_service.data[{{ .hostKey | quote }}]) + "{{ .writeEndpoint }}"
-    headers = { "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.metrics_service.data[{{ .tenantIdKey | quote }}]) }
+    headers = {
+      "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.metrics_service.data[{{ .tenantIdKey | quote }}]),
+{{- range $key, $value := .extraHeaders }}
+      {{ $key | quote }} = {{ $value | quote }},
+{{- end }}
+{{- range $key, $value := .extraHeadersFrom }}
+      {{ $key | quote }} = {{ $value }},
+{{- end }}
+    }
 {{- if .proxyURL }}
     proxy_url = {{ .proxyURL | quote }}
 {{- end }}

charts/k8s-monitoring/templates/alloy_config/_profiles_service.alloy.txt

@@ -9,7 +9,15 @@ remote.kubernetes.secret "profiles_service" {
 pyroscope.write "profiles_service" {
   endpoint {
     url = nonsensitive(remote.kubernetes.secret.profiles_service.data[{{ .hostKey | quote }}])
-    headers = { "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.profiles_service.data[{{ .tenantIdKey | quote }}]) }
+    headers = {
+      "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.profiles_service.data[{{ .tenantIdKey | quote }}]),
+{{- range $key, $value := .extraHeaders }}
+      {{ $key | quote }} = {{ $value | quote }},
+{{- end }}
+{{- range $key, $value := .extraHeadersFrom }}
+      {{ $key | quote }} = {{ $value }},
+{{- end }}
+    }
 {{ if eq .authMode "basic" }}
     basic_auth {
       username = nonsensitive(remote.kubernetes.secret.profiles_service.data[{{ .basicAuth.usernameKey | quote }}])

charts/k8s-monitoring/templates/alloy_config/_traces_service.alloy.txt

@@ -22,7 +22,15 @@ otelcol.exporter.otlphttp "traces_service" {
 {{ if eq .authMode "basic" }}
     auth = otelcol.auth.basic.traces_service.handler
 {{- end }}
-    headers = { "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.traces_service.data[{{ .tenantIdKey | quote }}]) }
+    headers = {
+      "X-Scope-OrgID" = nonsensitive(remote.kubernetes.secret.metrics_service.data[{{ .tenantIdKey | quote }}]),
+{{- range $key, $value := .extraHeaders }}
+      {{ $key | quote }} = {{ $value | quote }},
+{{- end }}
+{{- range $key, $value := .extraHeadersFrom }}
+      {{ $key | quote }} = {{ $value }},
+{{- end }}
+    }
 {{- if .tlsOptions }}
     tls {
 {{ .tlsOptions | indent 6 }}

charts/k8s-monitoring/values.yaml

@@ -33,16 +33,29 @@ externalServices:
     # -- HTTP proxy to proxy requests to Prometheus through.
     # @section -- External Services (Prometheus)
     proxyURL: ""
+
     # -- Prometheus metrics query endpoint. Preset for Grafana Cloud Metrics instances.
     # @section -- External Services (Prometheus)
     queryEndpoint: /api/prom/api/v1/query
     # -- Prometheus metrics write endpoint. Preset for Grafana Cloud Metrics instances.
     # @section -- External Services (Prometheus)
     writeEndpoint: /api/prom/push
-    # -- Custom labels to be added to all time series, all values are treated as strings and automatically quoted.
+
+    # -- Extra headers to be set when sending metrics.
+    # All values are treated as strings and automatically quoted.
+    # @section -- External Services (Prometheus)
+    extraHeaders: {}
+    # -- Extra headers to be set when sending metrics through a dynamic reference.
+    # All values are treated as raw strings and not quoted.
+    # @section -- External Services (Prometheus)
+    extraHeadersFrom: {}
+
+    # -- Custom labels to be added to all time series.
+    # All values are treated as strings and automatically quoted.
     # @section -- External Services (Prometheus)
     externalLabels: {}
-    # -- Custom labels to be added to all time series through a dynamic reference, all values are treated as raw strings and not quoted.
+    # -- Custom labels to be added to all time series through a dynamic reference.
+    # All values are treated as raw strings and not quoted.
     # @section -- External Services (Prometheus)
     externalLabelsFrom: {}
     # -- Rule blocks to be added to the [write_relabel_config block](https://grafana.com/docs/alloy/latest/reference/components/prometheus.remote_write/#write_relabel_config-block)
@@ -235,18 +248,33 @@ externalServices:
     # -- HTTP proxy to proxy requests to Loki through.
     # @section -- External Services (Loki)
     proxyURL: ""
+
     # -- Loki logs query endpoint.
     # @section -- External Services (Loki)
     queryEndpoint: /loki/api/v1/query
     # -- Loki logs write endpoint.
     # @section -- External Services (Loki)
     writeEndpoint: /loki/api/v1/push
-    # -- Custom labels to be added to all logs and events, all values are treated as strings and automatically quoted.
+
+    # -- Extra headers to be set when sending metrics.
+    # All values are treated as strings and automatically quoted.
+    # @section -- External Services (Loki)
+    extraHeaders: {}
+
+    # -- Extra headers to be set when sending metrics through a dynamic reference.
+    # All values are treated as raw strings and not quoted.
+    # @section -- External Services (Loki)
+    extraHeadersFrom: {}
+
+    # -- Custom labels to be added to all logs and events.
+    # All values are treated as strings and automatically quoted.
     # @section -- External Services (Loki)
     externalLabels: {}
-    # -- Custom labels to be added to all logs and events through a dynamic reference, all values are treated as raw strings and not quoted.
+    # -- Custom labels to be added to all logs and events through a dynamic reference.
+    # All values are treated as raw strings and not quoted.
     # @section -- External Services (Loki)
     externalLabelsFrom: {}
+
     # -- Loki tenant ID
     # @section -- External Services (Loki)
     tenantId: ""
@@ -365,6 +393,15 @@ externalServices:
     # @section -- External Services (Tempo)
     searchEndpoint: /api/search
 
+    # -- Extra headers to be set when sending metrics.
+    # All values are treated as strings and automatically quoted.
+    # @section -- External Services (Tempo)
+    extraHeaders: {}
+    # -- Extra headers to be set when sending metrics through a dynamic reference.
+    # All values are treated as raw strings and not quoted.
+    # @section -- External Services (Tempo)
+    extraHeadersFrom: {}
+
     # -- The type of server protocol for writing metrics
     # Options:
     #   * "otlp" will use OTLP
@@ -434,10 +471,21 @@ externalServices:
     # @section -- External Services (Pyroscope)
     proxyURL: ""
 
-    # -- Custom labels to be added to all profiles, all values are treated as strings and automatically quoted.
+    # -- Extra headers to be set when sending metrics.
+    # All values are treated as strings and automatically quoted.
+    # @section -- External Services (Pyroscope)
+    extraHeaders: {}
+    # -- Extra headers to be set when sending metrics through a dynamic reference.
+    # All values are treated as raw strings and not quoted.
+    # @section -- External Services (Pyroscope)
+    extraHeadersFrom: {}
+
+    # -- Custom labels to be added to all profiles.
+    # All values are treated as strings and automatically quoted.
     # @section -- External Services (Pyroscope)
     externalLabels: {}
-    # -- Custom labels to be added to all profiles through a dynamic reference, all values are treated as raw strings and not quoted.
+    # -- Custom labels to be added to all profiles through a dynamic reference.
+    # All values are treated as raw strings and not quoted.
     # @section -- External Services (Pyroscope)
     externalLabelsFrom: {}