Configuration option for HTTP request header redaction. (#15353)

Add redact_headers configuration option, which allows specific HTTP request headers to be redacted.

I've run into situations where people have added things like API keys into HTTP headers, which are making their way into our logs.

CHANGELOG.next.asciidoc

@@ -83,6 +83,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add dns.question.subdomain and dns.question.top_level_domain fields. {pull}14578[14578]
 - Add support for mongodb opcode 2013 (OP_MSG). {issue}6191[6191] {pull}8594[8594]
 - NFSv4: Always use opname `ILLEGAL` when failed to match request to a valid nfs operation. {pull}11503[11503]
+- Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. {pull}15353[15353]
 
 *Winlogbeat*
 

packetbeat/_meta/beat.reference.yml

@@ -200,6 +200,11 @@ packetbeat.protocols:
   # all headers by setting this option to true. The default is false.
   #send_all_headers: false
 
+  # A list of headers to redact if present in the HTTP request. This will keep
+  # the header field present, but will redact it's value to show the headers
+  # presence.
+  #redact_headers: []
+
   # The list of content types for which Packetbeat includes the full HTTP
   # payload. If the request's or response's Content-Type matches any on this
   # list, the full body will be included under the request or response field.

packetbeat/docs/packetbeat-options.asciidoc

@@ -682,6 +682,12 @@ headers are placed under the `headers` dictionary in the resulting JSON.
 Instead of sending a white list of headers to Elasticsearch, you can
 send all headers by setting this option to true. The default is false.
 
+===== `redact_headers`
+
+A list of headers to redact if present in the HTTP request. This will keep
+the header field present, but will redact it's value to show the header's
+presence.
+
 ===== `include_body_for`
 
 The list of content types for which Packetbeat exports the full HTTP payload. The HTTP body is available under

packetbeat/packetbeat.reference.yml

@@ -200,6 +200,11 @@ packetbeat.protocols:
   # all headers by setting this option to true. The default is false.
   #send_all_headers: false
 
+  # A list of headers to redact if present in the HTTP request. This will keep
+  # the header field present, but will redact it's value to show the headers
+  # presence.
+  #redact_headers: []
+
   # The list of content types for which Packetbeat includes the full HTTP
   # payload. If the request's or response's Content-Type matches any on this
   # list, the full body will be included under the request or response field.

packetbeat/protos/http/config.go

@@ -36,6 +36,7 @@ type httpConfig struct {
 	RedactAuthorization    bool     `config:"redact_authorization"`
 	MaxMessageSize         int      `config:"max_message_size"`
 	DecodeBody             bool     `config:"decode_body"`
+	RedactHeaders          []string `config:"redact_headers"`
 }
 
 var (

packetbeat/protos/http/http.go

@@ -88,6 +88,7 @@ type httpPlugin struct {
 	splitCookie         bool
 	hideKeywords        []string
 	redactAuthorization bool
+	redactHeaders       []string
 	maxMessageSize      int
 	mustDecodeBody      bool
 
@@ -147,6 +148,11 @@ func (http *httpPlugin) setFromConfig(config *httpConfig) {
 	http.transactionTimeout = config.TransactionTimeout
 	http.mustDecodeBody = config.DecodeBody
 
+	http.redactHeaders = make([]string, len(config.RedactHeaders))
+	for i, header := range config.RedactHeaders {
+		http.redactHeaders[i] = strings.ToLower(header)
+	}
+
 	for _, list := range [][]string{config.IncludeBodyFor, config.IncludeRequestBodyFor} {
 		http.parserConfig.includeRequestBodyFor = append(http.parserConfig.includeRequestBodyFor, list...)
 	}
@@ -725,6 +731,12 @@ func extractHostHeader(header string) (host string, port int) {
 }
 
 func (http *httpPlugin) hideHeaders(m *message) {
+	for _, header := range http.redactHeaders {
+		if _, exists := m.headers[header]; exists {
+			m.headers[header] = []byte("REDACTED")
+		}
+	}
+
 	if !m.isRequest || !http.redactAuthorization {
 		return
 	}

packetbeat/protos/http/http_test.go

@@ -990,6 +990,43 @@ func TestHttpParser_RedactAuthorization_Proxy_raw(t *testing.T) {
 	}
 }
 
+func TestHttpParser_RedactHeaders(t *testing.T) {
+	logp.TestingSetup(logp.WithSelectors("http", "httpdetailed"))
+
+	http := httpModForTests(nil)
+	http.redactAuthorization = true
+	http.parserConfig.sendHeaders = true
+	http.parserConfig.sendAllHeaders = true
+	http.redactHeaders = []string{"header-to-redact", "should-not-exist"}
+
+	data := []byte("POST /services/ObjectControl?ID=client0 HTTP/1.1\r\n" +
+		"User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; MS Web Services Client Protocol 2.0.50727.5472)\r\n" +
+		"Content-Type: text/xml; charset=utf-8\r\n" +
+		"SOAPAction: \"\"\r\n" +
+		"Header-To-Redact: sensitive-value\r\n" +
+		"Host: production.example.com\r\n" +
+		"Content-Length: 0\r\n" +
+		"Expect: 100-continue\r\n" +
+		"Accept-Encoding: gzip\r\n" +
+		"X-Forwarded-For: 10.216.89.132\r\n" +
+		"\r\n")
+
+	st := &stream{data: data, message: new(message)}
+
+	ok, _ := testParseStream(http, st, 0)
+
+	http.hideHeaders(st.message)
+
+	assert.True(t, ok)
+	var redactedString common.NetString = []byte("REDACTED")
+	var expectedAcceptEncoding common.NetString = []byte("gzip")
+	assert.Equal(t, redactedString, st.message.headers["header-to-redact"])
+	assert.Equal(t, expectedAcceptEncoding, st.message.headers["accept-encoding"])
+
+	_, invalidHeaderExists := st.message.headers["should-not-exist"]
+	assert.False(t, invalidHeaderExists)
+}
+
 func Test_splitCookiesHeader(t *testing.T) {
 	type io struct {
 		Input  string