From ddeeb023706b15b394e7f823301dbe269e112203 Mon Sep 17 00:00:00 2001 From: Matthew Costa Date: Wed, 8 Jan 2020 18:54:42 +0000 Subject: [PATCH] Configuration option for HTTP request header redaction. (#15353) Add redact_headers configuration option, which allows specific HTTP request headers to be redacted. I've run into situations where people have added things like API keys into HTTP headers, which are making their way into our logs. --- CHANGELOG.next.asciidoc | 1 + packetbeat/_meta/beat.reference.yml | 5 +++ packetbeat/docs/packetbeat-options.asciidoc | 6 ++++ packetbeat/packetbeat.reference.yml | 5 +++ packetbeat/protos/http/config.go | 1 + packetbeat/protos/http/http.go | 12 +++++++ packetbeat/protos/http/http_test.go | 37 +++++++++++++++++++++ 7 files changed, 67 insertions(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 16f79447679..cf38e7dc730 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -83,6 +83,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add dns.question.subdomain and dns.question.top_level_domain fields. {pull}14578[14578] - Add support for mongodb opcode 2013 (OP_MSG). {issue}6191[6191] {pull}8594[8594] - NFSv4: Always use opname `ILLEGAL` when failed to match request to a valid nfs operation. {pull}11503[11503] +- Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. {pull}15353[15353] *Winlogbeat* diff --git a/packetbeat/_meta/beat.reference.yml b/packetbeat/_meta/beat.reference.yml index a379b1e4f54..237eac991fe 100644 --- a/packetbeat/_meta/beat.reference.yml +++ b/packetbeat/_meta/beat.reference.yml @@ -200,6 +200,11 @@ packetbeat.protocols: # all headers by setting this option to true. The default is false. #send_all_headers: false + # A list of headers to redact if present in the HTTP request. This will keep + # the header field present, but will redact it's value to show the headers + # presence. + #redact_headers: [] + # The list of content types for which Packetbeat includes the full HTTP # payload. If the request's or response's Content-Type matches any on this # list, the full body will be included under the request or response field. diff --git a/packetbeat/docs/packetbeat-options.asciidoc b/packetbeat/docs/packetbeat-options.asciidoc index f8767956726..b87193a093c 100644 --- a/packetbeat/docs/packetbeat-options.asciidoc +++ b/packetbeat/docs/packetbeat-options.asciidoc @@ -682,6 +682,12 @@ headers are placed under the `headers` dictionary in the resulting JSON. Instead of sending a white list of headers to Elasticsearch, you can send all headers by setting this option to true. The default is false. +===== `redact_headers` + +A list of headers to redact if present in the HTTP request. This will keep +the header field present, but will redact it's value to show the header's +presence. + ===== `include_body_for` The list of content types for which Packetbeat exports the full HTTP payload. The HTTP body is available under diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 042e53b6b83..86ceb94d6c4 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -200,6 +200,11 @@ packetbeat.protocols: # all headers by setting this option to true. The default is false. #send_all_headers: false + # A list of headers to redact if present in the HTTP request. This will keep + # the header field present, but will redact it's value to show the headers + # presence. + #redact_headers: [] + # The list of content types for which Packetbeat includes the full HTTP # payload. If the request's or response's Content-Type matches any on this # list, the full body will be included under the request or response field. diff --git a/packetbeat/protos/http/config.go b/packetbeat/protos/http/config.go index 2197dbad088..14b3f55e359 100644 --- a/packetbeat/protos/http/config.go +++ b/packetbeat/protos/http/config.go @@ -36,6 +36,7 @@ type httpConfig struct { RedactAuthorization bool `config:"redact_authorization"` MaxMessageSize int `config:"max_message_size"` DecodeBody bool `config:"decode_body"` + RedactHeaders []string `config:"redact_headers"` } var ( diff --git a/packetbeat/protos/http/http.go b/packetbeat/protos/http/http.go index 69821429c6c..dbc76badfd8 100644 --- a/packetbeat/protos/http/http.go +++ b/packetbeat/protos/http/http.go @@ -88,6 +88,7 @@ type httpPlugin struct { splitCookie bool hideKeywords []string redactAuthorization bool + redactHeaders []string maxMessageSize int mustDecodeBody bool @@ -147,6 +148,11 @@ func (http *httpPlugin) setFromConfig(config *httpConfig) { http.transactionTimeout = config.TransactionTimeout http.mustDecodeBody = config.DecodeBody + http.redactHeaders = make([]string, len(config.RedactHeaders)) + for i, header := range config.RedactHeaders { + http.redactHeaders[i] = strings.ToLower(header) + } + for _, list := range [][]string{config.IncludeBodyFor, config.IncludeRequestBodyFor} { http.parserConfig.includeRequestBodyFor = append(http.parserConfig.includeRequestBodyFor, list...) } @@ -725,6 +731,12 @@ func extractHostHeader(header string) (host string, port int) { } func (http *httpPlugin) hideHeaders(m *message) { + for _, header := range http.redactHeaders { + if _, exists := m.headers[header]; exists { + m.headers[header] = []byte("REDACTED") + } + } + if !m.isRequest || !http.redactAuthorization { return } diff --git a/packetbeat/protos/http/http_test.go b/packetbeat/protos/http/http_test.go index 12e56feefed..f07af54008d 100644 --- a/packetbeat/protos/http/http_test.go +++ b/packetbeat/protos/http/http_test.go @@ -990,6 +990,43 @@ func TestHttpParser_RedactAuthorization_Proxy_raw(t *testing.T) { } } +func TestHttpParser_RedactHeaders(t *testing.T) { + logp.TestingSetup(logp.WithSelectors("http", "httpdetailed")) + + http := httpModForTests(nil) + http.redactAuthorization = true + http.parserConfig.sendHeaders = true + http.parserConfig.sendAllHeaders = true + http.redactHeaders = []string{"header-to-redact", "should-not-exist"} + + data := []byte("POST /services/ObjectControl?ID=client0 HTTP/1.1\r\n" + + "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; MS Web Services Client Protocol 2.0.50727.5472)\r\n" + + "Content-Type: text/xml; charset=utf-8\r\n" + + "SOAPAction: \"\"\r\n" + + "Header-To-Redact: sensitive-value\r\n" + + "Host: production.example.com\r\n" + + "Content-Length: 0\r\n" + + "Expect: 100-continue\r\n" + + "Accept-Encoding: gzip\r\n" + + "X-Forwarded-For: 10.216.89.132\r\n" + + "\r\n") + + st := &stream{data: data, message: new(message)} + + ok, _ := testParseStream(http, st, 0) + + http.hideHeaders(st.message) + + assert.True(t, ok) + var redactedString common.NetString = []byte("REDACTED") + var expectedAcceptEncoding common.NetString = []byte("gzip") + assert.Equal(t, redactedString, st.message.headers["header-to-redact"]) + assert.Equal(t, expectedAcceptEncoding, st.message.headers["accept-encoding"]) + + _, invalidHeaderExists := st.message.headers["should-not-exist"] + assert.False(t, invalidHeaderExists) +} + func Test_splitCookiesHeader(t *testing.T) { type io struct { Input string