diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5aa72b534f1..2aeb03c1f93 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -53,6 +53,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - TLS: Fields have been changed to adapt to ECS. {pull}15497[15497] - TLS: The behavior of send_certificates and include_raw_certificates options has changed. {pull}15497[15497] +- Add dns.question.subdomain and dns.question.top_level_domain fields. {pull}14578[14578] +- Add support for mongodb opcode 2013 (OP_MSG). {issue}6191[6191] {pull}8594[8594] +- NFSv4: Always use opname `ILLEGAL` when failed to match request to a valid nfs operation. {pull}11503[11503] +- Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. {pull}15353[15353] *Winlogbeat* diff --git a/packetbeat/_meta/beat.reference.yml b/packetbeat/_meta/beat.reference.yml index 49c1ef61647..6ddd057b4c9 100644 --- a/packetbeat/_meta/beat.reference.yml +++ b/packetbeat/_meta/beat.reference.yml @@ -207,6 +207,11 @@ packetbeat.protocols: # all headers by setting this option to true. The default is false. #send_all_headers: false + # A list of headers to redact if present in the HTTP request. This will keep + # the header field present, but will redact it's value to show the headers + # presence. + #redact_headers: [] + # The list of content types for which Packetbeat includes the full HTTP # payload. If the request's or response's Content-Type matches any on this # list, the full body will be included under the request or response field. diff --git a/packetbeat/docs/packetbeat-options.asciidoc b/packetbeat/docs/packetbeat-options.asciidoc index f8c594a779f..490e49f3ed7 100644 --- a/packetbeat/docs/packetbeat-options.asciidoc +++ b/packetbeat/docs/packetbeat-options.asciidoc @@ -716,6 +716,12 @@ headers are placed under the `headers` dictionary in the resulting JSON. Instead of sending a white list of headers to Elasticsearch, you can send all headers by setting this option to true. The default is false. +===== `redact_headers` + +A list of headers to redact if present in the HTTP request. This will keep +the header field present, but will redact it's value to show the header's +presence. + ===== `include_body_for` The list of content types for which Packetbeat exports the full HTTP payload. The HTTP body is available under diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml index 7931b4632ae..aeb541e1a3a 100644 --- a/packetbeat/packetbeat.reference.yml +++ b/packetbeat/packetbeat.reference.yml @@ -207,6 +207,11 @@ packetbeat.protocols: # all headers by setting this option to true. The default is false. #send_all_headers: false + # A list of headers to redact if present in the HTTP request. This will keep + # the header field present, but will redact it's value to show the headers + # presence. + #redact_headers: [] + # The list of content types for which Packetbeat includes the full HTTP # payload. If the request's or response's Content-Type matches any on this # list, the full body will be included under the request or response field. diff --git a/packetbeat/protos/http/config.go b/packetbeat/protos/http/config.go index 19f5ab20393..86f59fb36b1 100644 --- a/packetbeat/protos/http/config.go +++ b/packetbeat/protos/http/config.go @@ -36,6 +36,7 @@ type httpConfig struct { RedactAuthorization bool `config:"redact_authorization"` MaxMessageSize int `config:"max_message_size"` DecodeBody bool `config:"decode_body"` + RedactHeaders []string `config:"redact_headers"` } var ( diff --git a/packetbeat/protos/http/http.go b/packetbeat/protos/http/http.go index 4e40ad60ba0..efa344ab163 100644 --- a/packetbeat/protos/http/http.go +++ b/packetbeat/protos/http/http.go @@ -88,6 +88,7 @@ type httpPlugin struct { splitCookie bool hideKeywords []string redactAuthorization bool + redactHeaders []string maxMessageSize int mustDecodeBody bool @@ -147,6 +148,11 @@ func (http *httpPlugin) setFromConfig(config *httpConfig) { http.transactionTimeout = config.TransactionTimeout http.mustDecodeBody = config.DecodeBody + http.redactHeaders = make([]string, len(config.RedactHeaders)) + for i, header := range config.RedactHeaders { + http.redactHeaders[i] = strings.ToLower(header) + } + for _, list := range [][]string{config.IncludeBodyFor, config.IncludeRequestBodyFor} { http.parserConfig.includeRequestBodyFor = append(http.parserConfig.includeRequestBodyFor, list...) } @@ -725,6 +731,12 @@ func extractHostHeader(header string) (host string, port int) { } func (http *httpPlugin) hideHeaders(m *message) { + for _, header := range http.redactHeaders { + if _, exists := m.headers[header]; exists { + m.headers[header] = []byte("REDACTED") + } + } + if !m.isRequest || !http.redactAuthorization { return } diff --git a/packetbeat/protos/http/http_test.go b/packetbeat/protos/http/http_test.go index 5b782e2052e..69c8cd792ed 100644 --- a/packetbeat/protos/http/http_test.go +++ b/packetbeat/protos/http/http_test.go @@ -990,6 +990,43 @@ func TestHttpParser_RedactAuthorization_Proxy_raw(t *testing.T) { } } +func TestHttpParser_RedactHeaders(t *testing.T) { + logp.TestingSetup(logp.WithSelectors("http", "httpdetailed")) + + http := httpModForTests(nil) + http.redactAuthorization = true + http.parserConfig.sendHeaders = true + http.parserConfig.sendAllHeaders = true + http.redactHeaders = []string{"header-to-redact", "should-not-exist"} + + data := []byte("POST /services/ObjectControl?ID=client0 HTTP/1.1\r\n" + + "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; MS Web Services Client Protocol 2.0.50727.5472)\r\n" + + "Content-Type: text/xml; charset=utf-8\r\n" + + "SOAPAction: \"\"\r\n" + + "Header-To-Redact: sensitive-value\r\n" + + "Host: production.example.com\r\n" + + "Content-Length: 0\r\n" + + "Expect: 100-continue\r\n" + + "Accept-Encoding: gzip\r\n" + + "X-Forwarded-For: 10.216.89.132\r\n" + + "\r\n") + + st := &stream{data: data, message: new(message)} + + ok, _ := testParseStream(http, st, 0) + + http.hideHeaders(st.message) + + assert.True(t, ok) + var redactedString common.NetString = []byte("REDACTED") + var expectedAcceptEncoding common.NetString = []byte("gzip") + assert.Equal(t, redactedString, st.message.headers["header-to-redact"]) + assert.Equal(t, expectedAcceptEncoding, st.message.headers["accept-encoding"]) + + _, invalidHeaderExists := st.message.headers["should-not-exist"] + assert.False(t, invalidHeaderExists) +} + func Test_splitCookiesHeader(t *testing.T) { type io struct { Input string