Merge remote-tracking branch 'upstream/master' into dashboard_gcp

CHANGELOG.next.asciidoc

@@ -216,6 +216,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add Kerberos support to Elasticsearch output. {pull}17927[17927]
 - Add support for fixed length extraction in `dissect` processor. {pull}17191[17191]
 - Set `agent.name` to the hostname by default. {issue}16377[16377] {pull}18000[18000]
+- Add support for basic ECS logging. {pull}17974[17974]
 - Add config example of how to skip the `add_host_metadata` processor when forwarding logs. {issue}13920[13920] {pull}18153[18153]
 - When using the `decode_json_fields` processor, decoded fields are now deep-merged into existing event. {pull}17958[17958]
 
@@ -299,6 +300,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve ECS categorization field mappings in system module. {issue}16031[16031] {pull}18065[18065]
 - When using the `json.*` setting available on some inputs, decoded fields are now deep-merged into existing event. {pull}17958[17958]
 - Change the `json.*` input settings implementation to merge parsed json objects with existing objects in the event instead of fully replacing them. {pull}17958[17958]
+- Improve ECS categorization field mappings in osquery module. {issue}16176[16176] {pull}17881[17881]
 
 *Heartbeat*
 

NOTICE.txt

@@ -3263,7 +3263,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------
 Dependency: github.com/hashicorp/errwrap
-Revision: 7554cd9344ce
+Version: v1.0.0
 License type (autodetected): MPL-2.0
 ./vendor/github.com/hashicorp/errwrap/LICENSE:
 --------------------------------------------------------------------
@@ -3624,7 +3624,7 @@ Exhibit B - “Incompatible With Secondary Licenses” Notice
 
 --------------------------------------------------------------------
 Dependency: github.com/hashicorp/go-multierror
-Revision: ed905158d874
+Version: v1.0.0
 License type (autodetected): MPL-2.0
 ./vendor/github.com/hashicorp/go-multierror/LICENSE:
 --------------------------------------------------------------------
@@ -6580,8 +6580,7 @@ SoundCloud Ltd. (http://soundcloud.com/).
 
 --------------------------------------------------------------------
 Dependency: github.com/prometheus/procfs
-Version: v0.0.9
-Revision: 42f6e295b56f
+Version: v0.0.11
 License type (autodetected): Apache-2.0
 ./vendor/github.com/prometheus/procfs/LICENSE:
 --------------------------------------------------------------------
@@ -7180,7 +7179,7 @@ SOFTWARE.
 
 --------------------------------------------------------------------
 Dependency: github.com/stretchr/testify
-Version: v1.4.0
+Version: v1.5.0
 License type (autodetected): MIT
 ./vendor/github.com/stretchr/testify/LICENSE:
 --------------------------------------------------------------------
@@ -7538,6 +7537,117 @@ License type (autodetected): Apache-2.0
 Apache License 2.0
 
 
+--------------------------------------------------------------------
+Dependency: go.elastic.co/ecszap
+Version: v0.1.1
+Revision: cdd95a104193
+License type (autodetected): Apache-2.0
+./vendor/go.elastic.co/ecszap/LICENSE:
+--------------------------------------------------------------------
+Apache License 2.0
+
+-------NOTICE.txt-----
+ecszap
+Copyright 2020-2020 Elasticsearch B.V.
+
+==========================================================================
+Third party libraries used by the Elastic ecszap project:
+==========================================================================
+
+Dependency: github.com/stretchr/testify
+Version: v1.4.0
+License type: MIT
+https://github.com/stretchr/testify/blob/199de5f3a493a9bf2dcc7fa5bf841d7759c13d7d/LICENSE:
+--------------------------------------------------------------------
+MIT License
+
+Copyright (c) 2012-2018 Mat Ryer and Tyler Bunnell
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+--------------------------------------------------------------------
+
+
+Dependency: go.uber.org/zap
+Version: v1.14.0
+License type: MIT
+https://github.com/uber-go/zap/blob/0bd02a6308c1bac3a03b02dc385555297cb22f83/LICENSE.txt:
+--------------------------------------------------------------------
+Copyright (c) 2016-2017 Uber Technologies, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+--------------------------------------------------------------------
+
+
+Dependency: github.com/magefile/mage
+Version: v1.9.0
+License type: Apache-2.0
+https://github.com/magefile/mage/blob/324c6690ed410efc1d9b597e477c46d42cbeb340/LICENSE:
+--------------------------------------------------------------------
+Apache License 2.0
+
+
+--------------------------------------------------------------------
+Dependency: github.com/pkg/errors
+Version: v0.9.1
+License type (autodetected): BSD-2-Clause
+https://github.com/pkg/errors/blob/614d223910a179a466c1767a985424175c39b465/LICENSE
+--------------------------------------------------------------------
+Copyright (c) 2015, Dave Cheney <dave@cheney.net>
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+--------------------------------------------------------------------
 --------------------------------------------------------------------
 Dependency: go.elastic.co/fastjson
 Version: v1.0.0
@@ -7558,7 +7668,7 @@ Apache License 2.0
 
 --------------------------------------------------------------------
 Dependency: go.uber.org/atomic
-Version: v1.3.1
+Version: v1.5.0
 License type (autodetected): MIT
 ./vendor/go.uber.org/atomic/LICENSE.txt:
 --------------------------------------------------------------------
@@ -7584,8 +7694,7 @@ THE SOFTWARE.
 
 --------------------------------------------------------------------
 Dependency: go.uber.org/multierr
-Version: v1.1.1
-Revision: fb7d312c2c04
+Version: v1.3.0
 License type (autodetected): MIT
 ./vendor/go.uber.org/multierr/LICENSE.txt:
 --------------------------------------------------------------------
@@ -7609,9 +7718,35 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
+--------------------------------------------------------------------
+Dependency: go.uber.org/tools
+Revision: 2cfd321de3ee
+License type (autodetected): MIT
+./vendor/go.uber.org/tools/LICENSE:
+--------------------------------------------------------------------
+Copyright (c) 2017 Uber Technologies, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
 --------------------------------------------------------------------
 Dependency: go.uber.org/zap
-Version: v1.7.1
+Version: v1.14.0
 License type (autodetected): MIT
 ./vendor/go.uber.org/zap/LICENSE.txt:
 --------------------------------------------------------------------
@@ -7841,7 +7976,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------
 Dependency: golang.org/x/sys
-Revision: c96a22e43c9c
+Revision: b016eb3dc98e
 License type (autodetected): BSD-3-Clause
 ./vendor/golang.org/x/sys/LICENSE:
 --------------------------------------------------------------------

auditbeat/auditbeat.reference.yml

@@ -1294,6 +1294,11 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
+# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
+# information. Recommended to use in combination with `logging.json=true`
+# Defaults to false.
+#logging.ecs: false
+
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

filebeat/filebeat.reference.yml

@@ -2007,6 +2007,11 @@ logging.files:
 # Set to true to log messages in JSON format.
 #logging.json: false
 
+# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
+# information. Recommended to use in combination with `logging.json=true`
+# Defaults to false.
+#logging.ecs: false
+
 # ============================= X-Pack Monitoring ==============================
 # Filebeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

filebeat/module/apache/access/config/access.yml

@@ -4,3 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

filebeat/module/apache/error/config/error.yml

@@ -6,4 +6,8 @@ paths:
 exclude_files: [".gz$"]
 
 processors:
-- add_locale: ~
+  - add_locale: ~
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

filebeat/module/auditd/log/config/log.yml

@@ -4,3 +4,8 @@ paths:
  - {{$path}}
 {{ end }}
 exclude_files: [".gz$"]
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

filebeat/module/elasticsearch/audit/config/audit.yml

@@ -6,4 +6,8 @@ paths:
 exclude_files: [".gz$"]
 
 processors:
-- add_locale: ~
+  - add_locale: ~
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

filebeat/module/elasticsearch/deprecation/config/log.yml

@@ -12,3 +12,7 @@ multiline:
 processors:
 # Locale for time zone is only needed in non-json logs
 - add_locale.when.not.regexp.message: "^{"
+- add_fields:
+    target: ''
+    fields:
+      ecs.version: 1.5.0

filebeat/module/elasticsearch/gc/config/gc.yml

@@ -9,3 +9,8 @@ multiline:
   pattern: '^(\[?[0-9]{4}-[0-9]{2}-[0-9]{2}|{)'
   negate: true
   match: after
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

filebeat/module/elasticsearch/server/config/log.yml

@@ -12,3 +12,7 @@ multiline:
 processors:
 # Locale for time zone is only needed in non-json logs
 - add_locale.when.not.regexp.message: "^{"
+- add_fields:
+    target: ''
+    fields:
+      ecs.version: 1.5.0

filebeat/module/elasticsearch/slowlog/config/slowlog.yml

@@ -13,3 +13,7 @@ multiline:
 processors:
 # Locale for time zone is only needed in non-json logs
 - add_locale.when.not.regexp.message: "^{"
+- add_fields:
+    target: ''
+    fields:
+      ecs.version: 1.5.0

filebeat/module/haproxy/log/config/file.yml

@@ -5,4 +5,8 @@ paths:
 {{ end }}
 exclude_files: [".gz$"]
 processors:
-- add_locale: ~
+  - add_locale: ~
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

filebeat/module/haproxy/log/config/syslog.yml

@@ -2,4 +2,8 @@ type: syslog
 protocol.udp:
   host: "{{.syslog_host}}:{{.syslog_port}}"
 processors:
-- add_locale: ~
+  - add_locale: ~
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

filebeat/module/icinga/debug/config/debug.yml

@@ -8,3 +8,8 @@ multiline:
   pattern: '^\['
   negate: true
   match: after
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

filebeat/module/icinga/main/config/main.yml

@@ -8,3 +8,8 @@ multiline:
   pattern: '^\['
   negate: true
   match: after
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

filebeat/module/icinga/startup/config/startup.yml

@@ -8,3 +8,8 @@ multiline:
   pattern: '^[a-z]*\/[a-zA-Z]*:'
   negate: true
   match: after
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

filebeat/module/iis/access/config/iis-access.yml

@@ -5,3 +5,8 @@ paths:
 {{ end }}
 exclude_files: [".gz$"]
 exclude_lines: ["^#"]
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0

filebeat/module/iis/error/config/iis-error.yml

@@ -5,3 +5,8 @@ paths:
 {{ end }}
 exclude_files: [".gz$"]
 exclude_lines: ["^#"]
+processors:
+  - add_fields:
+      target: ''
+      fields:
+        ecs.version: 1.5.0