diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 9faaf5d5931..ef4ecda8836 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -4,6 +4,8 @@ Please label this PR with one of the following labels, depending on the scope of - Enhancement - Breaking change - Deprecation +- Cleanup +- Docs --> ## What does this PR do? diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc index cf02bc84960..0756e36fdf5 100644 --- a/CHANGELOG-developer.next.asciidoc +++ b/CHANGELOG-developer.next.asciidoc @@ -102,3 +102,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only. - Update Go version to 1.14.7. {pull}20508[20508] - Add packaging for docker image based on UBI minimal 8. {pull}20576[20576] - Make the mage binary used by the build process in the docker container to be statically compiled. {pull}20827[20827] +- Update ecszap to v0.3.0 for using ECS 1.6.0 in logs {pull}22267[22267] \ No newline at end of file diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 815b8d809f1..63e8d97ee74 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -385,6 +385,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - [Kubernetes] Remove redundant dockersock volume mount {pull}22009[22009] - Revert change to report `process.memory.rss` as `process.memory.wss` on Windows. {pull}22055[22055] - Add a switch to the driver definition on SQL module to use pretty names {pull}17378[17378] +- Remove io.time from windows {pull}22237[22237] - Add interval information to `monitor` metricset in azure. {pull}22152[22152] *Packetbeat* diff --git a/NOTICE.txt b/NOTICE.txt index 477f0b53201..640ee9045c7 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -14853,11 +14853,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmhttp@v -------------------------------------------------------------------------------- Dependency : go.elastic.co/ecszap -Version: v0.1.1-0.20200424093508-cdd95a104193 +Version: v0.3.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/go.elastic.co/ecszap@v0.1.1-0.20200424093508-cdd95a104193/LICENSE: +Contents of probable licence file $GOMODCACHE/go.elastic.co/ecszap@v0.3.0/LICENSE: Apache License diff --git a/filebeat/docs/modules/cisco.asciidoc b/filebeat/docs/modules/cisco.asciidoc index d087826245e..742fe8020bc 100644 --- a/filebeat/docs/modules/cisco.asciidoc +++ b/filebeat/docs/modules/cisco.asciidoc @@ -432,6 +432,15 @@ The ID for the access key used to read from the SQS queue. The secret token used for authenticating to the SQS queue. +*`var.visibility_timeout`*:: + +The duration that the received messages are hidden from ReceiveMessage request. +Default to be 300 seconds. + +*`var.api_timeout`*:: + +Maximum duration before AWS API request will be interrupted. Default to be 120 seconds. + :has-dashboards!: :fileset_ex!: diff --git a/go.mod b/go.mod index 2ef65606319..60a2299a32e 100644 --- a/go.mod +++ b/go.mod @@ -157,7 +157,7 @@ require ( go.elastic.co/apm v1.8.1-0.20200909061013-2aef45b9cf4b go.elastic.co/apm/module/apmelasticsearch v1.7.2 go.elastic.co/apm/module/apmhttp v1.7.2 - go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193 + go.elastic.co/ecszap v0.3.0 go.elastic.co/go-licence-detector v0.4.0 go.etcd.io/bbolt v1.3.4 go.uber.org/atomic v1.5.0 diff --git a/go.sum b/go.sum index 97f31d79292..697368fe0ab 100644 --- a/go.sum +++ b/go.sum @@ -740,8 +740,8 @@ go.elastic.co/apm/module/apmelasticsearch v1.7.2 h1:5STGHLZLSeAzxordMc+dFVKiyVtM go.elastic.co/apm/module/apmelasticsearch v1.7.2/go.mod h1:ZyNFuyWdt42GBZkz0SogoLzDBrBGj4orxpiUuxYeYq8= go.elastic.co/apm/module/apmhttp v1.7.2 h1:2mRh7SwBuEVLmJlX+hsMdcSg9xaielCLElaPn/+i34w= go.elastic.co/apm/module/apmhttp v1.7.2/go.mod h1:sTFWiWejnhSdZv6+dMgxGec2Nxe/ZKfHfz/xtRM+cRY= -go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193 h1:NjYJ/beChqugXSavTkH5tF6shvr/is8jdgJ331wfwT8= -go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193/go.mod h1:HTUi+QRmr3EuZMqxPX+5fyOdMNfUu5iPebgfhgsTJYQ= +go.elastic.co/ecszap v0.3.0 h1:Zo/Y4sJLqbWDlqCHI4F4Lzeg0Fs4+n5ldVis4h9xV8w= +go.elastic.co/ecszap v0.3.0/go.mod h1:HTUi+QRmr3EuZMqxPX+5fyOdMNfUu5iPebgfhgsTJYQ= go.elastic.co/fastjson v1.0.0/go.mod h1:PmeUOMMtLHQr9ZS9J9owrAVg0FkaZDRZJEFTTGHtchs= go.elastic.co/fastjson v1.1.0 h1:3MrGBWWVIxe/xvsbpghtkFoPciPhOCmjsR/HfwEeQR4= go.elastic.co/fastjson v1.1.0/go.mod h1:boNGISWMjQsUPy/t6yqt2/1Wx4YNPSe+mZjlyw9vKKI= diff --git a/metricbeat/module/system/diskio/_meta/data.json b/metricbeat/module/system/diskio/_meta/data.json index a2301ef8888..5d58850d187 100644 --- a/metricbeat/module/system/diskio/_meta/data.json +++ b/metricbeat/module/system/diskio/_meta/data.json @@ -15,7 +15,7 @@ "system": { "diskio": { "io": { - "time": 601740 + "time": 1296 }, "iostat": { "await": 0, @@ -48,16 +48,16 @@ } } }, - "name": "sdb1", + "name": "sda6", "read": { - "bytes": 25128030208, - "count": 3146154, - "time": 833872 + "bytes": 335872, + "count": 82, + "time": 1296 }, "write": { - "bytes": 34401640448, - "count": 861040, - "time": 11224168 + "bytes": 0, + "count": 0, + "time": 0 } } } diff --git a/metricbeat/module/system/diskio/diskio.go b/metricbeat/module/system/diskio/diskio.go index 4a7e2e2b5fe..793d2d23682 100644 --- a/metricbeat/module/system/diskio/diskio.go +++ b/metricbeat/module/system/diskio/diskio.go @@ -105,9 +105,6 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { "time": counters.WriteTime, "bytes": counters.WriteBytes, }, - "io": common.MapStr{ - "time": counters.IoTime, - }, } // accumulate values from all interfaces @@ -123,6 +120,10 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { event["iostat"] = iostat.AddLinuxIOStat(result) } + if runtime.GOOS != "windows" { + event.Put("io.time", counters.IoTime) + } + if counters.SerialNumber != "" { event["serial_number"] = counters.SerialNumber } diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py index 2e1b9e579d9..86b6dcd2d12 100644 --- a/metricbeat/module/system/test_system.py +++ b/metricbeat/module/system/test_system.py @@ -36,7 +36,7 @@ "softirq.norm.pct", "steal.norm.pct", "system.norm.pct", "user.norm.pct"] SYSTEM_DISKIO_FIELDS = ["name", "read.count", "write.count", "read.bytes", - "write.bytes", "read.time", "write.time", "io.time"] + "write.bytes", "read.time", "write.time"] SYSTEM_DISKIO_FIELDS_LINUX = ["name", "read.count", "write.count", "read.bytes", "write.bytes", "read.time", "write.time", "io.time", diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 80bfacbf2c3..c36845ab034 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -559,6 +559,21 @@ filebeat.modules: # "+02:00" for GMT+02:00 # var.tz_offset: local + umbrella: + enabled: true + + #var.input: s3 + # AWS SQS queue url + #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue + # Access ID to authenticate with the S3 input + #var.access_key_id: 123456 + # Access key to authenticate with the S3 input + #var.secret_access_key: PASSWORD + # The duration that the received messages are hidden from ReceiveMessage request + #var.visibility_timeout: 300s + # Maximum duration before AWS API request will be interrupted + #var.api_timeout: 120s + #--------------------------- Citrix NetScaler Module --------------------------- - module: citrix netscaler: diff --git a/x-pack/filebeat/input/s3/collector.go b/x-pack/filebeat/input/s3/collector.go index 9596b5ab23f..c3d3114c723 100644 --- a/x-pack/filebeat/input/s3/collector.go +++ b/x-pack/filebeat/input/s3/collector.go @@ -153,10 +153,8 @@ func (c *s3Collector) processorKeepAlive(svcSQS sqsiface.ClientAPI, message sqs. for { select { case <-c.cancellation.Done(): - fmt.Println("------- c.cancellation.Done()") return nil case err := <-errC: - fmt.Println("------- err = ", err) if err != nil { if err == context.DeadlineExceeded { c.logger.Info("Context deadline exceeded, updating visibility timeout") diff --git a/x-pack/filebeat/module/cisco/_meta/config.yml b/x-pack/filebeat/module/cisco/_meta/config.yml index 79f755c2c1f..c9c670fc095 100644 --- a/x-pack/filebeat/module/cisco/_meta/config.yml +++ b/x-pack/filebeat/module/cisco/_meta/config.yml @@ -89,3 +89,18 @@ # "local" (default) for system timezone. # "+02:00" for GMT+02:00 # var.tz_offset: local + + umbrella: + enabled: true + + #var.input: s3 + # AWS SQS queue url + #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue + # Access ID to authenticate with the S3 input + #var.access_key_id: 123456 + # Access key to authenticate with the S3 input + #var.secret_access_key: PASSWORD + # The duration that the received messages are hidden from ReceiveMessage request + #var.visibility_timeout: 300s + # Maximum duration before AWS API request will be interrupted + #var.api_timeout: 120s diff --git a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc index aaf285680e0..268d2eeea76 100644 --- a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc @@ -427,6 +427,15 @@ The ID for the access key used to read from the SQS queue. The secret token used for authenticating to the SQS queue. +*`var.visibility_timeout`*:: + +The duration that the received messages are hidden from ReceiveMessage request. +Default to be 300 seconds. + +*`var.api_timeout`*:: + +Maximum duration before AWS API request will be interrupted. Default to be 120 seconds. + :has-dashboards!: :fileset_ex!: diff --git a/x-pack/filebeat/module/cisco/umbrella/config/input.yml b/x-pack/filebeat/module/cisco/umbrella/config/input.yml index 8b0ccde6e2e..569392a26a9 100644 --- a/x-pack/filebeat/module/cisco/umbrella/config/input.yml +++ b/x-pack/filebeat/module/cisco/umbrella/config/input.yml @@ -4,6 +4,8 @@ type: s3 queue_url: {{ .queue_url }} access_key_id: {{ .access_key_id }} secret_access_key: {{ .secret_access_key }} +visibility_timeout: {{ .visibility_timeout }} +api_timeout: {{ .api_timeout }} {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/cisco/umbrella/manifest.yml b/x-pack/filebeat/module/cisco/umbrella/manifest.yml index 3a7150e714d..f85553d29eb 100644 --- a/x-pack/filebeat/module/cisco/umbrella/manifest.yml +++ b/x-pack/filebeat/module/cisco/umbrella/manifest.yml @@ -3,6 +3,10 @@ module_version: "1.0" var: - name: tags default: [cisco-umbrella, forwarded] + - name: visibility_timeout + default: 300 + - name: api_timeout + default: 120 ingest_pipeline: ingest/pipeline.yml input: config/input.yml diff --git a/x-pack/filebeat/modules.d/cisco.yml.disabled b/x-pack/filebeat/modules.d/cisco.yml.disabled index a2795fac8fc..2dc8389a8d2 100644 --- a/x-pack/filebeat/modules.d/cisco.yml.disabled +++ b/x-pack/filebeat/modules.d/cisco.yml.disabled @@ -92,3 +92,18 @@ # "local" (default) for system timezone. # "+02:00" for GMT+02:00 # var.tz_offset: local + + umbrella: + enabled: true + + #var.input: s3 + # AWS SQS queue url + #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue + # Access ID to authenticate with the S3 input + #var.access_key_id: 123456 + # Access key to authenticate with the S3 input + #var.secret_access_key: PASSWORD + # The duration that the received messages are hidden from ReceiveMessage request + #var.visibility_timeout: 300s + # Maximum duration before AWS API request will be interrupted + #var.api_timeout: 120s