Merge remote-tracking branch 'upstream/master' into feature/windows-7-32

* upstream/master: adding timeout options to cisco umbrella and fixing the missing config options in module config.yml (elastic#22295) [Metricbeat] Remove io.time from windows (elastic#22237) remove debug messages (elastic#22282) dependency upgrade: ecszap v0.3.0 (elastic#22267) [repo] Add docs + cleanup labels to the categories in the incoming PR template (elastic#22280)
v1v · Oct 30, 2020 · fb7e03d · fb7e03d
2 parents 546a12e + b18b333
commit fb7e03d
Show file tree

Hide file tree

Showing 17 changed files with 91 additions and 19 deletions.
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
@@ -4,6 +4,8 @@ Please label this PR with one of the following labels, depending on the scope of
 - Enhancement
 - Breaking change
 - Deprecation
+- Cleanup
+- Docs
 -->
 
 ## What does this PR do?

diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
@@ -102,3 +102,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Go version to 1.14.7. {pull}20508[20508]
 - Add packaging for docker image based on UBI minimal 8. {pull}20576[20576]
 - Make the mage binary used by the build process in the docker container to be statically compiled. {pull}20827[20827]
+- Update ecszap to v0.3.0 for using ECS 1.6.0 in logs {pull}22267[22267]
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -385,6 +385,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - [Kubernetes] Remove redundant dockersock volume mount {pull}22009[22009]
 - Revert change to report `process.memory.rss` as `process.memory.wss` on Windows. {pull}22055[22055]
 - Add a switch to the driver definition on SQL module to use pretty names {pull}17378[17378]
+- Remove io.time from windows {pull}22237[22237]
 - Add interval information to `monitor` metricset in azure. {pull}22152[22152]
 
 *Packetbeat*

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -14853,11 +14853,11 @@ Contents of probable licence file $GOMODCACHE/go.elastic.co/apm/module/apmhttp@v
 
 --------------------------------------------------------------------------------
 Dependency : go.elastic.co/ecszap
-Version: v0.1.1-0.20200424093508-cdd95a104193
+Version: v0.3.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/go.elastic.co/ecszap@v0.1.1-0.20200424093508-cdd95a104193/LICENSE:
+Contents of probable licence file $GOMODCACHE/go.elastic.co/ecszap@v0.3.0/LICENSE:
 
 
                                  Apache License

diff --git a/filebeat/docs/modules/cisco.asciidoc b/filebeat/docs/modules/cisco.asciidoc
@@ -432,6 +432,15 @@ The ID for the access key used to read from the SQS queue.
 
 The secret token used for authenticating to the SQS queue.
 
+*`var.visibility_timeout`*::
+
+The duration that the received messages are hidden from ReceiveMessage request.
+Default to be 300 seconds.
+
+*`var.api_timeout`*::
+
+Maximum duration before AWS API request will be interrupted. Default to be 120 seconds.
+
 :has-dashboards!:
 
 :fileset_ex!:

diff --git a/go.mod b/go.mod
@@ -157,7 +157,7 @@ require (
 	go.elastic.co/apm v1.8.1-0.20200909061013-2aef45b9cf4b
 	go.elastic.co/apm/module/apmelasticsearch v1.7.2
 	go.elastic.co/apm/module/apmhttp v1.7.2
-	go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193
+	go.elastic.co/ecszap v0.3.0
 	go.elastic.co/go-licence-detector v0.4.0
 	go.etcd.io/bbolt v1.3.4
 	go.uber.org/atomic v1.5.0

diff --git a/go.sum b/go.sum
@@ -740,8 +740,8 @@ go.elastic.co/apm/module/apmelasticsearch v1.7.2 h1:5STGHLZLSeAzxordMc+dFVKiyVtM
 go.elastic.co/apm/module/apmelasticsearch v1.7.2/go.mod h1:ZyNFuyWdt42GBZkz0SogoLzDBrBGj4orxpiUuxYeYq8=
 go.elastic.co/apm/module/apmhttp v1.7.2 h1:2mRh7SwBuEVLmJlX+hsMdcSg9xaielCLElaPn/+i34w=
 go.elastic.co/apm/module/apmhttp v1.7.2/go.mod h1:sTFWiWejnhSdZv6+dMgxGec2Nxe/ZKfHfz/xtRM+cRY=
-go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193 h1:NjYJ/beChqugXSavTkH5tF6shvr/is8jdgJ331wfwT8=
-go.elastic.co/ecszap v0.1.1-0.20200424093508-cdd95a104193/go.mod h1:HTUi+QRmr3EuZMqxPX+5fyOdMNfUu5iPebgfhgsTJYQ=
+go.elastic.co/ecszap v0.3.0 h1:Zo/Y4sJLqbWDlqCHI4F4Lzeg0Fs4+n5ldVis4h9xV8w=
+go.elastic.co/ecszap v0.3.0/go.mod h1:HTUi+QRmr3EuZMqxPX+5fyOdMNfUu5iPebgfhgsTJYQ=
 go.elastic.co/fastjson v1.0.0/go.mod h1:PmeUOMMtLHQr9ZS9J9owrAVg0FkaZDRZJEFTTGHtchs=
 go.elastic.co/fastjson v1.1.0 h1:3MrGBWWVIxe/xvsbpghtkFoPciPhOCmjsR/HfwEeQR4=
 go.elastic.co/fastjson v1.1.0/go.mod h1:boNGISWMjQsUPy/t6yqt2/1Wx4YNPSe+mZjlyw9vKKI=

diff --git a/metricbeat/module/system/diskio/_meta/data.json b/metricbeat/module/system/diskio/_meta/data.json
@@ -15,7 +15,7 @@
     "system": {
         "diskio": {
             "io": {
-                "time": 601740
+                "time": 1296
             },
             "iostat": {
                 "await": 0,
@@ -48,16 +48,16 @@
                     }
                 }
             },
-            "name": "sdb1",
+            "name": "sda6",
             "read": {
-                "bytes": 25128030208,
-                "count": 3146154,
-                "time": 833872
+                "bytes": 335872,
+                "count": 82,
+                "time": 1296
             },
             "write": {
-                "bytes": 34401640448,
-                "count": 861040,
-                "time": 11224168
+                "bytes": 0,
+                "count": 0,
+                "time": 0
             }
         }
     }

diff --git a/metricbeat/module/system/diskio/diskio.go b/metricbeat/module/system/diskio/diskio.go
@@ -105,9 +105,6 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
 				"time":  counters.WriteTime,
 				"bytes": counters.WriteBytes,
 			},
-			"io": common.MapStr{
-				"time": counters.IoTime,
-			},
 		}
 
 		// accumulate values from all interfaces
@@ -123,6 +120,10 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error {
 			event["iostat"] = iostat.AddLinuxIOStat(result)
 		}
 
+		if runtime.GOOS != "windows" {
+			event.Put("io.time", counters.IoTime)
+		}
+
 		if counters.SerialNumber != "" {
 			event["serial_number"] = counters.SerialNumber
 		}

diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py
@@ -36,7 +36,7 @@
                                                "softirq.norm.pct", "steal.norm.pct", "system.norm.pct", "user.norm.pct"]
 
 SYSTEM_DISKIO_FIELDS = ["name", "read.count", "write.count", "read.bytes",
-                        "write.bytes", "read.time", "write.time", "io.time"]
+                        "write.bytes", "read.time", "write.time"]
 
 SYSTEM_DISKIO_FIELDS_LINUX = ["name", "read.count", "write.count", "read.bytes",
                               "write.bytes", "read.time", "write.time", "io.time",

diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
@@ -559,6 +559,21 @@ filebeat.modules:
     # "+02:00" for GMT+02:00
     # var.tz_offset: local
 
+  umbrella:
+    enabled: true
+
+    #var.input: s3
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue
+    # Access ID to authenticate with the S3 input
+    #var.access_key_id: 123456
+    # Access key to authenticate with the S3 input
+    #var.secret_access_key: PASSWORD
+    # The duration that the received messages are hidden from ReceiveMessage request
+    #var.visibility_timeout: 300s
+    # Maximum duration before AWS API request will be interrupted
+    #var.api_timeout: 120s
+
 #--------------------------- Citrix NetScaler Module ---------------------------
 - module: citrix
   netscaler:

diff --git a/x-pack/filebeat/input/s3/collector.go b/x-pack/filebeat/input/s3/collector.go
@@ -153,10 +153,8 @@ func (c *s3Collector) processorKeepAlive(svcSQS sqsiface.ClientAPI, message sqs.
 	for {
 		select {
 		case <-c.cancellation.Done():
-			fmt.Println("------- c.cancellation.Done()")
 			return nil
 		case err := <-errC:
-			fmt.Println("------- err = ", err)
 			if err != nil {
 				if err == context.DeadlineExceeded {
 					c.logger.Info("Context deadline exceeded, updating visibility timeout")

diff --git a/x-pack/filebeat/module/cisco/_meta/config.yml b/x-pack/filebeat/module/cisco/_meta/config.yml
@@ -89,3 +89,18 @@
     # "local" (default) for system timezone.
     # "+02:00" for GMT+02:00
     # var.tz_offset: local
+
+  umbrella:
+    enabled: true
+
+    #var.input: s3
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue
+    # Access ID to authenticate with the S3 input
+    #var.access_key_id: 123456
+    # Access key to authenticate with the S3 input
+    #var.secret_access_key: PASSWORD
+    # The duration that the received messages are hidden from ReceiveMessage request
+    #var.visibility_timeout: 300s
+    # Maximum duration before AWS API request will be interrupted
+    #var.api_timeout: 120s
diff --git a/x-pack/filebeat/module/cisco/_meta/docs.asciidoc b/x-pack/filebeat/module/cisco/_meta/docs.asciidoc
@@ -427,6 +427,15 @@ The ID for the access key used to read from the SQS queue.
 
 The secret token used for authenticating to the SQS queue.
 
+*`var.visibility_timeout`*::
+
+The duration that the received messages are hidden from ReceiveMessage request.
+Default to be 300 seconds.
+
+*`var.api_timeout`*::
+
+Maximum duration before AWS API request will be interrupted. Default to be 120 seconds.
+
 :has-dashboards!:
 
 :fileset_ex!:

diff --git a/x-pack/filebeat/module/cisco/umbrella/config/input.yml b/x-pack/filebeat/module/cisco/umbrella/config/input.yml
@@ -4,6 +4,8 @@ type: s3
 queue_url: {{ .queue_url }}
 access_key_id: {{ .access_key_id }}
 secret_access_key: {{ .secret_access_key }}
+visibility_timeout: {{ .visibility_timeout }}
+api_timeout: {{ .api_timeout }}
 
 {{ else if eq .input "file" }}
 

diff --git a/x-pack/filebeat/module/cisco/umbrella/manifest.yml b/x-pack/filebeat/module/cisco/umbrella/manifest.yml
@@ -3,6 +3,10 @@ module_version: "1.0"
 var:
   - name: tags
     default: [cisco-umbrella, forwarded]
+  - name: visibility_timeout
+    default: 300
+  - name: api_timeout
+    default: 120
 
 ingest_pipeline: ingest/pipeline.yml
 input: config/input.yml
diff --git a/x-pack/filebeat/modules.d/cisco.yml.disabled b/x-pack/filebeat/modules.d/cisco.yml.disabled
@@ -92,3 +92,18 @@
     # "local" (default) for system timezone.
     # "+02:00" for GMT+02:00
     # var.tz_offset: local
+
+  umbrella:
+    enabled: true
+
+    #var.input: s3
+    # AWS SQS queue url
+    #var.queue_url: https://sqs.us-east-1.amazonaws.com/ID/CiscoQueue
+    # Access ID to authenticate with the S3 input
+    #var.access_key_id: 123456
+    # Access key to authenticate with the S3 input
+    #var.secret_access_key: PASSWORD
+    # The duration that the received messages are hidden from ReceiveMessage request
+    #var.visibility_timeout: 300s
+    # Maximum duration before AWS API request will be interrupted
+    #var.api_timeout: 120s