7.x update beats + cherry-pick proxy fix (#3281)

* Update beats framework to 32f1a9e

* Update TLS config for elasticsearch client (#3278)

Follow up on elastic/beats#15516 to pass TLS
options to forward proxies.

NOTICE.txt

@@ -621,7 +621,7 @@ SOFTWARE.
 --------------------------------------------------------------------
 Dependency: github.com/elastic/beats
 Version: 7.x
-Revision: 8425cf098f305832dd7e3aea7d50b50a104fc6dd
+Revision: 32f1a9eb60b45a7157d9b9bcc7d626182baad633
 License type (autodetected): Apache-2.0
 ./vendor/github.com/elastic/beats/LICENSE.txt:
 --------------------------------------------------------------------
@@ -2519,7 +2519,7 @@ THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 --------------------------------------------------------------------
 Dependency: github.com/magefile/mage
-Revision: aedfce64c122eef47009b7f80c9771044753215d
+Revision: 1c36bf78a98209d91af71354deb001cca75e11fc
 License type (autodetected): Apache-2.0
 ./vendor/github.com/magefile/mage/LICENSE:
 --------------------------------------------------------------------

_beats/.go-version

@@ -1 +1 @@
-1.13.6
+1.13.7

_beats/CHANGELOG.asciidoc

@@ -13,6 +13,7 @@ https://github.com/elastic/beats/compare/v7.5.0...v7.5.1[View commits]
 
 - Fix `proxy_url` option in Elasticsearch output. {pull}14950[14950]
 - Fix bug with potential concurrent reads and writes from event.Meta map by Kafka output. {issue}14542[14542] {pull}14568[14568]
+- Fix license detection, when a beats successfully connect to Elasticsearch the detected license will be show in the log at info level. {pull}15834[15834]
 
 *Filebeat*
 
@@ -186,6 +187,7 @@ processing events. (CVE-2019-17596) See https://www.elastic.co/community/securit
 
 - Fill `event.provider`. {pull}13937[13937]
 - Add support for user management events to the Security module. {pull}13530[13530]
+- Made the event parser more lenient w.r.t. invalid event log definition version numbers. {issue}15838[15838]
 
 ==== Deprecated
 

_beats/CHANGELOG.next.asciidoc

@@ -20,6 +20,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Users can now specify `monitoring.cloud.*` to override `monitoring.elasticsearch.*` settings. {issue}14399[14399] {pull}15254[15254]
 - Refactor metadata generator to support adding metadata across resources {pull}14875[14875]
 - Update to ECS 1.4.0. {pull}14844[14844]
+- The document id fields has been renamed from @metadata.id to @metadata._id {pull}15859[15859]
+
 
 *Auditbeat*
 
@@ -58,11 +60,19 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix a race condition with the Kafka pipeline client, it is possible that `Close()` get called before `Connect()` . {issue}11945[11945]
 - Allow users to configure only `cluster_uuid` setting under `monitoring` namespace. {pull}14338[14338]
 - Fix spooling to disk blocking infinitely if the lock file can not be acquired. {pull}15338[15338]
+- Update replicaset group to apps/v1 {pull}15854[15802]
 - Fix `metricbeat test output` with an ipv6 ES host in the output.hosts. {pull}15368[15368]
 - Fix `convert` processor conversion of string to integer with leading zeros. {issue}15513[15513] {pull}15557[15557]
+- Fix panic in the Logstash output when trying to send events to closed connection. {pull}15568[15568]
+- Fix missing output in dockerlogbeat {pull}15719[15719]
+- Fix logging target settings being ignored when Beats are started via systemd or docker. {issue}12024[12024] {pull}15422[15442]
+- Do not load dashboards where not available. {pull}15802[15802]
+- Fix issue where default go logger is not discarded when either * or stdout is selected. {issue}10251[10251] {pull}15708[15708]
+- Fix issue where TLS settings would be ignored when a forward proxy was in use. {pull}15516{15516}
 
 *Auditbeat*
 
+- system/socket: Fixed compatibility issue with kernel 5.x. {pull}15771[15771]
 
 *Filebeat*
 
@@ -77,10 +87,14 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - netflow: Fix bytes/packets counters on some devices (NSEL and Netstream). {pull}15449[15449]
 - netflow: Fix compatibility with some Cisco devices by changing the field `class_id` from short to long. {pull}15449[15449]
 - Fixed dashboard for Cisco ASA Firewall. {issue}15420[15420] {pull}15553[15553]
+- Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656]
+- Fix typos in zeek notice fileset config file. {issue}15764[15764] {pull}15765[15765]
+- Fix mapping error when zeek weird logs do not contain IP addresses. {pull}15906[15906]
 
 *Heartbeat*
 
 - Fix recording of SSL cert metadata for Expired/Unvalidated x509 certs. {pull}13687[13687]
+- Fixed excessive memory usage introduced in 7.5 due to over-allocating memory for HTTP checks. {pull}15639[15639]
 
 *Journalbeat*
 
@@ -98,6 +112,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Make `logstash` module more resilient to Logstash unavailability. {issue}15276[15276] {pull}15306[15306]
 - Add username/password in Metricbeat autodiscover hints {pull}15349[15349]
 - Fix CPU count in docker/cpu in cases where no `online_cpus` are reported {pull}15070[15070]
+- Add dedot for tags in ec2 metricset and cloudwatch metricset. {issue}15843[15843] {pull}15844[15844]
+- Use RFC3339 format for timestamps collected using the SQL module. {pull}15847[15847]
+- Avoid parsing errors returned from prometheus endpoints. {pull}15712[15712]
+- Add dedot for cloudwatch metric name. {issue}15916[15916] {pull}15917[15917]
 
 *Packetbeat*
 
@@ -130,11 +148,14 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Spooling to disk creates a lockfile on each platform. {pull}15338[15338]
 - Fingerprint processor adds a new xxhash hashing algorithm {pull}15418[15418]
 - Enable DEP (Data Execution Protection) for Windows packages. {pull}15149[15149]
+- Add document_id setting to decode_json_fields processor. {pull}15859[15859]
+
 
 *Auditbeat*
 
 
 *Filebeat*
+- Add dashboard for AWS ELB fileset. {pull}15804[15804]
 
 - `container` and `docker` inputs now support reading of labels and env vars written by docker JSON file logging driver. {issue}8358[8358]
 - Add `index` option to all inputs to directly set a per-input index value. {pull}14010[14010]
@@ -148,22 +169,44 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Integrate the azure-eventhub with filebeat azure module (replace the kafka input). {pull}15480[15480]
 - Release aws s3access fileset to GA. {pull}15431[15431] {issue}15430[15430]
 - Add cloudtrail fileset to AWS module. {issue}14657[14657] {pull}15227[15227]
+- New fileset googlecloud/firewall for ingesting Google Cloud Firewall logs. {pull}14553[14553]
+- google-pubsub input: ACK pub/sub message when acknowledged by publisher. {issue}13346[13346] {pull}14715[14715]
+- Remove Beta label from google-pubsub input. {issue}13346[13346] {pull}14715[14715]
+- Set event.outcome field based on googlecloud audit log output. {pull}15731[15731]
+- Add dashboard for AWS vpcflow fileset. {pull}16007[16007]
 
 *Heartbeat*
 
+- Allow a list of status codes for HTTP checks. {pull}15587[15587]
+
+
+*Heartbeat*
 
 *Journalbeat*
 
 *Metricbeat*
 
-
+- Add lambda metricset in aws module. {pull}15260[15260]
 - Expand data for the `system/memory` metricset {pull}15492[15492]
 - Add azure `storage` metricset in order to retrieve metric values for storage accounts. {issue}14548[14548] {pull}15342[15342]
 - Add cost warnings for the azure module. {pull}15356[15356]
 - Add DynamoDB AWS Metricbeat light module {pull}15097[15097]
 - Release elb module as GA. {pull}15485[15485]
 - Add a `system/network_summary` metricset {pull}15196[15196]
 - Add IBM MQ light-weight Metricbeat module {pull}15301[15301]
+- Enable script processor. {pull}14711[14711]
+- Add mixer metricset for Istio Metricbeat module {pull}15696[15696]
+- Add mesh metricset for Istio Metricbeat module{pull}15535[15535]
+- Add pilot metricset for Istio Metricbeat module {pull}15761[15761]
+- Add galley metricset for Istio Metricbeat module {pull}15857[15857]
+- Add STAN dashboard {pull}15654[15654]
+- Add `key/value` mode for SQL module. {issue}15770[15770] {pull]15845[15845]
+- Add support for Unix socket in Memcached metricbeat module. {issue}13685[13685] {pull}15822[15822]
+- Make the `system/cpu` metricset collect normalized CPU metrics by default. {issue}15618[15618] {pull}15729[15729]
+- Add `up` metric to prometheus metrics collected from host {pull}15948[15948]
+- Add citadel metricset for Istio Metricbeat module {pull}15990[15990]
+- Add support for processors in light modules. {issue}14740[14740] {pull}15923[15923]
+- Reuse connections in SQL module. {pull}16001[16001]
 
 *Packetbeat*
 

_beats/dev-tools/cherrypick_pr

@@ -62,6 +62,12 @@ def main():
                         "(requires token in ~/.elastic/github.token)")
     parser.add_argument("--diff", action="store_true",
                         help="Display the diff before pushing the PR")
+    parser.add_argument("--remote", default="",
+                        help="Which remote to push the backport branch to")
+    parser.add_argument("--zube-team", default="",
+                        help="Team the PR belongs to")
+    parser.add_argument("--keep-backport-label", action="store_true",
+                        help="Preserve label needs_backport in original PR")
     args = parser.parse_args()
 
     print(args)
@@ -109,7 +115,10 @@ def main():
             return 1
 
     print("Ready to push branch.")
-    remote = raw_input("To which remote should I push? (your fork): ")
+
+    remote = args.remote
+    if not remote:
+        remote = raw_input("To which remote should I push? (your fork): ")
     call("git push {} :{} > /dev/null".format(remote, tmp_branch),
          shell=True)
     check_call("git push --set-upstream {} {}"
@@ -145,11 +154,24 @@ def main():
         new_pr = request.json()
 
         # add labels
+        labels = ["backport"]
+
+        if args.zube_team:
+            resp = session.get(base + "/labels/Team:"+args.zube_team)
+            if resp.status_code != 200:
+                print("Cannot find team label", resp.text)
+                sys.exit(1)
+            labels.append("Team:"+args.zube_team)
+            labels.append("[zube]: In Review")
+        else:
+            labels.append("review")
+
         session.post(
-            base + "/issues/{}/labels".format(new_pr["number"]), json=["backport", "review"])
+            base + "/issues/{}/labels".format(new_pr["number"]), json=labels)
 
-        # remove needs backport label from the original PR
-        session.delete(base + "/issues/{}/labels/needs_backport".format(args.pr_number))
+        if not args.keep_backport_label:
+            # remove needs backport label from the original PR
+            session.delete(base + "/issues/{}/labels/needs_backport".format(args.pr_number))
 
         # get version and set a version label on the original PR
         version = get_version(os.getcwd())

_beats/dev-tools/make/mage.mk

@@ -1,4 +1,4 @@
-MAGE_VERSION     ?= v1.8.0
+MAGE_VERSION     ?= v1.9.0
 MAGE_PRESENT     := $(shell mage --version 2> /dev/null | grep $(MAGE_VERSION))
 MAGE_IMPORT_PATH ?= github.com/elastic/beats/vendor/github.com/magefile/mage
 export MAGE_IMPORT_PATH

_beats/dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl

@@ -2,23 +2,25 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
 <dict>
-	<key>Label</key>
+    <key>Label</key>
         <string>{{.identifier}}</string>
-	<key>ProgramArguments</key>
-	<array>
-            <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}</string>
-            <string>-c</string>
-            <string>/etc/{{.BeatName}}/{{.BeatName}}.yml</string>
-            <string>--path.home</string>
-            <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}</string>
-            <string>--path.config</string>
-            <string>/etc/{{.BeatName}}</string>
-            <string>--path.data</string>
-            <string>/var/lib/{{.BeatName}}</string>
-            <string>--path.logs</string>
-            <string>/var/log/{{.BeatName}}</string>
-        </array>
-	<key>RunAtLoad</key>
-        <true/>
+    <key>ProgramArguments</key>
+    <array>
+        <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}</string>
+        <string>-environment</string>
+        <string>macOS_service</string>
+        <string>-c</string>
+        <string>/etc/{{.BeatName}}/{{.BeatName}}.yml</string>
+        <string>--path.home</string>
+        <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}</string>
+        <string>--path.config</string>
+        <string>/etc/{{.BeatName}}</string>
+        <string>--path.data</string>
+        <string>/var/lib/{{.BeatName}}</string>
+        <string>--path.logs</string>
+        <string>/var/log/{{.BeatName}}</string>
+    </array>
+    <key>RunAtLoad</key>
+    <true/>
 </dict>
 </plist>

_beats/dev-tools/packaging/templates/docker/Dockerfile.tmpl

@@ -50,4 +50,4 @@ EXPOSE {{ $port }}
 
 WORKDIR {{ $beatHome }}
 ENTRYPOINT ["/usr/local/bin/docker-entrypoint"]
-CMD ["-e"]
+CMD ["-environment", "container"]

_beats/dev-tools/packaging/templates/linux/systemd.unit.tmpl

@@ -9,10 +9,10 @@ After=network-online.target
 User={{ .BeatUser }}
 Group={{ .BeatUser }}
 {{- end }}
-Environment="BEAT_LOG_OPTS=-e"
+Environment="BEAT_LOG_OPTS="
 Environment="BEAT_CONFIG_OPTS=-c /etc/{{.BeatName}}/{{.BeatName}}.yml"
 Environment="BEAT_PATH_OPTS=-path.home /usr/share/{{.BeatName}} -path.config /etc/{{.BeatName}} -path.data /var/lib/{{.BeatName}} -path.logs /var/log/{{.BeatName}}"
-ExecStart=/usr/share/{{.BeatName}}/bin/{{.BeatName}} $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS
+ExecStart=/usr/share/{{.BeatName}}/bin/{{.BeatName}} -environment systemd $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS
 Restart=always
 
 [Install]

_beats/dev-tools/packaging/templates/windows/install-service.ps1.tmpl

@@ -11,7 +11,7 @@ $workdir = Split-Path $MyInvocation.MyCommand.Path
 # Create the new service.
 New-Service -name {{.BeatName}} `
   -displayName {{.BeatName | title}} `
-  -binaryPathName "`"$workdir\{{.BeatName}}.exe`" -c `"$workdir\{{.BeatName}}.yml`" -path.home `"$workdir`" -path.data `"C:\ProgramData\{{.BeatName}}`" -path.logs `"C:\ProgramData\{{.BeatName}}\logs`" -E logging.files.redirect_stderr=true"
+  -binaryPathName "`"$workdir\{{.BeatName}}.exe`" -environment=windows_service -c `"$workdir\{{.BeatName}}.yml`" -path.home `"$workdir`" -path.data `"C:\ProgramData\{{.BeatName}}`" -path.logs `"C:\ProgramData\{{.BeatName}}\logs`" -E logging.files.redirect_stderr=true"
 
 # Attempt to set the service to delayed start using sc config.
 Try {

_beats/libbeat/docs/version.asciidoc

@@ -1,6 +1,6 @@
 :stack-version: 7.7.0
 :doc-branch: 7.x
-:go-version: 1.13.6
+:go-version: 1.13.7
 :release-state: unreleased
 :python: 2.7.9
 :docker: 1.12

_beats/libbeat/scripts/Makefile

@@ -197,7 +197,7 @@ unit: ## @testing Runs the unit tests without coverage reports.
 integration-tests: ## @testing Run integration tests. Unit tests are run as part of the integration tests.
 integration-tests: prepare-tests mage
 	rm -f docker-compose.yml.lock
-	mage goIntegTest
+	$(COVERAGE_TOOL) -tags=integration $(RACE) -coverprofile=${COVERAGE_DIR}/integration.cov ${GOPACKAGES}
 
 .PHONY: integration-tests-environment
 integration-tests-environment:  ## @testing Runs the integration inside a virtual environment. This can be run on any docker-machine (local, remote)
@@ -207,7 +207,12 @@ integration-tests-environment: prepare-tests build-image
 	#
 	# This will make docker-compose command to display the logs on stdout on error, It's not enabled
 	# by default because it can create noise if the test inside the container fails.
-	${DOCKER_COMPOSE} run beat make integration-tests RACE_DETECTOR=$(RACE_DETECTOR) DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME}
+	${DOCKER_COMPOSE} run \
+	  -e RACE_DETECTOR=$(RACE_DETECTOR) \
+	  -e DOCKER_COMPOSE_PROJECT_NAME=${DOCKER_COMPOSE_PROJECT_NAME} \
+	  -e TEST_ENVIRONMENT=${TEST_ENVIRONMENT} \
+	  -e BEATS_DOCKER_INTEGRATION_TEST_ENV=${BEATS_DOCKER_INTEGRATION_TEST_ENV} \
+	  beat make integration-tests
 
 # Runs the system tests
 .PHONY: system-tests

_beats/testing/environments/latest.yml

@@ -3,7 +3,7 @@
 version: '2.3'
 services:
   elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.4.0
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.5.2
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9200"]
       retries: 300
@@ -16,7 +16,7 @@ services:
       - "xpack.security.enabled=false"
 
   logstash:
-    image: docker.elastic.co/logstash/logstash:7.4.0
+    image: docker.elastic.co/logstash/logstash:7.5.2
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9600/_node/stats"]
       retries: 300
@@ -26,7 +26,7 @@ services:
       - ./docker/logstash/pki:/etc/pki:ro
 
   kibana:
-    image: docker.elastic.co/kibana/kibana:7.4.0
+    image: docker.elastic.co/kibana/kibana:7.5.2
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:5601"]
       retries: 300