Merge branch 'master' into doc-decode_cef

.ci/packaging.groovy

@@ -23,6 +23,7 @@ pipeline {
   }
   triggers {
     issueCommentTrigger('(?i)^\\/packaging$')
+    upstream('Beats/beats-beats-mbp/master')
   }
   parameters {
     booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')

.travis.yml

@@ -111,23 +111,22 @@ jobs:
     # Metricbeat
     - os: linux
       before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
-      env: TARGETS="-C metricbeat unit-tests coverage-report"
+      env: TARGETS="-C metricbeat unit-tests"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
-      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
-      env: TARGETS="-C metricbeat integration-tests-environment coverage-report"
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C metricbeat integration-tests"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
-      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat vendor || travis_terminate 0
-      env: TARGETS="-C metricbeat update system-tests-environment coverage-report"
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C metricbeat system-tests"
       go: $TRAVIS_GO_VERSION
       stage: test
-
     - os: osx
       before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
-      env: TARGETS="TEST_ENVIRONMENT=0 -C metricbeat testsuite"
+      env: TARGETS="-C metricbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test
     - os: linux
@@ -137,6 +136,21 @@ jobs:
       stage: test
     - os: linux
       before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat unit-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat integration-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: linux
+      before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0
+      env: TARGETS="-C x-pack/metricbeat system-tests"
+      go: $TRAVIS_GO_VERSION
+      stage: test
+    - os: osx
+      before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0
       env: TARGETS="-C x-pack/metricbeat testsuite"
       go: $TRAVIS_GO_VERSION
       stage: test

CHANGELOG.next.asciidoc

@@ -114,6 +114,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove migrationVersion map 7.7.0 reference from Kibana dashboard file to fix backward compatibility issues. {pull}17425[17425]
 - Fix issue 17734 to retry on rate-limit error in the Filebeat httpjson input. {issue}17734[17734] {pull}17735[17735]
 - Fixed `cloudfoundry.access` to have the correct `cloudfoundry.app.id` contents. {pull}17847[17847]
+- Fixed typo in log message. {pull}17897[17897]
+- Fix Cisco ASA ASA 3020** and 106023 messages {pull}17964[17964]
 
 *Heartbeat*
 
@@ -273,6 +275,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve ECS categorization field mappings in postgresql module. {issue}16177[16177] {pull}17914[17914]
 - Improve ECS categorization field mappings in rabbitmq module. {issue}16178[16178] {pull}17916[17916]
 - Make `decode_cef` processor GA. {pull}17944[17944]
+- Improve ECS categorization field mappings in redis module. {issue}16179[16179] {pull}17918[17918]
 
 *Heartbeat*
 
@@ -346,6 +349,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Reference kubernetes manifests mount data directory from the host when running metricbeat as daemonset, so data persist between executions in the same node. {pull}17429[17429]
 - Add more detailed error messages, system tests and small refactoring to the service metricset in windows. {pull}17725[17725]
 - Stack Monitoring modules now auto-configure required metricsets when `xpack.enabled: true` is set. {issue}16471[[16471] {pull}17609[17609]
+- Add aggregation aligner as a config parameter for googlecloud stackdriver metricset. {issue}17141[[17141] {pull}17719[17719]
 - Move the perfmon metricset to GA. {issue}16608[16608] {pull}17879[17879]
 
 *Packetbeat*

Jenkinsfile

@@ -319,7 +319,20 @@ pipeline {
             }
           }
           steps {
-            mageTarget("Metricbeat OSS linux/amd64 (integTest)", "metricbeat", "integTest")
+            mageTarget("Metricbeat OSS linux/amd64 (goIntegTest)", "metricbeat", "goIntegTest")
+          }
+        }
+        stage('Metricbeat Python integration tests'){
+          agent { label 'ubuntu && immutable' }
+          options { skipDefaultCheckout() }
+          when {
+            beforeAgent true
+            expression {
+              return env.BUILD_METRICBEAT != "false"
+            }
+          }
+          steps {
+            mageTarget("Metricbeat OSS linux/amd64 (pythonIntegTest)", "metricbeat", "pythonIntegTest")
           }
         }
         stage('Metricbeat x-pack'){

Makefile

@@ -28,7 +28,7 @@ PROJECTS_XPACK_MAGE=$(PROJECTS_XPACK_PKG) x-pack/libbeat
 #
 # Includes
 #
-include dev-tools/make/mage.mk
+include dev-tools/make/mage-install.mk
 
 ## help : Show this help.
 help: Makefile

dev-tools/make/mage-install.mk

@@ -0,0 +1,13 @@
+MAGE_VERSION     ?= v1.9.0
+MAGE_PRESENT     := $(shell mage --version 2> /dev/null | grep $(MAGE_VERSION))
+MAGE_IMPORT_PATH ?= github.com/magefile/mage
+export MAGE_IMPORT_PATH
+
+.PHONY: mage
+mage:
+ifndef MAGE_PRESENT
+	@echo Installing mage $(MAGE_VERSION) from vendor dir.
+	@go install -mod=vendor -ldflags="-X $(MAGE_IMPORT_PATH)/mage.gitTag=$(MAGE_VERSION)" ${MAGE_IMPORT_PATH}/...
+	@-mage -clean
+endif
+	@true

dev-tools/make/mage.mk

@@ -1,13 +1,69 @@
-MAGE_VERSION     ?= v1.9.0
-MAGE_PRESENT     := $(shell mage --version 2> /dev/null | grep $(MAGE_VERSION))
-MAGE_IMPORT_PATH ?= github.com/magefile/mage
-export MAGE_IMPORT_PATH
-
-.PHONY: mage
-mage:
-ifndef MAGE_PRESENT
-	@echo Installing mage $(MAGE_VERSION) from vendor dir.
-	@go install -mod=vendor -ldflags="-X $(MAGE_IMPORT_PATH)/mage.gitTag=$(MAGE_VERSION)" ${MAGE_IMPORT_PATH}/...
-	@-mage -clean
-endif
-	@true
+# This is a minimal Makefile for Beats that are built with Mage. Its only
+# responsibility is to provide compatibility with existing Jenkins and Travis
+# setups.
+
+#
+# Variables
+#
+.DEFAULT_GOAL := help
+PWD           := $(CURDIR)
+
+#
+# Includes
+#
+include $(ES_BEATS)/dev-tools/make/mage-install.mk
+
+#
+# Targets (alphabetically sorted).
+#
+.PHONY: check
+check: mage
+	mage check
+
+.PHONY: clean
+clean: mage
+	mage clean
+
+fix-permissions:
+
+.PHONY: fmt
+fmt: mage
+	mage fmt
+
+# Default target.
+.PHONY: help
+help:
+	@echo Use mage rather than make. Here are the available mage targets:
+	@mage -l
+
+.PHONY: release
+release: mage
+	mage package
+
+stop-environment:
+
+.PHONY: unit-tests
+unit-tests: mage
+	mage unitTest
+
+.PHONY: integration-tests
+integration-tests: mage
+	rm -f build/TEST-go-integration.out
+	mage goIntegTest || ( cat build/TEST-go-integration.out && false )
+
+.PHONY: system-tests
+system-tests: mage
+	mage pythonIntegTest
+
+.PHONY: testsuite
+testsuite: mage
+	rm -f build/TEST-go-integration.out
+	mage update build unitTest integTest || ( cat build/TEST-go-integration.out && false )
+
+.PHONY: update
+update: mage
+	mage update
+
+.PHONY: crosscompile
+crosscompile: mage
+	mage crossBuild

filebeat/docs/fields.asciidoc

@@ -28240,6 +28240,15 @@ type: keyword
 Palo Alto Networks name for the threat.
 
 
+type: keyword
+
+--
+
+*`panw.panos.action`*::
++
+--
+Action taken for the session.
+
 type: keyword
 
 --

filebeat/docs/running-on-cloudfoundry.asciidoc

@@ -1,5 +1,5 @@
 [[running-on-cloudfoundry]]
-=== Running {beatname_uc} on Cloud Foundry
+=== Run {beatname_uc} on Cloud Foundry
 
 You can use {beatname_uc} on Cloud Foundry to retrieve and ship logs.
 
@@ -14,18 +14,19 @@ endif::[]
 [float]
 ==== Cloud Foundry credentials
 
-{beatname_uc} needs credentials created with UAA so it can connect to loggregator to receive the logs. The uaac
+{beatname_uc} needs credentials created with UAA so it can connect to loggregator to receive the logs. The `uaac`
 command will create the required credentials for connecting to loggregator.
 
-["source", "sh"]
+["source","sh",subs="attributes"]
 ------------------------------------------------
 uaac client add {beatname_lc} --name {beatname_lc} --secret changeme --authorized_grant_types client_credentials,refresh_token --authorities doppler.firehose,cloud_controller.admin_read_only
 ------------------------------------------------
 
 [WARNING]
 =======================================
-*Use a unique secret:* The uaac command above is just an example and the secret should be changed and the
-`{beatname_lc}.yml` should be updated with your choosen secret.
+*Use a unique secret:* The `uaac` command shown here is an example. Remember to
+replace `changeme` with your secret, and update the +{beatname_lc}.yml+ file to
+use your chosen secret.
 =======================================
 
 

filebeat/docs/running-on-kubernetes.asciidoc

@@ -1,5 +1,5 @@
 [[running-on-kubernetes]]
-=== Running {beatname_uc} on Kubernetes
+=== Run {beatname_uc} on Kubernetes
 
 You can use {beatname_uc} <<running-on-docker,Docker images>> on Kubernetes to
 retrieve and ship container logs.

filebeat/fileset/fileset_test.go

@@ -56,7 +56,7 @@ func TestLoadManifestNginx(t *testing.T) {
 	manifest, err := fs.readManifest()
 	assert.NoError(t, err)
 	assert.Equal(t, manifest.ModuleVersion, "1.0")
-	assert.Equal(t, manifest.IngestPipeline, []string{"ingest/default.json"})
+	assert.Equal(t, manifest.IngestPipeline, []string{"ingest/pipeline.yml"})
 	assert.Equal(t, manifest.Input, "config/nginx-access.yml")
 
 	vars := manifest.Vars
@@ -189,7 +189,7 @@ func TestGetInputConfigNginx(t *testing.T) {
 	assert.True(t, cfg.HasField("pipeline"))
 	pipelineID, err := cfg.String("pipeline", -1)
 	assert.NoError(t, err)
-	assert.Equal(t, "filebeat-5.2.0-nginx-access-default", pipelineID)
+	assert.Equal(t, "filebeat-5.2.0-nginx-access-pipeline", pipelineID)
 }
 
 func TestGetInputConfigNginxOverrides(t *testing.T) {
@@ -217,7 +217,7 @@ func TestGetInputConfigNginxOverrides(t *testing.T) {
 
 				pipelineID, err := c.String("pipeline", -1)
 				assert.NoError(t, err)
-				assert.Equal(t, "filebeat-5.2.0-nginx-access-default", pipelineID)
+				assert.Equal(t, "filebeat-5.2.0-nginx-access-pipeline", pipelineID)
 			},
 		},
 		"pipeline": {
@@ -276,7 +276,7 @@ func TestGetPipelineNginx(t *testing.T) {
 	assert.Len(t, pipelines, 1)
 
 	pipeline := pipelines[0]
-	assert.Equal(t, "filebeat-5.2.0-nginx-access-default", pipeline.id)
+	assert.Equal(t, "filebeat-5.2.0-nginx-access-pipeline", pipeline.id)
 	assert.Contains(t, pipeline.contents, "description")
 	assert.Contains(t, pipeline.contents, "processors")
 }

filebeat/fileset/modules_integration_test.go

@@ -115,7 +115,7 @@ func TestSetupNginx(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	status, _, _ := client.Request("GET", "/_ingest/pipeline/filebeat-5.2.0-nginx-access-default", "", nil, nil)
+	status, _, _ := client.Request("GET", "/_ingest/pipeline/filebeat-5.2.0-nginx-access-pipeline", "", nil, nil)
 	assert.Equal(t, 200, status)
 	status, _, _ = client.Request("GET", "/_ingest/pipeline/filebeat-5.2.0-nginx-error-pipeline", "", nil, nil)
 	assert.Equal(t, 200, status)

filebeat/input/log/input.go

@@ -161,7 +161,7 @@ func NewInput(
 // It goes through all states coming from the registry. Only the states which match the glob patterns of
 // the input will be loaded and updated. All other states will not be touched.
 func (p *Input) loadStates(states []file.State) error {
-	logp.Debug("input", "exclude_files: %s. Number of stats: %d", p.config.ExcludeFiles, len(states))
+	logp.Debug("input", "exclude_files: %s. Number of states: %d", p.config.ExcludeFiles, len(states))
 
 	for _, state := range states {
 		// Check if state source belongs to this input. If yes, update the state.

filebeat/inputsource/tcp/server_test.go

@@ -213,7 +213,7 @@ func TestReceiveNewEventsConcurrently(t *testing.T) {
 	to := func(message []byte, mt inputsource.NetworkMetadata) {
 		ch <- &info{message: string(message), mt: mt}
 	}
-	cfg, err := common.NewConfigFrom(map[string]interface{}{"host": ":0"})
+	cfg, err := common.NewConfigFrom(map[string]interface{}{"host": "127.0.0.1:0"})
 	if !assert.NoError(t, err) {
 		return
 	}

filebeat/magefile.go

@@ -185,7 +185,9 @@ func IntegTest() {
 // Use TEST_COVERAGE=true to enable code coverage profiling.
 // Use RACE_DETECTOR=true to enable the race detector.
 func GoIntegTest(ctx context.Context) error {
-	return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
+	return devtools.RunIntegTest("goIntegTest", func() error {
+		return devtools.GoTest(ctx, devtools.DefaultGoTestIntegrationArgs())
+	})
 }
 
 // PythonIntegTest executes the python system tests in the integration environment (Docker).