Merge remote-tracking branch 'origin/master' into metricbeat-mysql-8

CHANGELOG.next.asciidoc

@@ -44,6 +44,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Rename a few `nginx.error.*` fields to map to ECS. {pull}10007[10007]
 - Filesets with multiple ingest pipelines added in {pull}8914[8914] only work with Elasticsearch >= 6.5.0 {pull}10001[10001]
 - Remove service.name from Elastcsearch module. Replace by service.type. {pull}10042[10042]
+- Remove numeric coercions for `user.id` and `group.id`. IDs should be `keyword`. {pull}10233[10233]
 - Add grok pattern to support redis 5.0.3 log timestamp. {issue}9819[9819] {pull}10033[10033]
 - Now save the 'first seen' timestamp in `event.created` (previously `read_timestamp`),
   instead of saving the parsed date. Now aligned with `event.created` semantics elsewhere. {pull}10139[10139]
@@ -140,6 +141,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add missing file encoding to readers. {pull}10080[10080]
 - Introduce `migration.enabled` configuration. {pull}9805[9805]
 - Add alias field support in Kibana index pattern. {pull}10075[10075]
+- Add `add_fields` processor. {pull}10119[10119]
 
 *Auditbeat*
 
@@ -195,14 +197,15 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Release Couchbase module as GA. {pull}10201[10201]
 - Release RabbitMQ module GA. {pull}10165[10165]
 - Release envoyproxy module GA. {pull}10223[10223]
-- Making RabbitMQ Metricbeat module GA. {pull}10165[10165]
 - Release mongodb.metrics and mongodb.replstatus as GA. {pull}10242[10242]
 - Release mysql.galera_status as GA. {pull}10242[10242]
 - Release postgresql.statement as GA. {pull}10242[10242]
 - Release RabbitMQ Metricbeat module GA. {pull}10165[10165]
 - Release Dropwizard module as GA. {pull}10240[10240]
 - Release Graphite module as GA. {pull}10240[10240]
 - Release http.server metricset as GA. {pull}10240[10240]
+- Release Nats module as GA. {pull}10281[10281]
+- Release use of xpack.enabled: true flag in Elasticsearch and Kibana modules as GA. {pull}10222[10222]
 
 *Packetbeat*
 

auditbeat/docs/configuring-howto.asciidoc

@@ -24,6 +24,7 @@ The following topics describe how to configure {beatname_uc}:
 * <<{beatname_lc}-configuration-reloading>>
 * <<configuring-internal-queue>>
 * <<configuring-output>>
+* <<ilm>>
 * <<configuration-ssl>>
 * <<filtering-and-enhancing-data>>
 * <<configuring-ingest-node>>
@@ -53,6 +54,8 @@ include::{libbeat-dir}/docs/queueconfig.asciidoc[]
 
 include::{libbeat-dir}/docs/outputconfig.asciidoc[]
 
+include::{libbeat-dir}/docs/shared-ilm.asciidoc[]
+
 include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[]
 
 include::./auditbeat-filtering.asciidoc[]

dev-tools/mage/clean.go

@@ -33,8 +33,8 @@ var DefaultCleanPaths = []string{
 	"fields.yml",
 	"_meta/fields.generated.yml",
 	"_meta/kibana.generated",
-	"_meta/kibana/5/index-pattern/{{.BeatName}}.json",
 	"_meta/kibana/6/index-pattern/{{.BeatName}}.json",
+	"_meta/kibana/7/index-pattern/{{.BeatName}}.json",
 }
 
 // Clean clean generated build artifacts.

dev-tools/mage/dashboard.go

@@ -41,8 +41,8 @@ func ExportDashboard() error {
 		return err
 	}
 
-	// TODO: This is currently hardcoded for KB 6, we need to figure out what we do for KB 7
-	file := CWD("module", module, "_meta/kibana/6/dashboard", id+".json")
+	// TODO: This is currently hardcoded for KB 7, we need to figure out what we do for KB 8 if applicable
+	file := CWD("module", module, "_meta/kibana/7/dashboard", id+".json")
 
 	dashboardCmd := sh.RunCmd("go", "run",
 		filepath.Join(beatsDir, "dev-tools/cmd/dashboards/export_dashboards.go"),

dev-tools/mage/kibana.go

@@ -69,7 +69,7 @@ func KibanaDashboards(moduleDirs ...string) error {
 	// Convert 6.x dashboards to strings.
 	err = sh.Run("python",
 		filepath.Join(esBeatsDir, "libbeat/scripts/unpack_dashboards.py"),
-		"--glob="+filepath.Join(kibanaBuildDir, "6/dashboard/*.json"))
+		"--glob="+filepath.Join(kibanaBuildDir, "7/dashboard/*.json"))
 	if err != nil {
 		return err
 	}

filebeat/docs/configuring-howto.asciidoc

@@ -28,6 +28,7 @@ The following topics describe how to configure Filebeat:
 * <<filebeat-configuration-reloading>>
 * <<configuring-internal-queue>>
 * <<configuring-output>>
+* <<ilm>>
 * <<load-balancing>>
 * <<configuration-ssl>>
 * <<filtering-and-enhancing-data>>
@@ -61,6 +62,8 @@ include::{libbeat-dir}/docs/queueconfig.asciidoc[]
 
 include::{libbeat-dir}/docs/outputconfig.asciidoc[]
 
+include::../../libbeat/docs/shared-ilm.asciidoc[]
+
 include::./load-balancing.asciidoc[]
 
 include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[]

filebeat/module/system/auth/ingest/pipeline.json

@@ -13,8 +13,8 @@
                     "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:long}\\])?: %{DATA:event.action} user %{DATA:user.name} from %{IPORHOST:source.ip}",
                     "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sshd(?:\\[%{POSINT:process.pid:long}\\])?: Did not receive identification string from %{IPORHOST:system.auth.ssh.dropped_ip}",
                     "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} sudo(?:\\[%{POSINT:process.pid:long}\\])?: \\s*%{DATA:user.name} :( %{DATA:system.auth.sudo.error} ;)? TTY=%{DATA:system.auth.sudo.tty} ; PWD=%{DATA:system.auth.sudo.pwd} ; USER=%{DATA:system.auth.sudo.user} ; COMMAND=%{GREEDYDATA:system.auth.sudo.command}",
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} groupadd(?:\\[%{POSINT:process.pid:long}\\])?: new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id:long}",
-                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} useradd(?:\\[%{POSINT:process.pid:long}\\])?: new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id:long}, GID=%{NUMBER:group.id:long}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$",
+                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} groupadd(?:\\[%{POSINT:process.pid:long}\\])?: new group: name=%{DATA:group.name}, GID=%{NUMBER:group.id}",
+                    "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname} useradd(?:\\[%{POSINT:process.pid:long}\\])?: new user: name=%{DATA:user.name}, UID=%{NUMBER:user.id}, GID=%{NUMBER:group.id}, home=%{DATA:system.auth.useradd.home}, shell=%{DATA:system.auth.useradd.shell}$",
                     "%{SYSLOGTIMESTAMP:system.auth.timestamp} %{SYSLOGHOST:host.hostname}? %{DATA:process.name}(?:\\[%{POSINT:process.pid:long}\\])?: %{GREEDYMULTILINE:system.auth.message}"
                 ]
             }

filebeat/module/system/auth/test/test.log-expected.json

@@ -136,7 +136,7 @@
         "event.dataset": "system.auth",
         "event.module": "system",
         "fileset.name": "auth",
-        "group.id": 48,
+        "group.id": "48",
         "group.name": "apache",
         "host.hostname": "localhost",
         "input.type": "log",
@@ -149,15 +149,15 @@
         "event.dataset": "system.auth",
         "event.module": "system",
         "fileset.name": "auth",
-        "group.id": 48,
+        "group.id": "48",
         "host.hostname": "localhost",
         "input.type": "log",
         "log.offset": 934,
         "process.pid": 6995,
         "service.type": "system",
         "system.auth.useradd.home": "/usr/share/httpd",
         "system.auth.useradd.shell": "/sbin/nologin",
-        "user.id": 48,
+        "user.id": "48",
         "user.name": "apache"
     }
 ]

heartbeat/docs/configuring-howto.asciidoc

@@ -25,6 +25,7 @@ The following topics describe how to configure Heartbeat:
 * <<configuration-general-options>>
 * <<configuring-internal-queue>>
 * <<configuring-output>>
+* <<ilm>>
 * <<configuration-ssl>>
 * <<filtering-and-enhancing-data>>
 * <<configuring-ingest-node>>
@@ -50,6 +51,8 @@ include::{libbeat-dir}/docs/queueconfig.asciidoc[]
 
 include::{libbeat-dir}/docs/outputconfig.asciidoc[]
 
+include::{libbeat-dir}/docs/shared-ilm.asciidoc[]
+
 include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[]
 
 include::./heartbeat-filtering.asciidoc[]

heartbeat/monitors/wrappers/util.go

@@ -58,7 +58,7 @@ func URLFields(u *url.URL) common.MapStr {
 	}
 
 	if u.Port() != "" {
-		fields["port"], _ = strconv.ParseUint(u.Port(), 10, 8)
+		fields["port"], _ = strconv.ParseUint(u.Port(), 10, 16)
 	}
 
 	if u.Path != "" {

heartbeat/monitors/wrappers/util_test.go

@@ -63,11 +63,12 @@ func TestURLFields(t *testing.T) {
 		},
 		{
 			"complex",
-			"tcp+ssl://myuser:mypass@elastic.co/foo/bar?q=dosomething&x=y",
+			"tcp+ssl://myuser:mypass@elastic.co:65500/foo/bar?q=dosomething&x=y",
 			common.MapStr{
-				"full":     "tcp+ssl://myuser:%3Chidden%3E@elastic.co/foo/bar?q=dosomething&x=y",
+				"full":     "tcp+ssl://myuser:%3Chidden%3E@elastic.co:65500/foo/bar?q=dosomething&x=y",
 				"scheme":   "tcp+ssl",
 				"domain":   "elastic.co",
+				"port":     uint64(65500),
 				"path":     "/foo/bar",
 				"query":    "q=dosomething&x=y",
 				"username": "myuser",

journalbeat/docs/configuring-howto.asciidoc

@@ -16,6 +16,7 @@ The following topics describe how to configure {beatname_uc}:
 * <<configuration-general-options>>
 * <<configuring-internal-queue>>
 * <<configuring-output>>
+* <<ilm>>
 * <<configuration-ssl>>
 * <<filtering-and-enhancing-data>>
 * <<configuring-ingest-node>>
@@ -39,6 +40,8 @@ include::{libbeat-dir}/docs/queueconfig.asciidoc[]
 
 include::{libbeat-dir}/docs/outputconfig.asciidoc[]
 
+include::{libbeat-dir}/docs/shared-ilm.asciidoc[]
+
 include::{libbeat-dir}/docs/shared-ssl-config.asciidoc[]
 
 include::./filtering.asciidoc[]

libbeat/common/mapval/is_defs.go

@@ -126,7 +126,7 @@ func IsDeepEqual(to interface{}) IsDef {
 		return SimpleResult(
 			path,
 			false,
-			fmt.Sprintf("objects not equal: actual(%v) != expected(%v)", v, to),
+			fmt.Sprintf("objects not equal: actual(%T(%v)) != expected(%T(%v))", v, v, to, to),
 		)
 	})
 }

libbeat/dashboards/importer.go

@@ -27,7 +27,6 @@ import (
 	"os"
 	"path"
 	"path/filepath"
-	"strconv"
 	"strings"
 
 	errw "github.com/pkg/errors"
@@ -67,9 +66,9 @@ type Loader interface {
 
 func NewImporter(version common.Version, cfg *Config, loader Loader) (*Importer, error) {
 
-	// Current max version is 6
+	// Current max version is 7
 	if version.Major > 6 {
-		version.Major = 6
+		version.Major = 7
 	}
 
 	return &Importer{
@@ -301,7 +300,7 @@ func (imp Importer) downloadFile(url string, target string) (string, error) {
 func (imp Importer) ImportKibanaDir(dir string) error {
 	var err error
 
-	versionPath := strconv.Itoa(imp.version.Major)
+	versionPath := "7"
 
 	dir = path.Join(dir, versionPath)
 

libbeat/docs/outputconfig.asciidoc

@@ -86,13 +86,17 @@ output.elasticsearch:
   protocol: "https"
   username: "{beatname_lc}_internal"
   password: "{pwd}"
-
 ------------------------------------------------------------------------------
 
 
 For more information about securing {beatname_uc}, see
 <<securing-{beatname_lc}>>.
 
+If you are indexing large amounts of time-series data, you might also want to
+configure {beatname_uc} to use index lifecycle management. For more information
+about configuring and using index lifecycle management with {beatname_uc}, see
+<<ilm>>.
+
 ==== Compatibility
 
 This output works with all compatible versions of Elasticsearch. See the
@@ -338,6 +342,15 @@ This configuration results in indices named `sev1`, `sev2`, and `sev3`.
 The `mappings` setting simplifies the configuration, but is limited to string
 values. You cannot specify format strings within the mapping pairs.
 
+//TODO: MOVE ILM OPTIONS TO APPEAR LOGICALLY BASED ON LOCATION IN THE YAML FILE.
+
+[[ilm-es]]
+===== `ilm`
+
+Configuration options for index lifecycle management. 
+
+See <<ilm>> for more information.
+
 ifndef::no-pipeline[]
 [[pipeline-option-es]]
 ===== `pipeline`