Merge branch 'master' into ilm-createalias-exists

CHANGELOG.next.asciidoc

@@ -239,6 +239,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix out of date FreeBSD vagrantbox. {pull}25652[25652]
 - Fix handling of `file_selectors` in aws-s3 input. {pull}25792[25792]
 - Fix ILM alias creation when write alias exists and initial index does not exist {pull}26143[26143]
+- Include date separator in the filename prefix of `dateRotator` to make sure nothing gets purged accidentally {pull}26176[26176]
 
 *Auditbeat*
 
@@ -260,6 +261,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - system/socket: Fixed start failure when run under config reloader. {issue}20851[20851] {pull}21693[21693]
 - system/socket: Having some CPUs unavailable to Auditbeat could cause startup errors or event loss. {pull}22827[22827]
 - Note incompatibility of system/socket on ARM. {pull}23381[23381]
+- auditd: Fix kernel deadlock when netlink congestion causes "no buffer space available" errors. {issue}26031[26031] {pull}26032[26032]
 
 *Filebeat*
 
@@ -383,6 +385,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix `fortinet.firewall.addr` when its a string, not an IP address. {issue}25585[25585] {pull}25608[25608]
 - Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674]
 - Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675]
+- o365: Avoid mapping exception for `Parameters` and `ExtendedProperties` fields of string type. {pull}26164[26164]
 
 *Heartbeat*
 
@@ -809,6 +812,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add fingerprint processor to generate fixed ids for `google_workspace` events. {pull}25841[25841]
 - Update PanOS module to parse HIP Match logs. {issue}24350[24350] {pull}25686[25686]
 - Enhance GCP module to populate orchestrator.* fields for GKE / K8S logs {pull}25368[25368]
+- http_endpoint: Support multiple documents in a single request by POSTing an array or NDJSON format. {pull}25764[25764]
 
 *Heartbeat*
 
@@ -941,6 +945,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add additional network metrics to docker/network {pull}25354[25354]
 - Migrate ec2 metricsets to use cloudwatch input. {pull}25924[25924]
 - Reduce number of requests done by kubernetes metricsets to kubelet. {pull}25782[25782]
+- Migrate rds metricsets to use cloudwatch input. {pull}26077[26077]
 - Migrate sqs metricsets to use cloudwatch input. {pull}26117[26117]
 
 *Packetbeat*

auditbeat/module/auditd/audit_linux.go

@@ -51,6 +51,8 @@ const (
 
 	lostEventsUpdateInterval        = time.Second * 15
 	maxDefaultStreamBufferConsumers = 4
+
+	setPIDMaxRetries = 5
 )
 
 type backpressureStrategy uint8
@@ -137,10 +139,32 @@ func newAuditClient(c *Config, log *logp.Logger) (*libaudit.AuditClient, error)
 	return libaudit.NewAuditClient(nil)
 }
 
+func closeAuditClient(client *libaudit.AuditClient) error {
+	discard := func(bytes []byte) ([]syscall.NetlinkMessage, error) {
+		return nil, nil
+	}
+	// Drain the netlink channel in parallel to Close() to prevent a deadlock.
+	// This goroutine will terminate once receive from netlink errors (EBADF,
+	// EBADFD, or any other error). This happens because the fd is closed.
+	go func() {
+		for {
+			_, err := client.Netlink.Receive(true, discard)
+			switch err {
+			case nil, syscall.EINTR:
+			case syscall.EAGAIN:
+				time.Sleep(50 * time.Millisecond)
+			default:
+				return
+			}
+		}
+	}()
+	return client.Close()
+}
+
 // Run initializes the audit client and receives audit messages from the
 // kernel until the reporter's done channel is closed.
 func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
-	defer ms.client.Close()
+	defer closeAuditClient(ms.client)
 
 	if err := ms.addRules(reporter); err != nil {
 		reporter.Error(err)
@@ -164,7 +188,7 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
 		go func() {
 			defer func() { // Close the most recently allocated "client" instance.
 				if client != nil {
-					client.Close()
+					closeAuditClient(client)
 				}
 			}()
 			timer := time.NewTicker(lostEventsUpdateInterval)
@@ -178,7 +202,7 @@ func (ms *MetricSet) Run(reporter mb.PushReporterV2) {
 						ms.updateKernelLostMetric(status.Lost)
 					} else {
 						ms.log.Error("get status request failed:", err)
-						if err = client.Close(); err != nil {
+						if err = closeAuditClient(client); err != nil {
 							ms.log.Errorw("Error closing audit monitoring client", "error", err)
 						}
 						client, err = libaudit.NewAuditClient(nil)
@@ -233,7 +257,7 @@ func (ms *MetricSet) addRules(reporter mb.PushReporterV2) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to create audit client for adding rules")
 	}
-	defer client.Close()
+	defer closeAuditClient(client)
 
 	// Don't attempt to change configuration if audit rules are locked (enabled == 2).
 	// Will result in EPERM.
@@ -350,10 +374,12 @@ func (ms *MetricSet) initClient() error {
 			return errors.Wrap(err, "failed to enable auditing in the kernel")
 		}
 	}
+
 	if err := ms.client.WaitForPendingACKs(); err != nil {
 		return errors.Wrap(err, "failed to wait for ACKs")
 	}
-	if err := ms.client.SetPID(libaudit.WaitForReply); err != nil {
+
+	if err := ms.setPID(setPIDMaxRetries); err != nil {
 		if errno, ok := err.(syscall.Errno); ok && errno == syscall.EEXIST && status.PID != 0 {
 			return fmt.Errorf("failed to set audit PID. An audit process is already running (PID %d)", status.PID)
 		}
@@ -362,6 +388,20 @@ func (ms *MetricSet) initClient() error {
 	return nil
 }
 
+func (ms *MetricSet) setPID(retries int) (err error) {
+	if err = ms.client.SetPID(libaudit.WaitForReply); err == nil || errors.Cause(err) != syscall.ENOBUFS || retries == 0 {
+		return err
+	}
+	// At this point the netlink channel is congested (ENOBUFS).
+	// Drain and close the client, then retry with a new client.
+	closeAuditClient(ms.client)
+	if ms.client, err = newAuditClient(&ms.config, ms.log); err != nil {
+		return errors.Wrapf(err, "failed to recover from ENOBUFS")
+	}
+	ms.log.Info("Recovering from ENOBUFS ...")
+	return ms.setPID(retries - 1)
+}
+
 func (ms *MetricSet) updateKernelLostMetric(lost uint32) {
 	if !ms.kernelLost.enabled {
 		return

heartbeat/_meta/fields.common.yml

@@ -174,6 +174,52 @@
               type: text
             - name: stack
               type: text
+        - name: browser
+          type: group
+          fields:
+            - name: experience
+              type: group
+              fields:
+                - name: name
+                  type: keyword
+                - name: type
+                  type: text
+                  description: >
+                    denotes the 'mark' event
+                - name: start
+                  type: long
+                  description: >
+                    offset of time relative to journey start in milliseconds
+            - name: user_timing
+              type: group
+              fields:
+                - name: name
+                  type: keyword
+                - name: type
+                  type: text
+                  description: >
+                    could be one of mark or measure event types.
+                - name: start
+                  type: long
+                  description: >
+                    offset of time relative to journey start in milliseconds
+                - name: end
+                  type: long
+                  description: >
+                    offset of time relative to journey start in milliseconds
+            - name: layout_shift
+              type: group
+              fields:
+                - name: name
+                  type: keyword
+                - name: score
+                  type: integer
+                - name: exists
+                  type: boolean
+                  description: >
+                    flag that indicates if there was any layout shift events
+                    present on the page.
+
 - key: http
   title: "HTTP monitor"
   description:
@@ -379,12 +425,12 @@
           type: group
           description: Detailed x509 certificate metadata
           fields:
-              - name: version_number
-                type: keyword
-                ignore_above: 1024
-                description: Version of x509 format.
-                example: 3
-                default_field: false
+            - name: version_number
+              type: keyword
+              ignore_above: 1024
+              description: Version of x509 format.
+              example: 3
+              default_field: false
 
 - key: icmp
   title: "ICMP"

heartbeat/docs/fields.asciidoc

@@ -10499,6 +10499,98 @@ type: text
 
 --
 
+
+
+*`synthetics.browser.experience.name`*::
++
+--
+type: keyword
+
+--
+
+*`synthetics.browser.experience.type`*::
++
+--
+denotes the 'mark' event
+
+
+type: text
+
+--
+
+*`synthetics.browser.experience.start`*::
++
+--
+offset of time relative to journey start in milliseconds
+
+
+type: long
+
+--
+
+
+*`synthetics.browser.user_timing.name`*::
++
+--
+type: keyword
+
+--
+
+*`synthetics.browser.user_timing.type`*::
++
+--
+could be one of mark or measure event types.
+
+
+type: text
+
+--
+
+*`synthetics.browser.user_timing.start`*::
++
+--
+offset of time relative to journey start in milliseconds
+
+
+type: long
+
+--
+
+*`synthetics.browser.user_timing.end`*::
++
+--
+offset of time relative to journey start in milliseconds
+
+
+type: long
+
+--
+
+
+*`synthetics.browser.layout_shift.name`*::
++
+--
+type: keyword
+
+--
+
+*`synthetics.browser.layout_shift.score`*::
++
+--
+type: integer
+
+--
+
+*`synthetics.browser.layout_shift.exists`*::
++
+--
+flag that indicates if there was any layout shift events present on the page.
+
+
+type: boolean
+
+--
+
 [[exported-fields-tcp]]
 == TCP layer fields
 

libbeat/common/file/rotator.go

@@ -435,11 +435,11 @@ func newRotater(log Logger, s SuffixType, filename string, maxBackups uint, inte
 func newDateRotater(log Logger, filename string) rotater {
 	d := &dateRotator{
 		log:            log,
-		filenamePrefix: filename,
+		filenamePrefix: filename + "-",
 		format:         "20060102150405",
 	}
 
-	d.currentFilename = d.filenamePrefix + "-" + time.Now().Format(d.format)
+	d.currentFilename = d.filenamePrefix + time.Now().Format(d.format)
 	files, err := filepath.Glob(d.filenamePrefix + "*")
 	if err != nil {
 		return d
@@ -467,7 +467,7 @@ func (d *dateRotator) Rotate(reason rotateReason, rotateTime time.Time) error {
 		d.log.Debugw("Rotating file", "filename", d.currentFilename, "reason", reason)
 	}
 
-	d.currentFilename = d.filenamePrefix + "-" + rotateTime.Format(d.format)
+	d.currentFilename = d.filenamePrefix + rotateTime.Format(d.format)
 	return nil
 }
 
@@ -493,7 +493,7 @@ func (d *dateRotator) SortModTimeLogs(strings []string) {
 }
 
 func (d *dateRotator) OrderLog(filename string) time.Time {
-	ts, err := time.Parse(d.format, filepath.Base(filename))
+	ts, err := time.Parse(d.filenamePrefix+d.format, filepath.Base(filename))
 	if err != nil {
 		return time.Time{}
 	}

metricbeat/docs/modules/aws/rds.asciidoc

@@ -8,6 +8,7 @@ This file is generated! See scripts/mage/docs_collector.go
 
 include::../../../../x-pack/metricbeat/module/aws/rds/_meta/docs.asciidoc[]
 
+This is a default metricset. If the host module is unconfigured, this metricset is enabled by default.
 
 ==== Fields
 

x-pack/elastic-agent/CHANGELOG.next.asciidoc

@@ -68,6 +68,7 @@
 - Handle case where policy doesn't contain Fleet connection information {pull}25707[25707]
 - Fix fleet-server.yml spec to not overwrite existing keys {pull}25741[25741]
 - Agent sends wrong log level to Endpoint {issue}25583[25583]
+- Change timestamp in elatic-agent-json.log to use UTC {issue}25391[25391]
 
 ==== New features