Merge branch 'master' into archive-system-tests

* master:
  [MetricBeat] [AWS] Fix aws metric tags with resourcegroupstaggingapi paginator (elastic#26385) (elastic#26443)
  Move openmetrics module to oss (elastic#26561)
  Skip flaky test TestFilestreamMetadataUpdatedOnRename (elastic#26609)
  [filebeat][fortinet] Use default add_locale for fortinet.firewall (elastic#26524)
  Enroll proxy settings (elastic#26514)

CHANGELOG.next.asciidoc

@@ -499,6 +499,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
 - Major refactor of system/cpu and system/core metrics. {pull}25771[25771]
 - Fix GCP Project ID being ingested as `cloud.account.id` in `gcp.billing` module {issue}26357[26357] {pull}26412[26412]
+- Fix aws metric tags with resourcegroupstaggingapi paginator.  {issue}26385[26385] {pull}26443[26443]
 
 *Packetbeat*
 
@@ -842,6 +843,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `uri_parts` and `user_agent` ingest processors to `aws.elb` module. {issue}26435[26435] {pull}26441[26441]
 - Added dataset `recordedfuture` to the `threatintel` module to ingest indicators from Recorded Future Connect API {pull}26481[26481]
 - Update `fortinet` ingest pipelines. {issue}22136[22136] {issue}25254[25254] {pull}24816[24816]
+- Use default add_locale for fortinet.firewall {issue}20300[20300] {pull}26524[26524]
 
 *Heartbeat*
 
@@ -981,6 +983,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Collect linked account information in AWS billing. {pull}26285[26285]
 - Add total CPU to vSphere virtual machine metrics. {pull}26167[26167]
 - Add AWS Kinesis metricset. {pull}25989[25989]
+- Move openmetrics module to oss. {pull}26561[26561]
 
 *Packetbeat*
 

Jenkinsfile

@@ -744,6 +744,9 @@ def getCommonModuleInTheChangeSet(String directory) {
   def exclude = "^(${directoryExclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)"
   dir("${env.BASE_DIR}") {
     module = getGitMatchingGroup(pattern: pattern, exclude: exclude)
+    if(!fileExists("${directory}/module/${module}")) {
+      module = ''
+    }
   }
   return module
 }

filebeat/docs/modules/fortinet.asciidoc

@@ -27,7 +27,7 @@ include::../include/gs-link.asciidoc[]
 [float]
 === Compatibility
 
-This module has been tested against FortiOS version 6.0.x and 6.2.x. 
+This module has been tested against FortiOS version 6.0.x and 6.2.x.
 Versions above this are expected to work but have not been tested.
 
 include::../include/configuring-intro.asciidoc[]
@@ -51,6 +51,8 @@ include::../include/config-option-intro.asciidoc[]
 
 include::../include/var-paths.asciidoc[]
 
+include::../include/timezone-support.asciidoc[]
+
 *`var.input`*::
 
 The input to use, can be either the value `tcp`, `udp` or `file`.

filebeat/input/filestream/input_integration_test.go

@@ -80,6 +80,8 @@ func TestFilestreamCloseRenamed(t *testing.T) {
 }
 
 func TestFilestreamMetadataUpdatedOnRename(t *testing.T) {
+	t.Skip("Flaky test: https://github.com/elastic/beats/issues/26608")
+
 	if runtime.GOOS == "windows" {
 		t.Skip("renaming files while Filebeat is running is not supported on Windows")
 	}

metricbeat/docs/modules/openmetrics.asciidoc

@@ -3,7 +3,6 @@ This file is generated! See scripts/mage/docs_collector.go
 ////
 
 [[metricbeat-module-openmetrics]]
-[role="xpack"]
 == Openmetrics module
 
 beta[]

metricbeat/docs/modules/openmetrics/collector.asciidoc

@@ -3,12 +3,11 @@ This file is generated! See scripts/mage/docs_collector.go
 ////
 
 [[metricbeat-metricset-openmetrics-collector]]
-[role="xpack"]
 === Openmetrics collector metricset
 
 beta[]
 
-include::../../../../x-pack/metricbeat/module/openmetrics/collector/_meta/docs.asciidoc[]
+include::../../../module/openmetrics/collector/_meta/docs.asciidoc[]
 
 
 ==== Fields
@@ -20,5 +19,5 @@ Here is an example document generated by this metricset:
 
 [source,json]
 ----
-include::../../../../x-pack/metricbeat/module/openmetrics/collector/_meta/data.json[]
+include::../../../module/openmetrics/collector/_meta/data.json[]
 ----

metricbeat/metricbeat.reference.yml

@@ -716,6 +716,19 @@ metricbeat.modules:
   # Path to server status. Default nginx_status
   server_status_path: "nginx_status"
 
+#----------------------------- Openmetrics Module -----------------------------
+- module: openmetrics
+  metricsets: ['collector']
+  period: 10s
+  hosts: ['localhost:9090']
+
+  # This module uses the Prometheus collector metricset, all
+  # the options for this metricset are also available here.
+  metrics_path: /metrics
+  metrics_filters:
+    include: []
+    exclude: []
+
 #------------------------------- PHP_FPM Module -------------------------------
 - module: php_fpm
   metricsets:

metricbeat/module/openmetrics/collector/collector.go

@@ -0,0 +1,42 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package collector
+
+import (
+	"github.com/elastic/beats/v7/metricbeat/mb"
+	"github.com/elastic/beats/v7/metricbeat/mb/parse"
+	"github.com/elastic/beats/v7/metricbeat/module/prometheus/collector"
+)
+
+const (
+	defaultScheme = "http"
+	defaultPath   = "/metrics"
+)
+
+var (
+	hostParser = parse.URLHostParserBuilder{
+		DefaultScheme: defaultScheme,
+		DefaultPath:   defaultPath,
+	}.Build()
+)
+
+func init() {
+	mb.Registry.MustAddMetricSet("openmetrics", "collector",
+		collector.MetricSetBuilder("openmetrics", collector.DefaultPromEventsGeneratorFactory),
+		mb.WithHostParser(hostParser))
+}

metricbeat/module/openmetrics/collector/collector_test.go

@@ -0,0 +1,32 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build !integration
+
+package collector
+
+import (
+	"testing"
+
+	mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing"
+
+	_ "github.com/elastic/beats/v7/metricbeat/module/openmetrics"
+)
+
+func TestData(t *testing.T) {
+	mbtest.TestDataFiles(t, "openmetrics", "collector")
+}

x-pack/metricbeat/module/openmetrics/test_openmetrics.py → metricbeat/module/openmetrics/test_openmetrics.py

@@ -1,10 +1,10 @@
+import metricbeat
 import os
 import sys
 import unittest
-from xpack_metricbeat import XPackTest, metricbeat
 
 
-class Test(XPackTest):
+class Test(metricbeat.BaseTest):
 
     COMPOSE_SERVICES = ['openmetrics-node_exporter']
 
@@ -19,7 +19,7 @@ def test_openmetrics(self):
             "hosts": self.get_hosts(),
             "period": "5s",
         }])
-        proc = self.start_beat(home=self.beat_path)
+        proc = self.start_beat()
         self.wait_until(lambda: self.output_lines() > 0, 60)
         proc.check_kill_and_wait()
         self.assert_no_logged_warnings()

x-pack/elastic-agent/CHANGELOG.next.asciidoc

@@ -116,5 +116,6 @@
 - Enable agent to send custom headers to kibana/ES {pull}26275[26275]
 - Set `agent.id` to the Fleet Agent ID in events published from inputs backed by Beats. {issue}21121[21121] {pull}26394[26394] {pull}26548[26548]
 - Add proxy support to artifact downloader and communication with fleet server. {pull}25219[25219]
+- Add proxy support to enroll command. {pull}26514[26514]
 - Enable configuring monitoring namespace {issue}26439[26439]
 - Communicate with Fleet Server over HTTP2. {pull}26474[26474]

x-pack/elastic-agent/pkg/agent/cmd/enroll.go

@@ -66,6 +66,9 @@ func addEnrollFlags(cmd *cobra.Command) {
 	cmd.Flags().StringP("ca-sha256", "p", "", "Comma separated list of certificate authorities hash pins used for certificate verifications")
 	cmd.Flags().BoolP("insecure", "i", false, "Allow insecure connection to fleet-server")
 	cmd.Flags().StringP("staging", "", "", "Configures agent to download artifacts from a staging build")
+	cmd.Flags().StringP("proxy-url", "", "", "Configures the proxy url")
+	cmd.Flags().BoolP("proxy-disabled", "", false, "Disable proxy support including environment variables")
+	cmd.Flags().StringSliceP("proxy-header", "", []string{}, "Proxy headers used with CONNECT request")
 }
 
 func buildEnrollmentFlags(cmd *cobra.Command, url string, token string) []string {
@@ -89,6 +92,9 @@ func buildEnrollmentFlags(cmd *cobra.Command, url string, token string) []string
 	sha256, _ := cmd.Flags().GetString("ca-sha256")
 	insecure, _ := cmd.Flags().GetBool("insecure")
 	staging, _ := cmd.Flags().GetString("staging")
+	fProxyURL, _ := cmd.Flags().GetString("proxy-url")
+	fProxyDisabled, _ := cmd.Flags().GetBool("proxy-disabled")
+	fProxyHeaders, _ := cmd.Flags().GetStringSlice("proxy-header")
 
 	args := []string{}
 	if url != "" {
@@ -155,6 +161,20 @@ func buildEnrollmentFlags(cmd *cobra.Command, url string, token string) []string
 		args = append(args, "--staging")
 		args = append(args, staging)
 	}
+
+	if fProxyURL != "" {
+		args = append(args, "--proxy-url")
+		args = append(args, fProxyURL)
+	}
+	if fProxyDisabled {
+		args = append(args, "--proxy-disabled")
+		args = append(args, "true")
+	}
+	for k, v := range mapFromEnvList(fProxyHeaders) {
+		args = append(args, "--proxy-header")
+		args = append(args, k+"="+v)
+	}
+
 	return args
 }
 
@@ -228,6 +248,9 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, args []string) error {
 	fCert, _ := cmd.Flags().GetString("fleet-server-cert")
 	fCertKey, _ := cmd.Flags().GetString("fleet-server-cert-key")
 	fInsecure, _ := cmd.Flags().GetBool("fleet-server-insecure-http")
+	fProxyURL, _ := cmd.Flags().GetString("proxy-url")
+	fProxyDisabled, _ := cmd.Flags().GetBool("proxy-disabled")
+	fProxyHeaders, _ := cmd.Flags().GetStringSlice("proxy-header")
 
 	caStr, _ := cmd.Flags().GetString("certificate-authorities")
 	CAs := cli.StringToSlice(caStr)
@@ -257,6 +280,9 @@ func enroll(streams *cli.IOStreams, cmd *cobra.Command, args []string) error {
 			Insecure:        fInsecure,
 			SpawnAgent:      !fromInstall,
 			Headers:         mapFromEnvList(fHeaders),
+			ProxyURL:        fProxyURL,
+			ProxyDisabled:   fProxyDisabled,
+			ProxyHeaders:    mapFromEnvList(fProxyHeaders),
 		},
 	}