[7.x](backport #26828) Add definition for logging.metrics.enabled, pass config from agent to beats

libbeat/logp/config.go

@@ -39,7 +39,8 @@ type Config struct {
 	ToFiles     bool `config:"to_files" yaml:"to_files"`
 	ToEventLog  bool `config:"to_eventlog" yaml:"to_eventlog"`
 
-	Files FileConfig `config:"files"`
+	Files   FileConfig    `config:"files"`
+	Metrics MetricsConfig `config:"metrics"`
 
 	environment Environment
 	addCaller   bool // Adds package and line number info to messages.
@@ -59,6 +60,14 @@ type FileConfig struct {
 	RedirectStderr  bool            `config:"redirect_stderr" yaml:"redirect_stderr"`
 }
 
+// MetricsConfig contains configuration used by the monitor to output metrics into the logstream.
+//
+// Currently these options are not used through this object in beats (as monitoring is setup elsewhere).
+type MetricsConfig struct {
+	Enabled bool          `config:"enabled"`
+	Period  time.Duration `config:"period"`
+}
+
 const (
 	defaultLevel = InfoLevel
 )
@@ -76,6 +85,10 @@ func DefaultConfig(environment Environment) Config {
 			Interval:        0,
 			RotateOnStartup: true,
 		},
+		Metrics: MetricsConfig{
+			Enabled: true,
+			Period:  30 * time.Second,
+		},
 		environment: environment,
 		addCaller:   true,
 	}

libbeat/tests/system/config/mockbeat.yml.j2

@@ -101,6 +101,9 @@ setup.ilm:
 {% if metrics_period -%}
 logging.metrics.period: {{ metrics_period }}
 {%- endif %}
+{% if file_name -%}
+logging.files.name: {{ file_name }}
+{%- endif %}
 
 {% if keystore_path %}
 #================================ keystore =====================================

libbeat/tests/system/test_cmd_export_config.py

@@ -19,22 +19,22 @@ def test_default(self):
         """
         Test export config works
         """
-        self.render_config_template(self.beat_name, self.output, metrics_period='1234')
+        self.render_config_template(self.beat_name, self.output, file_name='some-file')
         exit_code = self.run_beat(extra_args=["export", "config"], config=self.config)
 
         assert exit_code == 0
         assert self.log_contains("filename: mockbeat")
-        assert self.log_contains("period: 1234")
+        assert self.log_contains("name: some-file")
 
     def test_config_environment_variable(self):
         """
         Test export config works but doesn"t expose environment variable.
         """
         self.render_config_template(self.beat_name, self.output,
-                                    metrics_period="${METRIC_PERIOD}")
+                                    file_name="${FILE_NAME}")
         exit_code = self.run_beat(extra_args=["export", "config"], config=self.config,
-                                  env={'METRIC_PERIOD': '1234'})
+                                  env={'FILE_NAME': 'some-file'})
 
         assert exit_code == 0
         assert self.log_contains("filename: mockbeat")
-        assert self.log_contains("period: ${METRIC_PERIOD}")
+        assert self.log_contains("name: ${FILE_NAME}")

x-pack/elastic-agent/CHANGELOG.next.asciidoc

@@ -125,6 +125,7 @@
 - Add proxy support to enroll command. {pull}26514[26514]
 - Enable configuring monitoring namespace {issue}26439[26439]
 - Communicate with Fleet Server over HTTP2. {pull}26474[26474]
+- Pass logging.metrics.enabled to beats to stop beats from adding metrics into their logs. {issue}26758[26758] {pull}26828[26828]
 - Support Node and Service autodiscovery in kubernetes dynamic provider. {pull}26801[26801]
 - Increase Agent's mem limits in k8s. {pull}27153[27153]
 - Add new --enroll-delay option for install and enroll commands. {pull}27118[27118]

x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl

@@ -154,7 +154,8 @@ agent.logging.to_stderr: true
 # If enabled, Elastic-Agent periodically logs its internal metrics that have changed
 # in the last period. For each metric that changed, the delta from the value at
 # the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
+# all non-zero internal metrics are logged on shutdown. This setting is also passed
+# to beats running under the agent. The default is true.
 #agent.logging.metrics.enabled: true
 
 # The period after which to log the internal metrics. The default is 30s.

x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl

@@ -154,7 +154,8 @@ agent.logging.to_stderr: true
 # If enabled, Elastic-Agent periodically logs its internal metrics that have changed
 # in the last period. For each metric that changed, the delta from the value at
 # the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
+# all non-zero internal metrics are logged on shutdown. This setting is also passed
+# to beats running under the agent. The default is true.
 #agent.logging.metrics.enabled: true
 
 # The period after which to log the internal metrics. The default is 30s.

x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl

@@ -154,7 +154,8 @@ agent.logging.to_stderr: true
 # If enabled, Elastic-Agent periodically logs its internal metrics that have changed
 # in the last period. For each metric that changed, the delta from the value at
 # the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
+# all non-zero internal metrics are logged on shutdown. This setting is also passed
+# to beats running under the agent. The default is true.
 #agent.logging.metrics.enabled: true
 
 # The period after which to log the internal metrics. The default is 30s.

x-pack/elastic-agent/_meta/elastic-agent.yml

@@ -149,7 +149,8 @@ agent.logging.to_stderr: true
 # If enabled, Elastic-Agent periodically logs its internal metrics that have changed
 # in the last period. For each metric that changed, the delta from the value at
 # the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
+# all non-zero internal metrics are logged on shutdown. This setting is also passed
+# to beats running under the agent. The default is true.
 #agent.logging.metrics.enabled: true
 
 # The period after which to log the internal metrics. The default is 30s.

x-pack/elastic-agent/elastic-agent.docker.yml

@@ -154,7 +154,8 @@ agent.logging.to_stderr: true
 # If enabled, Elastic-Agent periodically logs its internal metrics that have changed
 # in the last period. For each metric that changed, the delta from the value at
 # the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
+# all non-zero internal metrics are logged on shutdown. This setting is also passed
+# to beats running under the agent. The default is true.
 #agent.logging.metrics.enabled: true
 
 # The period after which to log the internal metrics. The default is 30s.

x-pack/elastic-agent/elastic-agent.reference.yml

@@ -160,7 +160,8 @@ agent.logging.to_stderr: true
 # If enabled, Elastic-Agent periodically logs its internal metrics that have changed
 # in the last period. For each metric that changed, the delta from the value at
 # the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
+# all non-zero internal metrics are logged on shutdown. This setting is also passed
+# to beats running under the agent. The default is true.
 #agent.logging.metrics.enabled: true
 
 # The period after which to log the internal metrics. The default is 30s.

x-pack/elastic-agent/elastic-agent.yml

@@ -160,7 +160,8 @@ agent.logging.to_stderr: true
 # If enabled, Elastic-Agent periodically logs its internal metrics that have changed
 # in the last period. For each metric that changed, the delta from the value at
 # the beginning of the period is logged. Also, the total values for
-# all non-zero internal metrics are logged on shutdown. The default is true.
+# all non-zero internal metrics are logged on shutdown. This setting is also passed
+# to beats running under the agent. The default is true.
 #agent.logging.metrics.enabled: true
 
 # The period after which to log the internal metrics. The default is 30s.

x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go

@@ -32,10 +32,11 @@ type Monitor struct {
 }
 
 // NewMonitor creates a beats monitor.
-func NewMonitor(downloadConfig *artifact.Config, monitoringCfg *monitoringConfig.MonitoringConfig) *Monitor {
+func NewMonitor(downloadConfig *artifact.Config, monitoringCfg *monitoringConfig.MonitoringConfig, logMetrics bool) *Monitor {
 	if monitoringCfg == nil {
 		monitoringCfg = monitoringConfig.DefaultConfig()
 	}
+	monitoringCfg.LogMetrics = logMetrics
 
 	return &Monitor{
 		operatingSystem: downloadConfig.OS(),
@@ -55,6 +56,11 @@ func (b *Monitor) Reload(rawConfig *config.Config) error {
 		b.config = monitoringConfig.DefaultConfig()
 	} else {
 		b.config = cfg.Settings.MonitoringConfig
+		logMetrics := true
+		if cfg.Settings.LoggingConfig != nil {
+			logMetrics = cfg.Settings.LoggingConfig.Metrics.Enabled
+		}
+		b.config.LogMetrics = logMetrics
 	}
 
 	return nil
@@ -135,6 +141,12 @@ func (b *Monitor) EnrichArgs(spec program.Spec, pipelineID string, args []string
 			"-E", "logging.files.permission=0640",
 			"-E", "logging.files.interval=1h",
 		)
+
+		if !b.config.LogMetrics {
+			appendix = append(appendix,
+				"-E", "logging.metrics.enabled=false",
+			)
+		}
 	}
 
 	return append(args, appendix...)

x-pack/elastic-agent/pkg/core/monitoring/config/config.go

@@ -12,6 +12,7 @@ type MonitoringConfig struct {
 	Enabled        bool                  `yaml:"enabled" config:"enabled"`
 	MonitorLogs    bool                  `yaml:"logs" config:"logs"`
 	MonitorMetrics bool                  `yaml:"metrics" config:"metrics"`
+	LogMetrics     bool                  `yaml:"-" config:"-"`
 	HTTP           *MonitoringHTTPConfig `yaml:"http" config:"http"`
 	Namespace      string                `yaml:"namespace" config:"namespace"`
 }
@@ -31,6 +32,7 @@ func DefaultConfig() *MonitoringConfig {
 		Enabled:        true,
 		MonitorLogs:    true,
 		MonitorMetrics: true,
+		LogMetrics:     true,
 		HTTP: &MonitoringHTTPConfig{
 			Enabled: false,
 			Port:    defaultPort,

x-pack/elastic-agent/pkg/core/monitoring/monitor.go

@@ -31,5 +31,9 @@ type Monitor interface {
 
 // NewMonitor creates a monitor based on a process configuration.
 func NewMonitor(cfg *configuration.SettingsConfig) (Monitor, error) {
-	return beats.NewMonitor(cfg.DownloadConfig, cfg.MonitoringConfig), nil
+	logMetrics := true
+	if cfg.LoggingConfig != nil {
+		logMetrics = cfg.LoggingConfig.Metrics.Enabled
+	}
+	return beats.NewMonitor(cfg.DownloadConfig, cfg.MonitoringConfig, logMetrics), nil
 }