From b868b69cc425c6ac08d5ca6c90d18808ea0d1de9 Mon Sep 17 00:00:00 2001
From: Alex K <8418476+fearful-symmetry@users.noreply.github.com>
Date: Thu, 19 Oct 2023 14:45:08 -0700
Subject: [PATCH] Default `allow_older_versions` to true (#36884)

* default allow_older_versions to true

* fix removed file

* fix error message

* Changing tests according to behavior change

* update docs

* add changelog

---------

Co-authored-by: Pierre HILBERT <pierre.hilbert@elastic.co>
---
 CHANGELOG.next.asciidoc                                |  1 +
 auditbeat/auditbeat.reference.yml                      |  5 ++---
 filebeat/filebeat.reference.yml                        |  5 ++---
 heartbeat/heartbeat.reference.yml                      |  5 ++---
 .../config/output-elasticsearch.reference.yml.tmpl     |  5 ++---
 libbeat/cmd/instance/beat.go                           | 10 ++++++----
 libbeat/cmd/instance/beat_test.go                      |  6 +++---
 metricbeat/metricbeat.reference.yml                    |  5 ++---
 packetbeat/packetbeat.reference.yml                    |  5 ++---
 winlogbeat/winlogbeat.reference.yml                    |  5 ++---
 x-pack/auditbeat/auditbeat.reference.yml               |  5 ++---
 x-pack/filebeat/filebeat.reference.yml                 |  5 ++---
 x-pack/functionbeat/functionbeat.reference.yml         |  5 ++---
 x-pack/heartbeat/heartbeat.reference.yml               |  5 ++---
 x-pack/metricbeat/metricbeat.reference.yml             |  5 ++---
 x-pack/osquerybeat/osquerybeat.reference.yml           |  5 ++---
 x-pack/packetbeat/packetbeat.reference.yml             |  5 ++---
 x-pack/winlogbeat/winlogbeat.reference.yml             |  5 ++---
 18 files changed, 40 insertions(+), 52 deletions(-)

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 37bb60d04e3..be6b54838e7 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -166,6 +166,7 @@ is collected by it.
 - Add support for AWS external IDs. {issue}36321[36321] {pull}36322[36322]
 - [Enhanncement for host.ip and host.mac] Disabling netinfo.enabled option of add-host-metadata processor {pull}36506[36506]
   Setting environmental variable ELASTIC_NETINFO:false in Elastic Agent pod will disable the netinfo.enabled option of add_host_metadata processor
+- Beats will now connect to older Elasticsearch instances by default {pull}36884[36884]
 
 *Auditbeat*
 
diff --git a/auditbeat/auditbeat.reference.yml b/auditbeat/auditbeat.reference.yml
index 32fbef2da04..d4214eaf604 100644
--- a/auditbeat/auditbeat.reference.yml
+++ b/auditbeat/auditbeat.reference.yml
@@ -525,9 +525,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # auditbeat expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents auditbeat from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
   # Use SSL settings for HTTPS.
   #ssl.enabled: true
diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
index 7f86b5aa9d2..45aff60ce23 100644
--- a/filebeat/filebeat.reference.yml
+++ b/filebeat/filebeat.reference.yml
@@ -1621,9 +1621,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # filebeat expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents filebeat from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
   # Use SSL settings for HTTPS.
   #ssl.enabled: true
diff --git a/heartbeat/heartbeat.reference.yml b/heartbeat/heartbeat.reference.yml
index 89fc08ef8e6..e8b74f8c075 100644
--- a/heartbeat/heartbeat.reference.yml
+++ b/heartbeat/heartbeat.reference.yml
@@ -617,9 +617,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # heartbeat expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents heartbeat from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
   # Use SSL settings for HTTPS.
   #ssl.enabled: true
diff --git a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl
index 48f1ba2c007..4acd341da01 100644
--- a/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl
+++ b/libbeat/_meta/config/output-elasticsearch.reference.yml.tmpl
@@ -84,9 +84,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # {{.BeatName}} expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents {{.BeatName}} from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
 {{include "ssl.reference.yml.tmpl" . | indent 2 }}
 
diff --git a/libbeat/cmd/instance/beat.go b/libbeat/cmd/instance/beat.go
index 4e72996c966..e6596106b83 100644
--- a/libbeat/cmd/instance/beat.go
+++ b/libbeat/cmd/instance/beat.go
@@ -1028,16 +1028,18 @@ func (b *Beat) loadDashboards(ctx context.Context, force bool) error {
 func (b *Beat) registerESVersionCheckCallback() error {
 	_, err := elasticsearch.RegisterGlobalCallback(func(conn *eslegclient.Connection) error {
 		if !isElasticsearchOutput(b.Config.Output.Name()) {
-			return errors.New("Elasticsearch output is not configured")
+			return errors.New("elasticsearch output is not configured")
 		}
-		if b.isConnectionToOlderVersionAllowed() {
+		// if we allow older versions, return early and don't check versions
+		// versions don't matter on serverless, so always bypass
+		if b.isConnectionToOlderVersionAllowed() || conn.IsServerless() {
 			return nil
 		}
 
 		esVersion := conn.GetVersion()
 		beatVersion, err := libversion.New(b.Info.Version)
 		if err != nil {
-			return err
+			return fmt.Errorf("error fetching version from elasticsearch: %w", err)
 		}
 		if esVersion.LessThanMajorMinor(beatVersion) {
 			return fmt.Errorf("%w ES=%s, Beat=%s", elasticsearch.ErrTooOld, esVersion.String(), b.Info.Version)
@@ -1051,7 +1053,7 @@ func (b *Beat) registerESVersionCheckCallback() error {
 func (b *Beat) isConnectionToOlderVersionAllowed() bool {
 	config := struct {
 		AllowOlder bool `config:"allow_older_versions"`
-	}{false}
+	}{true}
 
 	_ = b.Config.Output.Config().Unpack(&config)
 
diff --git a/libbeat/cmd/instance/beat_test.go b/libbeat/cmd/instance/beat_test.go
index b6834d89b5d..fc8c88a4915 100644
--- a/libbeat/cmd/instance/beat_test.go
+++ b/libbeat/cmd/instance/beat_test.go
@@ -236,7 +236,7 @@ func TestReloader(t *testing.T) {
 elasticsearch:
   hosts: ["https://127.0.0.1:9200"]
   username: "elastic"
-  allow_older_versions: true
+  allow_older_versions: false
 `
 		c, err := config.NewConfigWithYAML([]byte(cfg), cfg)
 		require.NoError(t, err)
@@ -248,13 +248,13 @@ elasticsearch:
 		reloader := b.makeOutputReloader(m)
 
 		require.False(t, b.Config.Output.IsSet(), "the output should not be set yet")
-		require.False(t, b.isConnectionToOlderVersionAllowed(), "the flag should not be present in the empty configuration")
+		require.True(t, b.isConnectionToOlderVersionAllowed(), "allow_older_versions flag should be true from 8.11")
 		err = reloader.Reload(update)
 		require.NoError(t, err)
 		require.True(t, b.Config.Output.IsSet(), "now the output should be set")
 		require.Equal(t, outCfg, b.Config.Output.Config())
 		require.Same(t, c, m.cfg.Config)
-		require.True(t, b.isConnectionToOlderVersionAllowed(), "the flag should be present")
+		require.False(t, b.isConnectionToOlderVersionAllowed(), "allow_older_versions flag should now be set to false")
 	})
 }
 
diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml
index 88e353e883b..fc79ddb514c 100644
--- a/metricbeat/metricbeat.reference.yml
+++ b/metricbeat/metricbeat.reference.yml
@@ -1360,9 +1360,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # metricbeat expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents metricbeat from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
   # Use SSL settings for HTTPS.
   #ssl.enabled: true
diff --git a/packetbeat/packetbeat.reference.yml b/packetbeat/packetbeat.reference.yml
index aebeb1947a6..cc05f7b5212 100644
--- a/packetbeat/packetbeat.reference.yml
+++ b/packetbeat/packetbeat.reference.yml
@@ -991,9 +991,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # packetbeat expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents packetbeat from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
   # Use SSL settings for HTTPS.
   #ssl.enabled: true
diff --git a/winlogbeat/winlogbeat.reference.yml b/winlogbeat/winlogbeat.reference.yml
index 5619e9a6375..110370957cf 100644
--- a/winlogbeat/winlogbeat.reference.yml
+++ b/winlogbeat/winlogbeat.reference.yml
@@ -407,9 +407,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # winlogbeat expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents winlogbeat from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
   # Use SSL settings for HTTPS.
   #ssl.enabled: true
diff --git a/x-pack/auditbeat/auditbeat.reference.yml b/x-pack/auditbeat/auditbeat.reference.yml
index 999f0416354..6d9a71ca99c 100644
--- a/x-pack/auditbeat/auditbeat.reference.yml
+++ b/x-pack/auditbeat/auditbeat.reference.yml
@@ -581,9 +581,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # auditbeat expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents auditbeat from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
   # Use SSL settings for HTTPS.
   #ssl.enabled: true
diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml
index 29704a80ad4..bff96ef1997 100644
--- a/x-pack/filebeat/filebeat.reference.yml
+++ b/x-pack/filebeat/filebeat.reference.yml
@@ -3991,9 +3991,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # filebeat expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents filebeat from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
   # Use SSL settings for HTTPS.
   #ssl.enabled: true
diff --git a/x-pack/functionbeat/functionbeat.reference.yml b/x-pack/functionbeat/functionbeat.reference.yml
index b9a54bd88ec..d3a2231a43e 100644
--- a/x-pack/functionbeat/functionbeat.reference.yml
+++ b/x-pack/functionbeat/functionbeat.reference.yml
@@ -649,9 +649,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # functionbeat expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents functionbeat from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
   # Use SSL settings for HTTPS.
   #ssl.enabled: true
diff --git a/x-pack/heartbeat/heartbeat.reference.yml b/x-pack/heartbeat/heartbeat.reference.yml
index 89fc08ef8e6..e8b74f8c075 100644
--- a/x-pack/heartbeat/heartbeat.reference.yml
+++ b/x-pack/heartbeat/heartbeat.reference.yml
@@ -617,9 +617,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # heartbeat expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents heartbeat from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
   # Use SSL settings for HTTPS.
   #ssl.enabled: true
diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml
index 63acc6f5245..436693bdfbc 100644
--- a/x-pack/metricbeat/metricbeat.reference.yml
+++ b/x-pack/metricbeat/metricbeat.reference.yml
@@ -1921,9 +1921,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # metricbeat expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents metricbeat from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
   # Use SSL settings for HTTPS.
   #ssl.enabled: true
diff --git a/x-pack/osquerybeat/osquerybeat.reference.yml b/x-pack/osquerybeat/osquerybeat.reference.yml
index d6770083e62..f17d16e28b8 100644
--- a/x-pack/osquerybeat/osquerybeat.reference.yml
+++ b/x-pack/osquerybeat/osquerybeat.reference.yml
@@ -368,9 +368,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # osquerybeat expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents osquerybeat from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
   # Use SSL settings for HTTPS.
   #ssl.enabled: true
diff --git a/x-pack/packetbeat/packetbeat.reference.yml b/x-pack/packetbeat/packetbeat.reference.yml
index aebeb1947a6..cc05f7b5212 100644
--- a/x-pack/packetbeat/packetbeat.reference.yml
+++ b/x-pack/packetbeat/packetbeat.reference.yml
@@ -991,9 +991,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # packetbeat expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents packetbeat from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
   # Use SSL settings for HTTPS.
   #ssl.enabled: true
diff --git a/x-pack/winlogbeat/winlogbeat.reference.yml b/x-pack/winlogbeat/winlogbeat.reference.yml
index 5d52a07b37b..eec0bca8077 100644
--- a/x-pack/winlogbeat/winlogbeat.reference.yml
+++ b/x-pack/winlogbeat/winlogbeat.reference.yml
@@ -409,9 +409,8 @@ output.elasticsearch:
   # Configure HTTP request timeout before failing a request to Elasticsearch.
   #timeout: 90
 
-  # winlogbeat expects Elasticsearch to be the same version or newer than the Beat.
-  # Lift the version restriction by setting allow_older_versions to true.
-  #allow_older_versions: false
+  # Prevents winlogbeat from connecting to older Elasticsearch versions when set to `false`
+  #allow_older_versions: true
 
   # Use SSL settings for HTTPS.
   #ssl.enabled: true