Merge branch 'main' into unify-local-ci-dev-experience

* main: (22 commits)
  chore(jjbb): increase build timeout to 2h (elastic#2661)
  chore: increase build timeout to 2h (elastic#2660)
  feat: run tests on windows 2019 (elastic#2468)
  Remove aws instance reaper Jenkins job (elastic#2647)
  Update k8s version to v1.24 (elastic#2644)
  chore: simplify extracting info from the supported platforms (elastic#2637)
  ci: avoid running on branches that are not available anymore (elastic#2627)
  Add support for Debian 11 and Ubuntu 22.04 platforms (elastic#2603)
  fix: retire 8.1 snapshot from upgrades (elastic#2616)
  bump stack version 8.4.0-42ce0eef (elastic#2613)
  docs: document AWS requirements (elastic#2589)
  chore: move elastic-agent's k8s-autodiscover scenarios from beats to elastic-agent profile (elastic#2601)
  Add test scenario for elastic agent tags (elastic#2552)
  bump stack version 8.4.0-3525aaf7 (elastic#2597)
  bump stack version 8.4.0-df27a645 (elastic#2591)
  fix: agent directories must belong to root (elastic#2590)
  chore: bump maintenance versions in mergify (elastic#2563)
  chore: bump kubernetes versions (elastic#2562)
  fix: use fixed, valid version for protobuffers (elastic#2569)
  chore: do not run apm-server helm chart scenarios for Beats (elastic#2564)
  ...

.ci/.e2e-platforms.yaml

@@ -4,39 +4,65 @@ PLATFORMS:
     description: "Debian 10 AMD64, used specifically for running the stack"
     image: "ami-0d90bed76900e679a"
     instance_type: "t3.xlarge"
+    shell_type: sh
     username: admin
   centos8_arm64:
     description: "CentOS Stream 8 aarch64"
     image: "ami-01cdc9e8306344fe0"
     instance_type: "a1.large"
+    shell_type: sh
     username: centos
   centos8_amd64:
     description: "CentOS Stream 8 x86_64"
     image: "ami-045b0a05944af45c1"
     instance_type: "t3.xlarge"
+    shell_type: sh
     username: centos
-  debian_arm64:
+  debian_10_arm64:
     description: "Debian 10 ARM64"
     image: "ami-06dac44ad759182bd"
     instance_type: "a1.large"
+    shell_type: sh
     username: admin
-  debian_amd64:
+  debian_10_amd64:
     description: "Debian 10 AMD64"
     image: "ami-0d90bed76900e679a"
     instance_type: "t3.xlarge"
+    shell_type: sh
+    username: admin
+  debian_11_amd64:
+    description: "Debian 11 AMD64"
+    image: "ami-0c7c4e3c6b4941f0f"
+    instance_type: "t3.xlarge"
+    shell_type: sh
     username: admin
   oracle_linux8:
     description: "Oracle Linux 8 update 3 for x86_64 HVM"
     image: "ami-00371eeb8fd8e0e16"
     instance_type: "t3.xlarge"
+    shell_type: sh
     username: ec2-user
   sles15:
     description: "SUSE Linux Enterprise Server 15 SP3 (HVM), SSD Volume Type -  64bit ARM"
     image: "ami-0f7cb53c916a75006"
     instance_type: "t3.xlarge"
+    shell_type: sh
     username: ec2-user
+  windows2019:
+    description: "Windows 2019 x86_64 (HVM)"
+    image: "ami-0587bd602f1da2f1d"
+    instance_type: "c5.2xlarge"
+    shell_type: cmd
+    username: ogc
   fleet_elastic_pkg:
     description: "Debian 10 AMD64"
-    image: "ami-0d90bed76900e679a"
+    image: "ami-038f1e4512905b02e"
     instance_type: "t3.xlarge"
+    shell_type: sh
     username: admin
+  ubuntu_22_04_amd64:
+    description: "Ubuntu 22.04 AMD64"
+    image: "ami-0aeb7c931a5a61206"
+    instance_type: "t3.xlarge"
+    shell_type: sh
+    username: ubuntu

.ci/.e2e-tests-beats.yaml

@@ -3,47 +3,41 @@ SUITES:
   - suite: "helm"
     provider: "docker"
     scenarios:
-      - name: "APM Server"
-        tags: "apm-server"
-        platforms: ["debian_amd64"]
       - name: "Filebeat"
         tags: "filebeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "Metricbeat"
         tags: "metricbeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
   - suite: "fleet"
     scenarios:
       - name: "Fleet"
         tags: "fleet_mode_agent"
-        platforms: ["centos8_arm64", "centos8_amd64", "debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["centos8_arm64", "centos8_amd64", " debian_10_arm64", "debian_10_amd64", "sles15"]
       - name: "Integrations"
         tags: "integrations"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "sles15"]
       - name: "APM Integration"
         tags: "apm_server"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "Linux Integration"
         tags: "linux_integration"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "sles15"]
       - name: "Backend Processes"
         tags: "backend_processes"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "sles15"]
       - name: "Beats Background Processes"
         tags: "running_on_beats"
-        platforms: ["debian_arm64", "debian_amd64", "oracle_linux8", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "oracle_linux8", "sles15"]
   - suite: "kubernetes-autodiscover"
     provider: "docker"
     scenarios:
-      - name: "kubernetes autodiscover with elastic-agent"
-        tags: "elastic-agent"
-        platforms: ["debian_amd64"]
       - name: "kubernetes autodiscover with filebeat"
         tags: "filebeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "kubernetes autodiscover with heartbeat"
         tags: "heartbeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "kubernetes autodiscover with metricbeat"
         tags: "metricbeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]

.ci/.e2e-tests-daily.yaml

@@ -5,51 +5,51 @@ SUITES:
     scenarios:
       - name: "APM Server"
         tags: "apm-server"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "Filebeat"
         tags: "filebeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "Metricbeat"
         tags: "metricbeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
   - suite: "fleet"
     scenarios:
       - name: "Fleet"
         tags: "fleet_mode"
-        platforms: ["centos8_arm64", "centos8_amd64", "debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["centos8_arm64", "centos8_amd64", "debian_10_arm64", "debian_10_amd64", "sles15", "debian_11_amd64", "ubuntu_22_04_amd64", "windows2019"]
       - name: "Integrations"
         tags: "integrations"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "sles15", "debian_11_amd64", "ubuntu_22_04_amd64"]
       - name: "APM Integration"
         tags: "apm_server"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "Linux Integration"
         tags: "linux_integration"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "sles15", "debian_11_amd64", "ubuntu_22_04_amd64"]
       - name: "System Integration"
         tags: "system_integration"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "sles15", "debian_11_amd64", "ubuntu_22_04_amd64"]
       - name: "Backend Processes"
         tags: "backend_processes"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "sles15", "debian_11_amd64", "ubuntu_22_04_amd64"]
       - name: "Beats Background Processes"
         tags: "running_on_beats"
-        platforms: ["debian_arm64", "debian_amd64", "oracle_linux8", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "oracle_linux8", "sles15", "debian_11_amd64", "ubuntu_22_04_amd64"]
       - name: "Upgrade Agent"
         tags: "upgrade_agent"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "sles15", "debian_11_amd64", "ubuntu_22_04_amd64"]
   - suite: "kubernetes-autodiscover"
     provider: "docker"
     scenarios:
       - name: "kubernetes autodiscover with elastic-agent"
         tags: "elastic-agent"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "kubernetes autodiscover with filebeat"
         tags: "filebeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "kubernetes autodiscover with heartbeat"
         tags: "heartbeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "kubernetes autodiscover with metricbeat"
         tags: "metricbeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]

.ci/.e2e-tests-for-elastic-agent.yaml

@@ -4,28 +4,34 @@ SUITES:
     scenarios:
       - name: "Fleet"
         tags: "fleet_mode"
-        platforms: ["centos8_arm64", "centos8_amd64", "debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["centos8_arm64", "centos8_amd64", "debian_10_arm64", "debian_11_amd64", "ubuntu_22_04_amd64", "debian_10_amd64", "sles15", "windows2019"]
       - name: "Integrations"
         tags: "integrations"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "debian_11_amd64", "ubuntu_22_04_amd64", "sles15"]
       - name: "APM Integration"
         tags: "apm_server"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "Linux Integration"
         tags: "linux_integration"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "debian_11_amd64", "ubuntu_22_04_amd64", "sles15"]
       # - name: "Permissions Output change"
       #   tags: "permission_change"
-      #   platforms: ["debian_arm64", "debian_amd64", "sles15"]
+      #   platforms: ["debian_10_arm64", "debian_10_amd64", "sles15"]
       - name: "System Integration"
         tags: "system_integration"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "debian_11_amd64", "ubuntu_22_04_amd64", "sles15"]
       # - name: "Stand-alone"
       #   tags: "stand_alone_mode"
       #   platforms: ["fleet_elastic_pkg"]
       - name: "Backend Processes"
         tags: "backend_processes"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "debian_11_amd64", "ubuntu_22_04_amd64", "sles15"]
       - name: "Beats Background Processes"
         tags: "running_on_beats"
-        platforms: ["debian_arm64", "debian_amd64", "oracle_linux8", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "debian_11_amd64", "ubuntu_22_04_amd64", "oracle_linux8", "sles15"]
+  - suite: "kubernetes-autodiscover"
+    provider: "docker"
+    scenarios:
+      - name: "kubernetes autodiscover with elastic-agent"
+        tags: "elastic-agent"
+        platforms: ["debian_10_amd64"]

.ci/.e2e-tests.yaml

@@ -5,54 +5,54 @@ SUITES:
     scenarios:
       - name: "APM Server"
         tags: "apm-server"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "Filebeat"
         tags: "filebeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "Metricbeat"
         tags: "metricbeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
   - suite: "fleet"
     scenarios:
       - name: "Fleet"
         tags: "fleet_mode"
-        platforms: ["centos8_arm64", "centos8_amd64", "debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["centos8_arm64", "centos8_amd64", "debian_10_arm64", "debian_10_amd64", "sles15", "debian_11_amd64", "ubuntu_22_04_amd64", "windows2019"]
       - name: "Integrations"
         tags: "integrations"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "sles15", "debian_11_amd64", "ubuntu_22_04_amd64"]
       - name: "APM Integration"
         tags: "apm_server"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "Linux Integration"
         tags: "linux_integration"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "sles15", "debian_11_amd64", "ubuntu_22_04_amd64"]
       # - name: "Permissions Output change"
       #   tags: "permission_change"
-      #   platforms: ["debian_arm64", "debian_amd64", "sles15"]
+      #   platforms: ["debian_10_arm64", "debian_10_amd64", "sles15"]
       - name: "System Integration"
         tags: "system_integration"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "sles15", "debian_11_amd64", "ubuntu_22_04_amd64"]
       # - name: "Stand-alone"
       #   tags: "stand_alone_mode"
       #   platforms: ["fleet_elastic_pkg"]
       - name: "Backend Processes"
         tags: "backend_processes"
-        platforms: ["debian_arm64", "debian_amd64", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "sles15", "debian_11_amd64", "ubuntu_22_04_amd64"]
       - name: "Beats Background Processes"
         tags: "running_on_beats"
-        platforms: ["debian_arm64", "debian_amd64", "oracle_linux8", "sles15"]
+        platforms: ["debian_10_arm64", "debian_10_amd64", "oracle_linux8", "sles15", "debian_11_amd64", "ubuntu_22_04_amd64"]
   - suite: "kubernetes-autodiscover"
     provider: "docker"
     scenarios:
       - name: "kubernetes autodiscover with elastic-agent"
         tags: "elastic-agent"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "kubernetes autodiscover with filebeat"
         tags: "filebeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "kubernetes autodiscover with heartbeat"
         tags: "heartbeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]
       - name: "kubernetes autodiscover with metricbeat"
         tags: "metricbeat"
-        platforms: ["debian_amd64"]
+        platforms: ["debian_10_amd64"]

.ci/Jenkinsfile

@@ -26,7 +26,7 @@ pipeline {
     RUN_ID = UUID.randomUUID().toString()
   }
   options {
-    timeout(time: 90, unit: 'MINUTES')
+    timeout(time: 120, unit: 'MINUTES')
     buildDiscarder(logRotator(numToKeepStr: '200', artifactNumToKeepStr: '30', daysToKeepStr: '30'))
     timestamps()
     ansiColor('xterm')
@@ -50,16 +50,16 @@ pipeline {
     booleanParam(name: "notifyOnGreenBuilds", defaultValue: false, description: "If it's needed to notify to Slack with green builds.")
     string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels')
     string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz')
-    string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '8.3.0-19aba912-SNAPSHOT', description: 'SemVer version of the Elastic Agent to be used for the tests. You can use here the tag of your PR to test your changes')
-    string(name: 'BEAT_VERSION', defaultValue: '8.3.0-19aba912-SNAPSHOT', description: 'SemVer version of the Beat to be used for the tests. You can use here the tag of your PR to test your changes')
+    string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '8.4.0-42ce0eef-SNAPSHOT', description: 'SemVer version of the Elastic Agent to be used for the tests. You can use here the tag of your PR to test your changes')
+    string(name: 'BEAT_VERSION', defaultValue: '8.4.0-42ce0eef-SNAPSHOT', description: 'SemVer version of the Beat to be used for the tests. You can use here the tag of your PR to test your changes')
     choice(name: 'LOG_LEVEL', choices: ['TRACE', 'DEBUG', 'INFO'], description: 'Log level to be used')
     choice(name: 'TIMEOUT_FACTOR', choices: ['5', '3', '7', '11'], description: 'Max number of minutes for timeout backoff strategies')
     string(name: 'KIBANA_VERSION', defaultValue: '', description: 'Docker tag of the kibana to be used for the tests. It will refer to an image related to a Kibana PR, under the Observability-CI namespace')
-    string(name: 'STACK_VERSION', defaultValue: '8.3.0-19aba912-SNAPSHOT', description: 'SemVer version of the stack to be used for the tests.')
-    string(name: 'HELM_CHART_VERSION', defaultValue: '7.11.2', description: 'SemVer version of Helm chart to be used.')
-    string(name: 'HELM_VERSION', defaultValue: '3.5.2', description: 'SemVer version of Helm to be used.')
-    string(name: 'KIND_VERSION', defaultValue: '0.10.0', description: 'SemVer version of Kind to be used.')
-    string(name: 'KUBERNETES_VERSION', defaultValue: '1.18.2', description: 'SemVer version of Kubernetes to be used.')
+    string(name: 'STACK_VERSION', defaultValue: '8.4.0-42ce0eef-SNAPSHOT', description: 'SemVer version of the stack to be used for the tests.')
+    string(name: 'HELM_CHART_VERSION', defaultValue: '7.17.3', description: 'SemVer version of Helm chart to be used.')
+    string(name: 'HELM_VERSION', defaultValue: '3.9.0', description: 'SemVer version of Helm to be used.')
+    string(name: 'KIND_VERSION', defaultValue: '0.14.0', description: 'SemVer version of Kind to be used.')
+    string(name: 'KUBERNETES_VERSION', defaultValue: '1.24.0', description: 'SemVer version of Kubernetes to be used.')
     string(name: 'GITHUB_CHECK_NAME', defaultValue: '', description: 'Name of the GitHub check to be updated. Only if this build is triggered from another parent stream.')
     string(name: 'GITHUB_CHECK_REPO', defaultValue: 'elastic-agent', description: 'Name of the GitHub repo to be updated. Only if this build is triggered from another parent stream.')
     string(name: 'GITHUB_CHECK_SHA1', defaultValue: '', description: 'Git SHA for the Beats upstream project (branch or PR)')
@@ -364,6 +364,14 @@ def getNodeIp(nodeType){
   return sh(label: "Get IP address of the ${nodeType}", script: "cat ${REAL_BASE_DIR}/.ci/.${nodeType}-host-ip", returnStdout: true)
 }
 
+def getRemoteE2EPath(testRunner, platform) {
+  if (platform.contains("windows")) {
+    return "C:\\Users\\${testRunner.user}\\e2e-testing\\"
+  }
+
+  return "/home/${testRunner.user}/e2e-testing/"
+}
+
 def getMachineInfo(platform){
   def machineYaml = readYaml(file: "${env.REAL_BASE_DIR}/.ci/.e2e-platforms.yaml")
   def machines = machineYaml['PLATFORMS']
@@ -521,13 +529,25 @@ def generateFunctionalTestStep(Map args = [:]){
               ciBuild() {
                   sh(label: 'Start node', script: "make -C .ci provision-node")
               }
+
+              if (platform.contains("windows")) {
+                // Ansible wait_for module is not enough to mitigate the timeout
+                log(level: 'DEBUG', text: "Sleeping 300 seconds on Windows so that SSH is accessible in the remote instance.")
+                sleep(300)
+              }
+
               ciBuild() {
                 retryWithSleep(retries: 3, seconds: 5, backoff: true){
                   sh(label: 'Configure node for testing', script: "make -C .ci setup-node")
                 }
               }
               ciBuild() {
+                if (platform.contains("windows")) {
+                  sh(label: 'Run tests in the node', script: "make -C .ci run-tests-win")
+                }
+                else {
                   sh(label: 'Run tests in the node', script: "make -C .ci run-tests")
+                }
               }
             }
           }