Merge branch 'master' into 23928-fix-docker-entrypoint

.backportrc.json

@@ -1,6 +1,6 @@
 {
   "upstream": "elastic/beats",
-  "branches": [ { "name": "7.x", "checked": true }, "7.10" ],
+  "branches": [ { "name": "7.x", "checked": true }, "7.11", "7.10" ],
   "labels": ["backport"],
   "autoAssign": true,
   "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"

.ci/heartbeat-synthetics.groovy

@@ -0,0 +1,122 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent { label 'ubuntu-18 && immutable' }
+  environment {
+    BASE_DIR = 'src/github.com/elastic/beats'
+    DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    DOCKER_REGISTRY = 'docker.elastic.co'
+    SYNTHETICS = "-synthetics"
+    PIPELINE_LOG_LEVEL = "INFO"
+    BEATS_FOLDER = "x-pack/heartbeat"
+  }
+  options {
+    timeout(time: 3, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    disableConcurrentBuilds()
+  }
+  triggers {
+    issueCommentTrigger('(?i)^\\/packag[ing|e] synthetics$')
+  }
+  parameters {
+    booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.')
+  }
+  stages {
+    stage('Checkout') {
+      options { skipDefaultCheckout() }
+      steps {
+        deleteDir()
+        gitCheckout(basedir: "${BASE_DIR}")
+        setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim())
+      }
+    }
+    stage('Build and test'){
+      steps {
+        withGithubNotify(context: "Build and test") {
+          withBeatsEnv{
+            dir("${env.BEATS_FOLDER}") {
+              sh(label: 'Build and test', script: 'mage build test')
+            }
+          }
+        }
+      }
+    }
+    stage('Package Linux'){
+      environment {
+        HOME = "${env.WORKSPACE}"
+        PLATFORMS = [
+          '+all',
+          'linux/amd64'
+        ].join(' ')
+      }
+      steps {
+        withGithubNotify(context: "Packaging Linux ${BEATS_FOLDER}") {
+          release()
+          pushCIDockerImages()
+        }
+      }
+    }
+  }
+}
+
+def pushCIDockerImages(){
+  catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
+    tagAndPush('heartbeat')
+  }
+}
+
+def tagAndPush(name){
+  def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim()
+
+  def tagName = "${libbetaVer}"
+  def oldName = "${DOCKER_REGISTRY}/beats/${name}:${libbetaVer}"
+  def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${libbetaVer}${env.SYNTHETICS}"
+  def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}:${env.GIT_BASE_COMMIT}"
+  dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+  retry(3){
+    sh(label:'Change tag and push', script: """
+      docker tag ${oldName} ${newName}
+      docker push ${newName}
+      docker tag ${oldName} ${commitName}
+      docker push ${commitName}
+    """)
+  }
+}
+
+def release(){
+  withBeatsEnv(){
+    dir("${env.BEATS_FOLDER}") {
+      sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+    }
+  }
+}
+
+/**
+* There is a specific folder structure in https://staging.elastic.co/ and https://artifacts.elastic.co/downloads/
+* therefore the storage bucket in GCP should follow the same folder structure.
+* This is required by https://github.com/elastic/beats-tester
+* e.g.
+* baseDir=name -> return name
+* baseDir=name1/name2/name3-> return name2
+*/
+def getBeatsName(baseDir) {
+  return baseDir.replace('x-pack/', '')
+}
+
+def withBeatsEnv(Closure body) {
+  withMageEnv(){
+    withEnv([
+      "PYTHON_ENV=${WORKSPACE}/python-env"
+    ]) {
+      dir("${env.BASE_DIR}"){
+        body()
+      }
+    }
+  }
+}

.ci/jobs/beats-schedule-weekly.yml

@@ -0,0 +1,27 @@
+---
+- job:
+    name: beats-schedule-weekly
+    display-name: Jobs scheduled weekly (Sunday)
+    description: Jobs scheduled weekly (Sunday)
+    view: Beats
+    project-type: pipeline
+    parameters:
+      - string:
+          name: branch_specifier
+          default: master
+          description: the Git branch specifier to build
+    pipeline-scm:
+      script-path: .ci/schedule-weekly.groovy
+      scm:
+        - git:
+            url: git@github.com:elastic/beats.git
+            refspec: +refs/heads/*:refs/remotes/origin/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/beats.git
+            branches:
+              - $branch_specifier
+    triggers:
+      - timed: 'H H(1-4) * * 0'

.ci/packaging.groovy

@@ -40,6 +40,7 @@ pipeline {
   parameters {
     booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
     booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.')
+    booleanParam(name: 'arm', defaultValue: true, description: 'Allow ARM stages.')
   }
   stages {
     stage('Filter build') {
@@ -83,12 +84,13 @@ pipeline {
               }
             }
             setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim())
+            // Stash without any build/dependencies context to support different architectures.
+            stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
             withMageEnv(){
               dir("${BASE_DIR}"){
                 setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
               }
             }
-            stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
           }
         }
         stage('Build Packages'){
@@ -172,12 +174,73 @@ pipeline {
                 }
                 steps {
                   withGithubNotify(context: "Packaging MacOS ${BEATS_FOLDER}") {
-                    deleteDir()
+                    deleteWorkspace()
                     withMacOSEnv(){
                       release()
                     }
                   }
                 }
+                post {
+                  always {
+                    // static workers require this
+                    deleteWorkspace()
+                  }
+                }
+              }
+            }
+          }
+        }
+        stage('Build Packages ARM'){
+          matrix {
+            axes {
+              axis {
+                name 'BEATS_FOLDER'
+                values (
+                  'auditbeat',
+                  'filebeat',
+                  'heartbeat',
+                  'journalbeat',
+                  'metricbeat',
+                  'packetbeat',
+                  'x-pack/auditbeat',
+                  'x-pack/elastic-agent',
+                  'x-pack/filebeat',
+                  'x-pack/heartbeat',
+                  'x-pack/metricbeat',
+                  'x-pack/packetbeat'
+                )
+              }
+            }
+            stages {
+              stage('Package Docker images for linux/arm64'){
+                agent { label 'arm' }
+                options { skipDefaultCheckout() }
+                when {
+                  beforeAgent true
+                  expression {
+                    return params.arm
+                  }
+                }
+                environment {
+                  HOME = "${env.WORKSPACE}"
+                  PACKAGES = "docker"
+                  PLATFORMS = [
+                    'linux/arm64',
+                  ].join(' ')
+                }
+                steps {
+                  withGithubNotify(context: "Packaging linux/arm64 ${BEATS_FOLDER}") {
+                    deleteWorkspace()
+                    release()
+                    pushCIDockerImages()
+                  }
+                }
+                post {
+                  always {
+                    // static workers require this
+                    deleteWorkspace()
+                  }
+                }
               }
             }
           }
@@ -339,14 +402,14 @@ def triggerE2ETests(String suite) {
     booleanParam(name: 'forceSkipGitChecks', value: true),
     booleanParam(name: 'forceSkipPresubmit', value: true),
     booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+    booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
     string(name: 'runTestsSuites', value: suite),
     string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
     string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
     string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
   ]
   if (isPR()) {
     def version = "pr-${env.CHANGE_ID}"
-    parameters.push(booleanParam(name: 'ELASTIC_AGENT_USE_CI_SNAPSHOTS', value: true))
     parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}"))
     parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}"))
   }
@@ -408,14 +471,43 @@ def getBeatsName(baseDir) {
 }
 
 def withBeatsEnv(Closure body) {
+  unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
+  fixPermissions()
   withMageEnv(){
     withEnv([
       "PYTHON_ENV=${WORKSPACE}/python-env"
     ]) {
-      unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
       dir("${env.BASE_DIR}"){
         body()
       }
     }
   }
 }
+
+/**
+* This method fixes the filesystem permissions after the build has happenend. The reason is to
+* ensure any non-ephemeral workers don't have any leftovers that could cause some environmental
+* issues.
+*/
+def deleteWorkspace() {
+  catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') {
+    fixPermissions()
+    deleteDir()
+  }
+}
+
+def fixPermissions() {
+  if(isUnix()) {
+    catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') {
+      dir("${env.BASE_DIR}") {
+        if (fileExists('script/fix_permissions.sh')) {
+          sh(label: 'Fix permissions', script: """#!/usr/bin/env bash
+            set +x
+            source ./dev-tools/common.bash
+            docker_setup
+            script/fix_permissions.sh ${WORKSPACE}""", returnStatus: true)
+        }
+      }
+    }
+  }
+}

.ci/schedule-weekly.groovy

@@ -0,0 +1,33 @@
+@Library('apm@current') _
+
+pipeline {
+  agent none
+  environment {
+    NOTIFY_TO = credentials('notify-to')
+    PIPELINE_LOG_LEVEL = 'INFO'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+  }
+  triggers {
+    cron('H H(1-4) * * 0')
+  }
+  stages {
+    stage('Nighly beats builds') {
+      steps {
+        build(quietPeriod: 0, job: 'Beats/beats/master', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+        build(quietPeriod: 1000, job: 'Beats/beats/7.x', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult(prComment: false)
+    }
+  }
+}

.ci/scripts/generate_build_table.py

@@ -31,6 +31,7 @@
                     if "withModule" in doc["stages"][stage]:
                         withModule = doc["stages"][stage]["withModule"]
                     if "when" in doc["stages"][stage]:
-                        when = f"optional"
+                        if "not_changeset_full_match" not in doc["stages"][stage]["when"]:
+                            when = "optional"
                     print("| {} | {} | `{}` | {} | `{}` | {} |".format(
                         module, stage, command, withModule, platforms, when))

.ci/scripts/install-tools.bat

@@ -79,4 +79,4 @@ gcc --version
 where gcc
 
 REM Reset the USERPROFILE
-SET USERPROFILE=%PREVIOUS_USERPROFILE%
+SET USERPROFILE=%PREVIOUS_USERPROFILE%

.go-version

@@ -1 +1 @@
-1.14.12
+1.15.7

CHANGELOG-developer.asciidoc

@@ -13,7 +13,7 @@ other Beats should be migrated.
 Note: This changelog was only started after the 6.3 release.
 
 === Beats version 8.0.0
-https://github.com/elastic/beats/compare/v7.x..master[Check the HEAD diff]
+https://github.com/elastic/beats/compare/7.x..master[Check the HEAD diff]
 
 ==== Breaking changes
  - Replace custom Pins type for a slice of string for defining the `ca_sha256` values.

CHANGELOG-developer.next.asciidoc

@@ -52,6 +52,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Replace `ACKCount`, `ACKEvents`, and `ACKLastEvent` callbacks with `ACKHandler` and interface in `beat.ClientConfig`. {pull}19632[19632]
 - Remove global ACK handler support via `SetACKHandler` from publisher pipeline. {pull}19632[19632]
 - Make implementing `Close` required for `reader.Reader` interfaces. {pull}20455[20455]
+- Remove `NumCPU` as clients should update the CPU count on the fly in case of config changes in a VM. {pull}23154[23154]
 
 ==== Bugfixes
 
@@ -104,3 +105,5 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Add packaging for docker image based on UBI minimal 8. {pull}20576[20576]
 - Make the mage binary used by the build process in the docker container to be statically compiled. {pull}20827[20827]
 - Update ecszap to v0.3.0 for using ECS 1.6.0 in logs {pull}22267[22267]
+- Add support for customized monitoring API. {pull}22605[22605]
+- Update Go version to 1.15.7. {pull}22495[22495]