Merge branch 'main' into 2338-bring-back-windows-support

* main: Add test scenario for elastic agent tags (elastic#2552) bump stack version 8.4.0-3525aaf7 (elastic#2597) bump stack version 8.4.0-df27a645 (elastic#2591) fix: agent directories must belong to root (elastic#2590) chore: bump maintenance versions in mergify (elastic#2563)
mdelapenya · Jun 3, 2022 · be919ae · be919ae
2 parents 59b9a3c + 5e682cf
commit be919ae
Show file tree

Hide file tree

Showing 27 changed files with 179 additions and 47 deletions.
diff --git a/.ci/Jenkinsfile b/.ci/Jenkinsfile
@@ -47,12 +47,12 @@ pipeline {
     booleanParam(name: "notifyOnGreenBuilds", defaultValue: false, description: "If it's needed to notify to Slack with green builds.")
     string(name: 'SLACK_CHANNEL', defaultValue: 'observablt-bots', description: 'The Slack channel(s) where errors will be posted. For multiple channels, use a comma-separated list of channels')
     string(name: 'ELASTIC_AGENT_DOWNLOAD_URL', defaultValue: '', description: 'If present, it will override the download URL for the Elastic agent artifact. (I.e. https://snapshots.elastic.co/8.0.0-59098054/downloads/beats/elastic-agent/elastic-agent-8.0.0-SNAPSHOT-linux-x86_64.tar.gz')
-    string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '8.3.0-e4aa1f83-SNAPSHOT', description: 'SemVer version of the Elastic Agent to be used for the tests. You can use here the tag of your PR to test your changes')
-    string(name: 'BEAT_VERSION', defaultValue: '8.3.0-e4aa1f83-SNAPSHOT', description: 'SemVer version of the Beat to be used for the tests. You can use here the tag of your PR to test your changes')
+    string(name: 'ELASTIC_AGENT_VERSION', defaultValue: '8.4.0-3525aaf7-SNAPSHOT', description: 'SemVer version of the Elastic Agent to be used for the tests. You can use here the tag of your PR to test your changes')
+    string(name: 'BEAT_VERSION', defaultValue: '8.4.0-3525aaf7-SNAPSHOT', description: 'SemVer version of the Beat to be used for the tests. You can use here the tag of your PR to test your changes')
     choice(name: 'LOG_LEVEL', choices: ['TRACE', 'DEBUG', 'INFO'], description: 'Log level to be used')
     choice(name: 'TIMEOUT_FACTOR', choices: ['5', '3', '7', '11'], description: 'Max number of minutes for timeout backoff strategies')
     string(name: 'KIBANA_VERSION', defaultValue: '', description: 'Docker tag of the kibana to be used for the tests. It will refer to an image related to a Kibana PR, under the Observability-CI namespace')
-    string(name: 'STACK_VERSION', defaultValue: '8.3.0-e4aa1f83-SNAPSHOT', description: 'SemVer version of the stack to be used for the tests.')
+    string(name: 'STACK_VERSION', defaultValue: '8.4.0-3525aaf7-SNAPSHOT', description: 'SemVer version of the stack to be used for the tests.')
     string(name: 'HELM_CHART_VERSION', defaultValue: '7.17.3', description: 'SemVer version of Helm chart to be used.')
     string(name: 'HELM_VERSION', defaultValue: '3.9.0', description: 'SemVer version of Helm to be used.')
     string(name: 'KIND_VERSION', defaultValue: '0.12.0', description: 'SemVer version of Kind to be used.')

diff --git a/.ci/README.md b/.ci/README.md
@@ -141,7 +141,7 @@ Once you have created and set up the remote machines with the above instructions
 
 > When submitting the pull request with your user to enable the SSH access, please remember to add the right backport labels (ex. `backport-v8.2.0`) so that you will be able to SSH into the CI machines for all supported maintenance branches.
 
-To SSH into the machines, please use the following commads:
+To SSH into the machines, please use the following commands:
 
 ```shell
 export SSH_KEY="PATH_TO_YOUR_SSH_KEY_WITH_ACCESS_TO_AWS"

diff --git a/.gitignore b/.gitignore
@@ -8,3 +8,4 @@ dist/
 .venv
 hosts
 *-sshhosts
+.idea
diff --git a/.mergify.yml b/.mergify.yml
@@ -18,29 +18,29 @@ pull_request_rules:
             git merge upstream/{{base}}
             git push upstream {{head}}
             ```
-  - name: backport patches to 8.2 branch
+  - name: backport patches to 8.3 branch
     conditions:
       - merged
       - base=main
-      - label=backport-v8.2.0
+      - label=backport-v8.3.0
     actions:
       backport:
         assignees:
           - "{{ author }}"
         branches:
-          - "8.2"
+          - "8.3"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
-  - name: backport patches to 8.1 branch
+  - name: backport patches to 8.2 branch
     conditions:
       - merged
       - base=main
-      - label=backport-v8.1.0
+      - label=backport-v8.2.0
     actions:
       backport:
         assignees:
           - "{{ author }}"
         branches:
-          - "8.1"
+          - "8.2"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
   - name: backport patches to 7.17 branch
     conditions:

diff --git a/.stack-version b/.stack-version
@@ -1 +1 @@
-8.3.0-e4aa1f83-SNAPSHOT
+8.4.0-3525aaf7-SNAPSHOT
diff --git a/cli/config/compose/profiles/fleet/docker-compose.yml b/cli/config/compose/profiles/fleet/docker-compose.yml
@@ -18,7 +18,7 @@ services:
       - xpack.security.authc.token.timeout=60m
       - ELASTIC_USERNAME=admin
       - ELASTIC_PASSWORD=changeme
-    image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-8.3.0-e4aa1f83-SNAPSHOT}"
+    image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-8.4.0-3525aaf7-SNAPSHOT}"
     platform: ${stackPlatform:-linux/amd64}
     ports:
       - "9200:9200"
@@ -34,7 +34,7 @@ services:
       test: "curl -f http://localhost:5601/login | grep kbn-injected-metadata 2>&1 >/dev/null"
       retries: 600
       interval: 1s
-    image: "docker.elastic.co/${kibanaDockerNamespace:-kibana}/kibana:${kibanaVersion:-8.3.0-e4aa1f83-SNAPSHOT}"
+    image: "docker.elastic.co/${kibanaDockerNamespace:-kibana}/kibana:${kibanaVersion:-8.4.0-3525aaf7-SNAPSHOT}"
     platform: ${stackPlatform:-linux/amd64}
     ports:
       - "5601:5601"

diff --git a/cli/config/compose/profiles/metricbeat/docker-compose.yml b/cli/config/compose/profiles/metricbeat/docker-compose.yml
@@ -9,7 +9,7 @@ services:
       - xpack.monitoring.collection.enabled=true
       - ELASTIC_USERNAME=elastic
       - ELASTIC_PASSWORD=changeme
-    image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-8.3.0-e4aa1f83-SNAPSHOT}"
+    image: "docker.elastic.co/elasticsearch/elasticsearch:${stackVersion:-8.4.0-3525aaf7-SNAPSHOT}"
     platform: ${stackPlatform:-linux/amd64}
     ports:
       - "9200:9200"
diff --git a/cli/config/compose/services/elastic-agent/cloud/docker-compose.yml b/cli/config/compose/services/elastic-agent/cloud/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '2.4'
 services:
   elastic-agent:
-    image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.3.0-e4aa1f83-SNAPSHOT}"
+    image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.4.0-3525aaf7-SNAPSHOT}"
     depends_on:
       elasticsearch:
         condition: service_healthy

diff --git a/cli/config/compose/services/elastic-agent/docker-compose.yml b/cli/config/compose/services/elastic-agent/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '2.4'
 services:
   elastic-agent:
-    image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.3.0-e4aa1f83-SNAPSHOT}"
+    image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.4.0-3525aaf7-SNAPSHOT}"
     depends_on:
       elasticsearch:
         condition: service_healthy

diff --git a/cli/config/compose/services/elastic-agent/fleet-server/docker-compose.yml b/cli/config/compose/services/elastic-agent/fleet-server/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '2.4'
 services:
   fleet-server:
-    image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.3.0-e4aa1f83-SNAPSHOT}"
+    image: "docker.elastic.co/${elasticAgentDockerNamespace:-beats}/elastic-agent${elasticAgentDockerImageSuffix}:${elasticAgentTag:-8.4.0-3525aaf7-SNAPSHOT}"
     depends_on:
       elasticsearch:
         condition: service_healthy

diff --git a/cli/config/compose/services/elasticsearch/docker-compose.yml b/cli/config/compose/services/elasticsearch/docker-compose.yml
@@ -9,7 +9,7 @@ services:
       - xpack.monitoring.collection.enabled=true
       - ELASTIC_USERNAME=elastic
       - ELASTIC_PASSWORD=changeme
-    image: "docker.elastic.co/observability-ci/elasticsearch:${elasticsearchTag:-8.3.0-e4aa1f83-SNAPSHOT}"
+    image: "docker.elastic.co/observability-ci/elasticsearch:${elasticsearchTag:-8.4.0-3525aaf7-SNAPSHOT}"
     platform: ${elasticsearchPlatform:-linux/amd64}
     ports:
       - "9200:9200"

diff --git a/cli/config/compose/services/kibana/docker-compose.yml b/cli/config/compose/services/kibana/docker-compose.yml
@@ -9,6 +9,6 @@ services:
       test: "curl -f http://localhost:5601/login | grep kbn-injected-metadata 2>&1 >/dev/null"
       retries: 600
       interval: 1s
-    image: "docker.elastic.co/kibana/kibana:${kibanaTag:-8.3.0-e4aa1f83-SNAPSHOT}"
+    image: "docker.elastic.co/kibana/kibana:${kibanaTag:-8.4.0-3525aaf7-SNAPSHOT}"
     ports:
       - "5601:5601"
diff --git a/cli/config/compose/services/metricbeat/docker-compose.yml b/cli/config/compose/services/metricbeat/docker-compose.yml
@@ -14,7 +14,7 @@ services:
     ]
     environment:
       - BEAT_STRICT_PERMS=${beatStricPerms:-false}
-    image: "docker.elastic.co/${metricbeatDockerNamespace:-beats}/metricbeat:${metricbeatTag:-8.3.0-e4aa1f83-SNAPSHOT}"
+    image: "docker.elastic.co/${metricbeatDockerNamespace:-beats}/metricbeat:${metricbeatTag:-8.4.0-3525aaf7-SNAPSHOT}"
     labels:
       co.elastic.logs/module: "${serviceName}"
     platform: ${stackPlatform:-linux/amd64}

diff --git a/cli/config/kubernetes/base/elasticsearch/deployment.yaml b/cli/config/kubernetes/base/elasticsearch/deployment.yaml
@@ -16,7 +16,7 @@ spec:
     spec:
       containers:
       - name: elasticsearch
-        image: docker.elastic.co/elasticsearch/elasticsearch:8.3.0-e4aa1f83-SNAPSHOT
+        image: docker.elastic.co/elasticsearch/elasticsearch:8.4.0-3525aaf7-SNAPSHOT
         envFrom:
           - configMapRef:
               name: elasticsearch-config

diff --git a/cli/config/kubernetes/base/fleet-server/deployment.yaml b/cli/config/kubernetes/base/fleet-server/deployment.yaml
@@ -16,7 +16,7 @@ spec:
     spec:
       containers:
       - name: fleet-server
-        image: docker.elastic.co/beats/elastic-agent:8.3.0-e4aa1f83-SNAPSHOT
+        image: docker.elastic.co/beats/elastic-agent:8.4.0-3525aaf7-SNAPSHOT
         env:
         - name: FLEET_SERVER_ENABLE
           value: "1"

diff --git a/cli/config/kubernetes/base/kibana/deployment.yaml b/cli/config/kubernetes/base/kibana/deployment.yaml
@@ -16,7 +16,7 @@ spec:
     spec:
       containers:
       - name: kibana
-        image: docker.elastic.co/kibana/kibana:8.3.0-e4aa1f83-SNAPSHOT
+        image: docker.elastic.co/kibana/kibana:8.4.0-3525aaf7-SNAPSHOT
         env:
         - name: ELASTICSEARCH_URL
           value: http://elasticsearch:9200

diff --git a/e2e/_suites/fleet/features/fleet_mode.feature b/e2e/_suites/fleet/features/fleet_mode.feature
@@ -12,6 +12,12 @@ Scenario Outline: Deploying the agent
   Then the agent is listed in Fleet as "online"
     And system package dashboards are listed in Fleet
 
+@install-including-tags
+Scenario Outline: Deploying the agent including command line --tag for tags
+  When an agent is deployed to Fleet with "tar" installer and "--tag=production,linux" flags
+  Then the agent is listed in Fleet as "online"
+    And the elastic agent index contains the tags
+
 # @enroll
 # Scenario Outline: Deploying the agent with enroll and then run on rpm and deb
 #   Given an agent is deployed to Fleet

diff --git a/e2e/_suites/fleet/fleet.go b/e2e/_suites/fleet/fleet.go
@@ -62,8 +62,9 @@ type FleetTestSuite struct {
 	AgentStoppedDate             time.Time
 	RuntimeDependenciesStartDate time.Time
 	// instrumentation
-	currentContext context.Context
-	DefaultAPIKey  string
+	currentContext    context.Context
+	DefaultAPIKey     string
+	ElasticAgentFlags string
 }
 
 func (fts *FleetTestSuite) getDeployer() deploy.Deployment {
@@ -148,6 +149,7 @@ func (fts *FleetTestSuite) afterScenario() {
 	fts.Image = ""
 	fts.StandAlone = false
 	fts.BeatsProcess = ""
+	fts.ElasticAgentFlags = ""
 }
 
 // beforeScenario creates the state needed by a scenario
@@ -255,6 +257,7 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) {
 	s.Step(`^a "([^"]*)" agent is deployed to Fleet$`, fts.anAgentIsDeployedToFleet)
 	s.Step(`^an agent is deployed to Fleet on top of "([^"]*)"$`, fts.anAgentIsDeployedToFleetOnTopOfBeat)
 	s.Step(`^an agent is deployed to Fleet with "([^"]*)" installer$`, fts.anAgentIsDeployedToFleetWithInstaller)
+	s.Step(`^an agent is deployed to Fleet with "([^"]*)" installer and "([^"]*)" flags$`, fts.anAgentIsDeployedToFleetWithInstallerAndTags)
 	s.Step(`^a "([^"]*)" stale agent is deployed to Fleet with "([^"]*)" installer$`, fts.anStaleAgentIsDeployedToFleetWithInstaller)
 	s.Step(`^agent is in "([^"]*)" version$`, fts.agentInVersion)
 	s.Step(`^agent is upgraded to "([^"]*)" version$`, fts.anAgentIsUpgradedToVersion)
@@ -272,6 +275,9 @@ func (fts *FleetTestSuite) contributeSteps(s *godog.ScenarioContext) {
 	s.Step(`^a Linux data stream exists with some data$`, fts.checkDataStream)
 	s.Step(`^the agent is enrolled into "([^"]*)" policy$`, fts.agentRunPolicy)
 
+	//flags steps
+	s.Step(`^the elastic agent index contains the tags$`, fts.tagsAreInTheElasticAgentIndex)
+
 	// endpoint steps
 	s.Step(`^the "([^"]*)" integration is "([^"]*)" in the policy$`, fts.theIntegrationIsOperatedInThePolicy)
 	s.Step(`^the "([^"]*)" datasource is shown in the policy as added$`, fts.thePolicyShowsTheDatasourceAdded)
@@ -537,6 +543,60 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstaller(installerType s
 	return fts.anAgentIsDeployedToFleetWithInstallerAndFleetServer(installerType)
 }
 
+// supported installers: tar, rpm, deb
+func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndTags(installerType string, flags string) error {
+	fts.BeatsProcess = ""
+
+	// FIXME: We need to cleanup the steps to support different operating systems
+	// for now we will force the zip installer type when the agent is running on windows
+	if runtime.GOOS == "windows" && common.Provider == "remote" {
+		installerType = "zip"
+	}
+	fts.ElasticAgentFlags = flags
+	return fts.anAgentIsDeployedToFleetWithInstallerAndFleetServer(installerType)
+}
+
+func (fts *FleetTestSuite) tagsAreInTheElasticAgentIndex() error {
+	var tagsArray []string
+	//ex of flags  "--tag production,linux" or "--tag=production,linux"
+	if fts.ElasticAgentFlags != "" {
+		tags := strings.TrimPrefix(fts.ElasticAgentFlags, "--tag")
+		tags = strings.TrimPrefix(tags, "=")
+		tags = strings.ReplaceAll(tags, " ", "")
+		tagsArray = strings.Split(tags, ",")
+	}
+	if len(tagsArray) == 0 {
+		return errors.Errorf("no tags were found, ElasticAgentFlags value %s", fts.ElasticAgentFlags)
+	}
+
+	var tagTerms []map[string]interface{}
+	for _, tag := range tagsArray {
+		tagTerms = append(tagTerms, map[string]interface{}{
+			"term": map[string]interface{}{
+				"tags": tag,
+			},
+		})
+	}
+
+	query := map[string]interface{}{
+		"query": map[string]interface{}{
+			"bool": map[string]interface{}{
+				"must": tagTerms,
+			},
+		},
+	}
+
+	indexName := ".fleet-agents"
+
+	_, err := elasticsearch.WaitForNumberOfHits(context.Background(), indexName, query, 1, 3*time.Minute)
+	if err != nil {
+		log.WithFields(log.Fields{
+			"error": err,
+		}).Warn(elasticsearch.WaitForIndices())
+	}
+	return err
+}
+
 func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(installerType string) error {
 	log.WithFields(log.Fields{
 		"installer": installerType,
@@ -564,7 +624,7 @@ func (fts *FleetTestSuite) anAgentIsDeployedToFleetWithInstallerAndFleetServer(i
 	}
 
 	agentInstaller, _ := installer.Attach(fts.currentContext, fts.getDeployer(), agentService, installerType)
-	err = deployAgentToFleet(fts.currentContext, agentInstaller, fts.CurrentToken)
+	err = deployAgentToFleet(fts.currentContext, agentInstaller, fts.CurrentToken, fts.ElasticAgentFlags)
 	if err != nil {
 		return err
 	}
@@ -1063,7 +1123,7 @@ func (fts *FleetTestSuite) theAgentIsReenrolledOnTheHost() error {
 	agentService := deploy.NewServiceRequest(common.ElasticAgentServiceName)
 	agentInstaller, _ := installer.Attach(fts.currentContext, fts.getDeployer(), agentService, fts.InstallerType)
 
-	err := agentInstaller.Enroll(fts.currentContext, fts.CurrentToken)
+	err := agentInstaller.Enroll(fts.currentContext, fts.CurrentToken, fts.ElasticAgentFlags)
 	if err != nil {
 		return err
 	}
@@ -1415,7 +1475,7 @@ func (fts *FleetTestSuite) anAttemptToEnrollANewAgentFails() error {
 	}
 
 	agentInstaller, _ := installer.Attach(fts.currentContext, fts.getDeployer(), agentService, fts.InstallerType)
-	err = deployAgentToFleet(fts.currentContext, agentInstaller, fts.CurrentToken)
+	err = deployAgentToFleet(fts.currentContext, agentInstaller, fts.CurrentToken, fts.ElasticAgentFlags)
 
 	if err == nil {
 		err = fmt.Errorf("the agent was enrolled although the token was previously revoked")
@@ -1546,7 +1606,7 @@ func (fts *FleetTestSuite) checkDataStream() error {
 	return err
 }
 
-func deployAgentToFleet(ctx context.Context, agentInstaller deploy.ServiceOperator, token string) error {
+func deployAgentToFleet(ctx context.Context, agentInstaller deploy.ServiceOperator, token string, flags string) error {
 	err := agentInstaller.Preinstall(ctx)
 	if err != nil {
 		return err
@@ -1557,7 +1617,7 @@ func deployAgentToFleet(ctx context.Context, agentInstaller deploy.ServiceOperat
 		return err
 	}
 
-	err = agentInstaller.Enroll(ctx, token)
+	err = agentInstaller.Enroll(ctx, token, flags)
 	if err != nil {
 		return err
 	}

diff --git a/internal/common/defaults.go b/internal/common/defaults.go
@@ -36,7 +36,7 @@ const FleetProfileName = "fleet"
 const FleetServerAgentServiceName = "fleet-server"
 
 // BeatVersionBase is the base version of the Beat to use
-var BeatVersionBase = "8.3.0-e4aa1f83-SNAPSHOT"
+var BeatVersionBase = "8.4.0-3525aaf7-SNAPSHOT"
 
 // BeatVersion is the version of the Beat to use
 // It can be overriden by BEAT_VERSION env var

diff --git a/internal/deploy/base.go b/internal/deploy/base.go
@@ -31,10 +31,10 @@ type Deployment interface {
 
 // ServiceOperator represents the operations that can be performed by a service
 type ServiceOperator interface {
-	AddFiles(ctx context.Context, files []string) error      // adds files to service environment
-	Enroll(ctx context.Context, token string) error          // handle any enrollment/registering of service
-	Exec(ctx context.Context, args []string) (string, error) // exec arbitrary commands in service environment
-	Inspect() (ServiceOperatorManifest, error)               // returns manifest for package
+	AddFiles(ctx context.Context, files []string) error                // adds files to service environment
+	Enroll(ctx context.Context, token string, extraFlags string) error // handle any enrollment/registering of service
+	Exec(ctx context.Context, args []string) (string, error)           // exec arbitrary commands in service environment
+	Inspect() (ServiceOperatorManifest, error)                         // returns manifest for package
 	Install(ctx context.Context) error
 	InstallCerts(ctx context.Context) error
 	Logs(ctx context.Context) error
@@ -71,6 +71,7 @@ type ServiceManifest struct {
 
 // ServiceInstallerMetadata information about the installer
 type ServiceInstallerMetadata struct {
+	AgentPath     string
 	Arch          string
 	Docker        bool
 	FileExtension string