[Elastic Agent] Add the ability to run the Fleet Server (#23736)

* Add the ability to run the Fleet Server. * Add test and changelog. * Fix changelog.
elastic · Feb 1, 2021 · d59f780 · d59f780
1 parent 5cb370e
commit d59f780
Show file tree

Hide file tree

Showing 10 changed files with 152 additions and 60 deletions.
diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml
@@ -108,6 +108,18 @@ shared:
         source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
+        skip_on_missing: true
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
+        mode: 0644
+        skip_on_missing: true
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
+        mode: 0644
+        skip_on_missing: true
 
 
 
@@ -201,6 +213,18 @@ shared:
         source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
+        skip_on_missing: true
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
+        mode: 0644
+        skip_on_missing: true
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
+        mode: 0644
+        skip_on_missing: true
 
   - &agent_binary_files
     '{{.BeatName}}{{.BinaryExt}}':
@@ -283,6 +307,18 @@ shared:
         source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
+        skip_on_missing: true
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512':
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
+        mode: 0644
+        skip_on_missing: true
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc':
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
+        mode: 0644
+        skip_on_missing: true
 
 
   # Binary package spec (zip for windows) for community beats.
@@ -334,6 +370,18 @@ shared:
         source: '{{.AgentDropPath}}/apm-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc'
         mode: 0644
         skip_on_missing: true
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
+        mode: 0644
+        skip_on_missing: true
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512':
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512'
+        mode: 0644
+        skip_on_missing: true
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc':
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc'
+        mode: 0644
+        skip_on_missing: true
 
   - &agent_docker_spec
     <<: *agent_binary_spec

diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc
@@ -64,3 +64,4 @@
 - Log level reloadable from fleet {pull}22690[22690]
 - Push log level downstream {pull}22815[22815]
 - Add metrics collection for Agent {pull}22793[22793]
+- Add support for Fleet Server {pull}23736[23736]
diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_decorator.go b/x-pack/elastic-agent/pkg/agent/application/fleet_decorator.go
@@ -54,7 +54,13 @@ func injectFleet(cfg *config.Config, hostInfo types.HostInfo, agentInfo *info.Ag
 			transpiler.NewKey("id", transpiler.NewStrVal(hostInfo.UniqueID)),
 		}))
 
-		fleet := transpiler.NewDict([]transpiler.Node{agent, token, kbn, host})
+		nodes := []transpiler.Node{agent, token, kbn, host}
+		server, ok := transpiler.Lookup(ast, "fleet.server")
+		if ok {
+			nodes = append(nodes, server)
+		}
+		fleet := transpiler.NewDict(nodes)
+
 		err = transpiler.Insert(rootAst, fleet, "fleet")
 		if err != nil {
 			return err

diff --git a/x-pack/elastic-agent/pkg/agent/program/program_test.go b/x-pack/elastic-agent/pkg/agent/program/program_test.go
@@ -384,8 +384,8 @@ func TestConfiguration(t *testing.T) {
 		err      bool
 	}{
 		"single_config": {
-			programs: []string{"filebeat", "heartbeat", "metricbeat", "endpoint", "packetbeat"},
-			expected: 5,
+			programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"},
+			expected: 6,
 		},
 		// "audit_config": {
 		// 	programs: []string{"auditbeat"},

diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go
diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-fleet-server.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-fleet-server.yml
@@ -0,0 +1,13 @@
+fleet:
+  agent:
+    id: fleet-agent-id
+
+output:
+  elasticsearch:
+    hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ]
+    username: fleet
+    password: fleetpassword
+
+inputs:
+  - id: fleet-server-id
+    type: fleet-server
diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml
@@ -8,6 +8,12 @@ fleet:
     protocol: https
     host: localhost:5601
     timeout: 30s
+  server:
+    output:
+      elasticsearch:
+        hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ]
+        username: fleet
+        password: fleetpassword
 
 outputs:
   default:
@@ -25,6 +31,11 @@ outputs:
     ca_sha256: "7lHLiyp4J8m9kw38SJ7SURJP4bXRZv/BNxyyXkCcE/M="
 
 inputs:
+- id: fleet-server-id
+  type: fleet-server
+  use_output: default
+  data_stream:
+    type: default
 - type: docker/metrics
   use_output: default
   streams:

diff --git a/x-pack/elastic-agent/pkg/core/app/descriptor.go b/x-pack/elastic-agent/pkg/core/app/descriptor.go
@@ -5,7 +5,6 @@
 package app
 
 import (
-	"path"
 	"path/filepath"
 	"strings"
 
@@ -29,7 +28,7 @@ func NewDescriptor(spec program.Spec, version string, config *artifact.Config, t
 		spec:         spec,
 		directory:    dir,
 		executionCtx: NewExecutionContext(spec.ServicePort, spec.Cmd, version, tags),
-		process:      specification(dir, spec.Cmd),
+		process:      specification(dir, spec),
 	}
 }
 
@@ -78,23 +77,12 @@ func (p *Descriptor) Directory() string {
 	return p.directory
 }
 
-func defaultSpec(dir string, binaryName string) ProcessSpec {
-	if !isKnownBeat(binaryName) {
-		return ProcessSpec{
-			BinaryPath: path.Join(dir, binaryName),
-		}
-	}
-
+func specification(dir string, spec program.Spec) ProcessSpec {
 	return ProcessSpec{
-		BinaryPath: path.Join(dir, binaryName),
-		Args:       []string{},
+		BinaryPath:    filepath.Join(dir, spec.Cmd),
+		Args:          spec.Args,
+		Configuration: nil,
 	}
-
-}
-
-func specification(directory, binaryName string) ProcessSpec {
-	defaultSpec := defaultSpec(directory, binaryName)
-	return populateSpec(directory, binaryName, defaultSpec)
 }
 
 func directory(spec program.Spec, version string, config *artifact.Config) string {
@@ -114,42 +102,3 @@ func directory(spec program.Spec, version string, config *artifact.Config) strin
 
 	return strings.TrimSuffix(path, suffix)
 }
-
-func isKnownBeat(name string) bool {
-	switch name {
-	case "filebeat":
-		fallthrough
-	case "metricbeat":
-		return true
-	}
-
-	return false
-}
-
-func populateSpec(dir, binaryName string, spec ProcessSpec) ProcessSpec {
-	var programSpec program.Spec
-	var found bool
-	binaryName = strings.ToLower(binaryName)
-	for _, prog := range program.Supported {
-		if binaryName != strings.ToLower(prog.Name) {
-			continue
-		}
-		found = true
-		programSpec = prog
-		break
-	}
-
-	if !found {
-		return spec
-	}
-
-	if programSpec.Cmd != "" {
-		spec.BinaryPath = filepath.Join(dir, programSpec.Cmd)
-	}
-
-	if len(programSpec.Args) > 0 {
-		spec.Args = programSpec.Args
-	}
-
-	return spec
-}
diff --git a/x-pack/elastic-agent/spec/endpoint.yml b/x-pack/elastic-agent/spec/endpoint.yml
@@ -53,6 +53,8 @@ rules:
         key: access_api_key
     - remove_key:
         key: kibana
+    - remove_key:
+        key: server
 
 - filter:
     selectors:

diff --git a/x-pack/elastic-agent/spec/fleet-server.yml b/x-pack/elastic-agent/spec/fleet-server.yml
@@ -0,0 +1,61 @@
+name: Fleet Server
+cmd: fleet-server
+args: ["--agent-mode"]
+artifact: fleet-server
+rules:
+  - fix_stream: {}
+
+  - filter_values:
+      selector: inputs
+      key: type
+      values:
+        - fleet-server
+
+  - filter_values:
+      selector: inputs
+      key: enabled
+      values:
+        - true
+
+  - remove_key:
+      key: output
+
+  - select_into:
+      selectors: [ fleet.server.output.elasticsearch ]
+      path: output
+
+  - map:
+      path: fleet
+      rules:
+        - remove_key:
+            key: enabled
+        - remove_key:
+            key: access_api_key
+        - remove_key:
+            key: kibana
+        - remove_key:
+            key: reporting
+        - remove_key:
+            key: server
+
+  - map:
+      path: inputs
+      rules:
+        - remove_key:
+            key: use_output
+        - remove_key:
+            key: data_stream
+        - remove_key:
+            key: data_stream.namespace
+        - remove_key:
+            key: data_stream.dataset
+        - remove_key:
+            key: streams
+
+  - filter:
+      selectors:
+        - fleet
+        - inputs
+        - output
+
+when: length(${fleet}) > 0 and length(${inputs}) > 0 and hasKey(${output}, 'elasticsearch')