Introduce apm-server.auth.* config (#5457)

* Introduce `apm-server.auth.*` config

Introduce the new AgentAuth config structure, which
holds API Key and secret token auth. Later we will
add "anonymous" auth here too.

We also introduce a new YAML naming scheme for the
config, `apm-server.auth.*`. The old config is
deprecated and copied across to the new config fields.

* docs: update config names

* apmpackage: update auth config keys

(cherry picked from commit fc60576)

# Conflicts:
#	changelogs/head.asciidoc

_meta/beat.yml

@@ -6,6 +6,19 @@ apm-server:
   # Defines the host and port the server is listening on. Use "unix:/path/to.sock" to listen on a unix domain socket.
   host: "{{ .listen_hostport }}"
 
+  # Agent authorization configuration. If no methods are defined, all requests will be allowed.
+  #auth:
+    # Agent authorization using Elasticsearch API Keys.
+    #api_key:
+      #enabled: false
+      #
+      # Restrict how many unique API keys are allowed per minute. Should be set to at least the amount of different
+      # API keys configured in your monitored services. Every unique API key triggers one request to Elasticsearch.
+      #limit: 100
+
+    # Define a shared secret token for authorizing agents using the "Bearer" authorization method.
+    #secret_token:
+
   # Maximum permitted size in bytes of a request's header accepted by the server to be processed.
   #max_header_size: 1048576
 
@@ -102,11 +115,17 @@ apm-server:
   # Agents include the token in the following format: Authorization: Bearer <secret-token>.
   # It is recommended to use an authorization token in combination with SSL enabled,
   # and save the token in the apm-server keystore.
+  #
+  # WARNING: This configuration is deprecated and replaced with `apm-server.auth.secret_token`, and will be removed
+  # in the 8.0 release. If that config is defined, this one will be ignored.
   #secret_token:
 
   # Enable API key authorization by setting enabled to true. By default API key support is disabled.
   # Agents include a valid API key in the following format: Authorization: ApiKey <token>.
   # The key must be the base64 encoded representation of the API key's "id:key".
+  #
+  # WARNING: This configuration is deprecated and replaced with `apm-server.auth.api_key`, and will be removed
+  # in the 8.0 release. If that config is defined, this one will be ignored.
   #api_key:
     #enabled: false
 

apm-server.docker.yml

@@ -6,6 +6,19 @@ apm-server:
   # Defines the host and port the server is listening on. Use "unix:/path/to.sock" to listen on a unix domain socket.
   host: "0.0.0.0:8200"
 
+  # Agent authorization configuration. If no methods are defined, all requests will be allowed.
+  #auth:
+    # Agent authorization using Elasticsearch API Keys.
+    #api_key:
+      #enabled: false
+      #
+      # Restrict how many unique API keys are allowed per minute. Should be set to at least the amount of different
+      # API keys configured in your monitored services. Every unique API key triggers one request to Elasticsearch.
+      #limit: 100
+
+    # Define a shared secret token for authorizing agents using the "Bearer" authorization method.
+    #secret_token:
+
   # Maximum permitted size in bytes of a request's header accepted by the server to be processed.
   #max_header_size: 1048576
 
@@ -102,11 +115,17 @@ apm-server:
   # Agents include the token in the following format: Authorization: Bearer <secret-token>.
   # It is recommended to use an authorization token in combination with SSL enabled,
   # and save the token in the apm-server keystore.
+  #
+  # WARNING: This configuration is deprecated and replaced with `apm-server.auth.secret_token`, and will be removed
+  # in the 8.0 release. If that config is defined, this one will be ignored.
   #secret_token:
 
   # Enable API key authorization by setting enabled to true. By default API key support is disabled.
   # Agents include a valid API key in the following format: Authorization: ApiKey <token>.
   # The key must be the base64 encoded representation of the API key's "id:key".
+  #
+  # WARNING: This configuration is deprecated and replaced with `apm-server.auth.api_key`, and will be removed
+  # in the 8.0 release. If that config is defined, this one will be ignored.
   #api_key:
     #enabled: false
 

apm-server.yml

@@ -6,6 +6,19 @@ apm-server:
   # Defines the host and port the server is listening on. Use "unix:/path/to.sock" to listen on a unix domain socket.
   host: "localhost:8200"
 
+  # Agent authorization configuration. If no methods are defined, all requests will be allowed.
+  #auth:
+    # Agent authorization using Elasticsearch API Keys.
+    #api_key:
+      #enabled: false
+      #
+      # Restrict how many unique API keys are allowed per minute. Should be set to at least the amount of different
+      # API keys configured in your monitored services. Every unique API key triggers one request to Elasticsearch.
+      #limit: 100
+
+    # Define a shared secret token for authorizing agents using the "Bearer" authorization method.
+    #secret_token:
+
   # Maximum permitted size in bytes of a request's header accepted by the server to be processed.
   #max_header_size: 1048576
 
@@ -102,11 +115,17 @@ apm-server:
   # Agents include the token in the following format: Authorization: Bearer <secret-token>.
   # It is recommended to use an authorization token in combination with SSL enabled,
   # and save the token in the apm-server keystore.
+  #
+  # WARNING: This configuration is deprecated and replaced with `apm-server.auth.secret_token`, and will be removed
+  # in the 8.0 release. If that config is defined, this one will be ignored.
   #secret_token:
 
   # Enable API key authorization by setting enabled to true. By default API key support is disabled.
   # Agents include a valid API key in the following format: Authorization: ApiKey <token>.
   # The key must be the base64 encoded representation of the API key's "id:key".
+  #
+  # WARNING: This configuration is deprecated and replaced with `apm-server.auth.api_key`, and will be removed
+  # in the 8.0 release. If that config is defined, this one will be ignored.
   #api_key:
     #enabled: false
 

apmpackage/apm/agent/input/template.yml.hbs

@@ -1,6 +1,10 @@
 apm-server:
     host: {{host}}
-    secret_token: {{secret_token}}
+    auth:
+        secret_token: {{secret_token}}
+        api_key:
+            enabled: {{api_key_enabled}}
+            limit: {{api_key_limit}}
     max_event_size: {{max_event_bytes}}
     capture_personal_data: {{capture_personal_data}}
     default_service_environment: {{default_service_environment}}
@@ -15,6 +19,3 @@ apm-server:
         response_headers: {{rum_response_headers}}
         event_rate.limit: {{rum_event_rate_limit}}
         event_rate.lru_size: {{rum_event_rate_lru_size}}
-    api_key:
-        enabled: {{api_key_enabled}}
-        limit: {{api_key_limit}}

beater/api/mux.go

@@ -78,7 +78,7 @@ func NewMux(
 	mux := http.NewServeMux()
 	logger := logp.NewLogger(logs.Handler)
 
-	auth, err := authorization.NewBuilder(beaterConfig)
+	auth, err := authorization.NewBuilder(beaterConfig.AgentAuth)
 	if err != nil {
 		return nil, err
 	}

beater/api/mux_config_agent_test.go

@@ -33,7 +33,7 @@ import (
 func TestConfigAgentHandler_AuthorizationMiddleware(t *testing.T) {
 	t.Run("Unauthorized", func(t *testing.T) {
 		cfg := configEnabledConfigAgent()
-		cfg.SecretToken = "1234"
+		cfg.AgentAuth.SecretToken = "1234"
 		rec, err := requestToMuxerWithPattern(cfg, AgentConfigPath)
 		require.NoError(t, err)
 		require.Equal(t, http.StatusUnauthorized, rec.Code)
@@ -42,7 +42,7 @@ func TestConfigAgentHandler_AuthorizationMiddleware(t *testing.T) {
 
 	t.Run("Authorized", func(t *testing.T) {
 		cfg := configEnabledConfigAgent()
-		cfg.SecretToken = "1234"
+		cfg.AgentAuth.SecretToken = "1234"
 		header := map[string]string{headers.Authorization: "Bearer 1234"}
 		queryString := map[string]string{"service.name": "service1"}
 		rec, err := requestToMuxerWithHeaderAndQueryString(cfg, AgentConfigPath, http.MethodGet, header, queryString)

beater/api/mux_intake_backend_test.go

@@ -34,7 +34,7 @@ import (
 func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) {
 	t.Run("Unauthorized", func(t *testing.T) {
 		cfg := config.DefaultConfig()
-		cfg.SecretToken = "1234"
+		cfg.AgentAuth.SecretToken = "1234"
 		rec, err := requestToMuxerWithPattern(cfg, IntakePath)
 		require.NoError(t, err)
 
@@ -44,7 +44,7 @@ func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) {
 
 	t.Run("Authorized", func(t *testing.T) {
 		cfg := config.DefaultConfig()
-		cfg.SecretToken = "1234"
+		cfg.AgentAuth.SecretToken = "1234"
 		h := map[string]string{headers.Authorization: "Bearer 1234"}
 		rec, err := requestToMuxerWithHeader(cfg, IntakePath, http.MethodGet, h)
 		require.NoError(t, err)

beater/api/mux_intake_rum_test.go

@@ -67,7 +67,7 @@ func TestOPTIONS(t *testing.T) {
 
 func TestRUMHandler_NoAuthorizationRequired(t *testing.T) {
 	cfg := cfgEnabledRUM()
-	cfg.SecretToken = "1234"
+	cfg.AgentAuth.SecretToken = "1234"
 	rec, err := requestToMuxerWithPattern(cfg, IntakeRUMPath)
 	require.NoError(t, err)
 	assert.NotEqual(t, http.StatusUnauthorized, rec.Code)

beater/api/mux_root_test.go

@@ -33,7 +33,7 @@ import (
 
 func TestRootHandler_AuthorizationMiddleware(t *testing.T) {
 	cfg := config.DefaultConfig()
-	cfg.SecretToken = "1234"
+	cfg.AgentAuth.SecretToken = "1234"
 
 	t.Run("No auth", func(t *testing.T) {
 		rec, err := requestToMuxerWithPattern(cfg, RootPath)

beater/api/mux_sourcemap_handler_test.go

@@ -33,7 +33,7 @@ import (
 func TestSourcemapHandler_AuthorizationMiddleware(t *testing.T) {
 	t.Run("Unauthorized", func(t *testing.T) {
 		cfg := cfgEnabledRUM()
-		cfg.SecretToken = "1234"
+		cfg.AgentAuth.SecretToken = "1234"
 		rec, err := requestToMuxerWithPattern(cfg, AssetSourcemapPath)
 		require.NoError(t, err)
 		require.Equal(t, http.StatusUnauthorized, rec.Code)
@@ -42,7 +42,7 @@ func TestSourcemapHandler_AuthorizationMiddleware(t *testing.T) {
 
 	t.Run("Authorized", func(t *testing.T) {
 		cfg := cfgEnabledRUM()
-		cfg.SecretToken = "1234"
+		cfg.AgentAuth.SecretToken = "1234"
 		h := map[string]string{headers.Authorization: "Bearer 1234"}
 		rec, err := requestToMuxerWithHeader(cfg, AssetSourcemapPath, http.MethodPost, h)
 		require.NoError(t, err)

beater/authorization/apikey_test.go

@@ -197,11 +197,11 @@ func (tc *apikeyTestcase) setup(t *testing.T) {
 	}
 
 	cfg := config.DefaultConfig()
-	cfg.APIKeyConfig.Enabled = true
+	cfg.AgentAuth.APIKey.Enabled = true
 	if tc.apiKeyLimit > 0 {
-		cfg.APIKeyConfig.LimitPerMin = tc.apiKeyLimit
+		cfg.AgentAuth.APIKey.LimitPerMin = tc.apiKeyLimit
 	}
-	tc.builder, err = NewBuilder(cfg)
+	tc.builder, err = NewBuilder(cfg.AgentAuth)
 	require.NoError(t, err)
 	tc.builder.apikey.esClient = tc.client
 	tc.cache = tc.builder.apikey.cache

beater/authorization/builder.go

@@ -19,6 +19,7 @@ package authorization
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"time"
 
@@ -27,6 +28,16 @@ import (
 	"github.com/elastic/apm-server/elasticsearch"
 )
 
+// ErrUnauthorized is an error that can be used to indicate that the client is
+// unauthorized for some action and resource. This should be wrapped to provide
+// a reason, and checked using `errors.Is`.
+//
+// This is not returned from AuthorizedFor methods; those methods return a
+// Result that indicates whether or not an operation is authorized, as
+// auth may be optional. The error is used where auth is absolutely required,
+// and will be communicated up the stack to set HTTP/gRPC status codes.
+var ErrUnauthorized = errors.New("unauthorized")
+
 // Builder creates an authorization Handler depending on configuration options
 type Builder struct {
 	apikey *apikeyBuilder
@@ -86,19 +97,19 @@ const (
 // NewBuilder creates authorization builder based off of the given information
 // if apm-server.api_key is enabled, authorization is granted/denied solely
 // based on the request Authorization header
-func NewBuilder(cfg *config.Config) (*Builder, error) {
+func NewBuilder(cfg config.AgentAuth) (*Builder, error) {
 	b := Builder{}
-	if cfg.APIKeyConfig.Enabled {
+	if cfg.APIKey.Enabled {
 		// do not use username+password for API Key requests
-		cfg.APIKeyConfig.ESConfig.Username = ""
-		cfg.APIKeyConfig.ESConfig.Password = ""
-		cfg.APIKeyConfig.ESConfig.APIKey = ""
-		client, err := elasticsearch.NewClient(cfg.APIKeyConfig.ESConfig)
+		cfg.APIKey.ESConfig.Username = ""
+		cfg.APIKey.ESConfig.Password = ""
+		cfg.APIKey.ESConfig.APIKey = ""
+		client, err := elasticsearch.NewClient(cfg.APIKey.ESConfig)
 		if err != nil {
 			return nil, err
 		}
 
-		cache := newPrivilegesCache(cacheTimeoutMinute, cfg.APIKeyConfig.LimitPerMin)
+		cache := newPrivilegesCache(cacheTimeoutMinute, cfg.APIKey.LimitPerMin)
 		b.apikey = newApikeyBuilder(client, cache, []elasticsearch.PrivilegeAction{})
 	}
 	if cfg.SecretToken != "" {

beater/authorization/builder_test.go

@@ -40,13 +40,12 @@ func TestBuilder(t *testing.T) {
 	} {
 
 		setup := func() *Builder {
-			cfg := config.DefaultConfig()
-
+			var cfg config.AgentAuth
 			if tc.withBearer {
 				cfg.SecretToken = "xvz"
 			}
 			if tc.withApikey {
-				cfg.APIKeyConfig = config.APIKeyConfig{
+				cfg.APIKey = config.APIKeyAgentAuth{
 					Enabled: true, LimitPerMin: 100,
 					ESConfig: elasticsearch.DefaultConfig(),
 				}

beater/authprocessor.go

@@ -19,7 +19,7 @@ package beater
 
 import (
 	"context"
-	"errors"
+	"fmt"
 
 	"github.com/elastic/apm-server/beater/authorization"
 	"github.com/elastic/apm-server/model"
@@ -38,6 +38,5 @@ func verifyAuthorizedFor(ctx context.Context, meta *model.Metadata) error {
 	if result.Authorized {
 		return nil
 	}
-	// TODO(axw) specific error type to control response code?
-	return errors.New(result.Reason)
+	return fmt.Errorf("%w: %s", authorization.ErrUnauthorized, result.Reason)
 }

beater/config/api_key.go → beater/config/auth.go

@@ -20,44 +20,59 @@ package config
 import (
 	"github.com/pkg/errors"
 
-	"github.com/elastic/beats/v7/libbeat/logp"
-
 	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/logp"
 
 	"github.com/elastic/apm-server/elasticsearch"
 )
 
-const apiKeyLimit = 100
+// AgentAuth holds config related to agent auth.
+type AgentAuth struct {
+	APIKey      APIKeyAgentAuth `config:"api_key"`
+	SecretToken string          `config:"secret_token"`
+}
 
-// APIKeyConfig can be used for authorizing against the APM Server via API Keys.
-type APIKeyConfig struct {
+// APIKeyAgentAuth holds config related to API Key auth for agents.
+type APIKeyAgentAuth struct {
 	Enabled     bool                  `config:"enabled"`
 	LimitPerMin int                   `config:"limit"`
 	ESConfig    *elasticsearch.Config `config:"elasticsearch"`
 
-	esConfigured bool
+	configured   bool // api_key explicitly defined
+	esConfigured bool // api_key.elasticsearch explicitly defined
+}
+
+func (a *APIKeyAgentAuth) Unpack(in *common.Config) error {
+	type underlyingAPIKeyAgentAuth APIKeyAgentAuth
+	if err := in.Unpack((*underlyingAPIKeyAgentAuth)(a)); err != nil {
+		return errors.Wrap(err, "error unpacking api_key config")
+	}
+	a.configured = true
+	a.esConfigured = in.HasField("elasticsearch")
+	return nil
 }
 
-func (c *APIKeyConfig) setup(log *logp.Logger, outputESCfg *common.Config) error {
-	if !c.Enabled || c.esConfigured || outputESCfg == nil {
+func (a *APIKeyAgentAuth) setup(log *logp.Logger, outputESCfg *common.Config) error {
+	if !a.Enabled || a.esConfigured || outputESCfg == nil {
 		return nil
 	}
 	log.Info("Falling back to elasticsearch output for API Key usage")
-	if err := outputESCfg.Unpack(c.ESConfig); err != nil {
+	if err := outputESCfg.Unpack(&a.ESConfig); err != nil {
 		return errors.Wrap(err, "unpacking Elasticsearch config into API key config")
 	}
 	return nil
 }
 
-func defaultAPIKeyConfig() APIKeyConfig {
-	return APIKeyConfig{Enabled: false, LimitPerMin: apiKeyLimit, ESConfig: elasticsearch.DefaultConfig()}
+func defaultAgentAuth() AgentAuth {
+	return AgentAuth{
+		APIKey: defaultAPIKeyAgentAuth(),
+	}
 }
 
-func (c *APIKeyConfig) Unpack(inp *common.Config) error {
-	type underlyingAPIKeyConfig APIKeyConfig
-	if err := inp.Unpack((*underlyingAPIKeyConfig)(c)); err != nil {
-		return errors.Wrap(err, "error unpacking api_key config")
+func defaultAPIKeyAgentAuth() APIKeyAgentAuth {
+	return APIKeyAgentAuth{
+		Enabled:     false,
+		LimitPerMin: 100,
+		ESConfig:    elasticsearch.DefaultConfig(),
 	}
-	c.esConfigured = inp.HasField("elasticsearch")
-	return nil
 }