Introduce apm-server.auth.anonymous config

_meta/beat.yml

@@ -19,6 +19,34 @@ apm-server:
     # Define a shared secret token for authorizing agents using the "Bearer" authorization method.
     #secret_token:
 
+    # Allow anonymous access only for specified agents and/or services. This is primarily intended to allow
+    # limited access for untrusted agents, such as Real User Monitoring.
+    #anonymous:
+      # By default anonymous auth is automatically enabled when either auth.api_key or
+      # auth.secret_token is enabled, and RUM is enabled. Otherwise, anonymous auth is
+      # disabled by default.
+      #
+      # When anonymous auth is enabled, only agents matching allow_agent and services
+      # matching allow_service are allowed. See below for details on default values for
+      # allow_agent.
+      #enabled:
+
+      # Allow anonymous access only for specified agents.
+      #allow_agent: [rum-js, js-base]
+
+      # Allow anonymous access only for specified service names. By default, all service names are allowed.
+      #allow_service: []
+
+      # Rate-limit anonymous access by IP and number of events.
+      #rate_limit:
+        # Rate limiting is defined per unique client IP address, for a limited number of IP addresses.
+        # Sites with many concurrent clients should consider increasing this limit. Defaults to 1000.
+        #ip_limit: 1000
+
+        # Defines the maximum amount of events allowed per IP per second. Defaults to 300. The overall
+        # maximum event throughput for anonymous access is (event_limit * ip_limit).
+        #event_limit: 300
+
   # Maximum permitted size in bytes of a request's header accepted by the server to be processed.
   #max_header_size: 1048576
 
@@ -214,6 +242,10 @@ apm-server:
   #rum:
     #enabled: false
 
+    # Rate-limit RUM agents.
+    #
+    # WARNING: This configuration is deprecated and replaced with `apm-server.auth.anonymous.rate_limit`,
+    # and will be removed in the 8.0 release. If that config is defined, this one will be ignored.
     #event_rate:
 
       # Defines the maximum amount of events allowed to be sent to the APM Server RUM
@@ -230,6 +262,9 @@ apm-server:
     # A list of service names to allow, to limit service-specific indices and data streams
     # created for unauthenticated RUM events.
     # If the list is empty, any service name is allowed.
+    #
+    # WARNING: This configuration is deprecated and replaced with `apm-server.auth.anonymous.allow_service`,
+    # and will be removed in the 8.0 release. If that config is defined, this one will be ignored.
     #allow_service_names: []
 
     # A list of permitted origins for real user monitoring.

apm-server.docker.yml

@@ -19,6 +19,34 @@ apm-server:
     # Define a shared secret token for authorizing agents using the "Bearer" authorization method.
     #secret_token:
 
+    # Allow anonymous access only for specified agents and/or services. This is primarily intended to allow
+    # limited access for untrusted agents, such as Real User Monitoring.
+    #anonymous:
+      # By default anonymous auth is automatically enabled when either auth.api_key or
+      # auth.secret_token is enabled, and RUM is enabled. Otherwise, anonymous auth is
+      # disabled by default.
+      #
+      # When anonymous auth is enabled, only agents matching allow_agent and services
+      # matching allow_service are allowed. See below for details on default values for
+      # allow_agent.
+      #enabled:
+
+      # Allow anonymous access only for specified agents.
+      #allow_agent: [rum-js, js-base]
+
+      # Allow anonymous access only for specified service names. By default, all service names are allowed.
+      #allow_service: []
+
+      # Rate-limit anonymous access by IP and number of events.
+      #rate_limit:
+        # Rate limiting is defined per unique client IP address, for a limited number of IP addresses.
+        # Sites with many concurrent clients should consider increasing this limit. Defaults to 1000.
+        #ip_limit: 1000
+
+        # Defines the maximum amount of events allowed per IP per second. Defaults to 300. The overall
+        # maximum event throughput for anonymous access is (event_limit * ip_limit).
+        #event_limit: 300
+
   # Maximum permitted size in bytes of a request's header accepted by the server to be processed.
   #max_header_size: 1048576
 
@@ -214,6 +242,10 @@ apm-server:
   #rum:
     #enabled: false
 
+    # Rate-limit RUM agents.
+    #
+    # WARNING: This configuration is deprecated and replaced with `apm-server.auth.anonymous.rate_limit`,
+    # and will be removed in the 8.0 release. If that config is defined, this one will be ignored.
     #event_rate:
 
       # Defines the maximum amount of events allowed to be sent to the APM Server RUM
@@ -230,6 +262,9 @@ apm-server:
     # A list of service names to allow, to limit service-specific indices and data streams
     # created for unauthenticated RUM events.
     # If the list is empty, any service name is allowed.
+    #
+    # WARNING: This configuration is deprecated and replaced with `apm-server.auth.anonymous.allow_service`,
+    # and will be removed in the 8.0 release. If that config is defined, this one will be ignored.
     #allow_service_names: []
 
     # A list of permitted origins for real user monitoring.

apm-server.yml

@@ -19,6 +19,34 @@ apm-server:
     # Define a shared secret token for authorizing agents using the "Bearer" authorization method.
     #secret_token:
 
+    # Allow anonymous access only for specified agents and/or services. This is primarily intended to allow
+    # limited access for untrusted agents, such as Real User Monitoring.
+    #anonymous:
+      # By default anonymous auth is automatically enabled when either auth.api_key or
+      # auth.secret_token is enabled, and RUM is enabled. Otherwise, anonymous auth is
+      # disabled by default.
+      #
+      # When anonymous auth is enabled, only agents matching allow_agent and services
+      # matching allow_service are allowed. See below for details on default values for
+      # allow_agent.
+      #enabled:
+
+      # Allow anonymous access only for specified agents.
+      #allow_agent: [rum-js, js-base]
+
+      # Allow anonymous access only for specified service names. By default, all service names are allowed.
+      #allow_service: []
+
+      # Rate-limit anonymous access by IP and number of events.
+      #rate_limit:
+        # Rate limiting is defined per unique client IP address, for a limited number of IP addresses.
+        # Sites with many concurrent clients should consider increasing this limit. Defaults to 1000.
+        #ip_limit: 1000
+
+        # Defines the maximum amount of events allowed per IP per second. Defaults to 300. The overall
+        # maximum event throughput for anonymous access is (event_limit * ip_limit).
+        #event_limit: 300
+
   # Maximum permitted size in bytes of a request's header accepted by the server to be processed.
   #max_header_size: 1048576
 
@@ -214,6 +242,10 @@ apm-server:
   #rum:
     #enabled: false
 
+    # Rate-limit RUM agents.
+    #
+    # WARNING: This configuration is deprecated and replaced with `apm-server.auth.anonymous.rate_limit`,
+    # and will be removed in the 8.0 release. If that config is defined, this one will be ignored.
     #event_rate:
 
       # Defines the maximum amount of events allowed to be sent to the APM Server RUM
@@ -230,6 +262,9 @@ apm-server:
     # A list of service names to allow, to limit service-specific indices and data streams
     # created for unauthenticated RUM events.
     # If the list is empty, any service name is allowed.
+    #
+    # WARNING: This configuration is deprecated and replaced with `apm-server.auth.anonymous.allow_service`,
+    # and will be removed in the 8.0 release. If that config is defined, this one will be ignored.
     #allow_service_names: []
 
     # A list of permitted origins for real user monitoring.

beater/api/intake/handler.go

@@ -27,6 +27,7 @@ import (
 
 	"github.com/elastic/beats/v7/libbeat/monitoring"
 
+	"github.com/elastic/apm-server/beater/authorization"
 	"github.com/elastic/apm-server/beater/headers"
 	"github.com/elastic/apm-server/beater/ratelimit"
 	"github.com/elastic/apm-server/beater/request"
@@ -154,6 +155,8 @@ func writeStreamResult(c *request.Context, sr *stream.Result) {
 					errID = request.IDResponseErrorsValidate
 				case errors.Is(err, ratelimit.ErrRateLimitExceeded):
 					errID = request.IDResponseErrorsRateLimit
+				case errors.Is(err, authorization.ErrUnauthorized):
+					errID = request.IDResponseErrorsUnauthorized
 				}
 			}
 			jsonResult.Errors[i] = jsonError{Message: err.Error()}

beater/api/mux.go

@@ -18,7 +18,6 @@
 package api
 
 import (
-	"context"
 	"net/http"
 	"net/http/pprof"
 
@@ -39,7 +38,6 @@ import (
 	"github.com/elastic/apm-server/beater/request"
 	logs "github.com/elastic/apm-server/log"
 	"github.com/elastic/apm-server/model"
-	"github.com/elastic/apm-server/model/modelprocessor"
 	"github.com/elastic/apm-server/processor/stream"
 	"github.com/elastic/apm-server/publish"
 )
@@ -69,13 +67,6 @@ const (
 	IntakeRUMV3Path = "/intake/v3/rum/events"
 )
 
-var (
-	// rumAgents holds the current and previous agent names for the
-	// RUM JavaScript agent. This is used for restricting which config
-	// is supplied to anonymous agents.
-	rumAgents = []string{"rum-js", "js-base"}
-)
-
 // NewMux registers apm handlers to paths building up the APM Server API.
 func NewMux(
 	beatInfo beat.Info,
@@ -179,16 +170,16 @@ func (r *routeBuilder) rumIntakeHandler(newProcessor func(*config.Config) *strea
 		requestMetadataFunc = rumRequestMetadata
 	}
 	return func() (request.Handler, error) {
-		batchProcessor := r.batchProcessor
-		batchProcessor = batchProcessorWithAllowedServiceNames(batchProcessor, r.cfg.RumConfig.AllowServiceNames)
-		h := intake.Handler(newProcessor(r.cfg), requestMetadataFunc, batchProcessor)
-		return middleware.Wrap(h, rumMiddleware(r.cfg, nil, r.ratelimitStore, intake.MonitoringMap)...)
+		authHandler := r.authBuilder.ForPrivilege(authorization.PrivilegeEventWrite.Action)
+		h := intake.Handler(newProcessor(r.cfg), requestMetadataFunc, r.batchProcessor)
+		return middleware.Wrap(h, rumMiddleware(r.cfg, authHandler, r.ratelimitStore, intake.MonitoringMap)...)
 	}
 }
 
 func (r *routeBuilder) sourcemapHandler() (request.Handler, error) {
 	h := sourcemap.Handler(r.reporter)
 	authHandler := r.authBuilder.ForPrivilege(authorization.PrivilegeSourcemapWrite.Action)
+	authHandler = authHandler.WithAnonymousDisabled()
 	return middleware.Wrap(h, sourcemapMiddleware(r.cfg, authHandler, r.ratelimitStore)...)
 }
 
@@ -206,7 +197,8 @@ func (r *routeBuilder) backendAgentConfigHandler(f agentcfg.Fetcher) func() (req
 
 func (r *routeBuilder) rumAgentConfigHandler(f agentcfg.Fetcher) func() (request.Handler, error) {
 	return func() (request.Handler, error) {
-		return agentConfigHandler(r.cfg, nil, r.ratelimitStore, rumMiddleware, f)
+		authHandler := r.authBuilder.ForPrivilege(authorization.PrivilegeAgentConfigRead.Action)
+		return agentConfigHandler(r.cfg, authHandler, r.ratelimitStore, rumMiddleware, f)
 	}
 }
 
@@ -220,7 +212,7 @@ func agentConfigHandler(
 	f agentcfg.Fetcher,
 ) (request.Handler, error) {
 	mw := middlewareFunc(cfg, authHandler, ratelimitStore, agent.MonitoringMap)
-	h := agent.NewHandler(f, cfg.KibanaAgentConfig, cfg.DefaultServiceEnvironment, rumAgents)
+	h := agent.NewHandler(f, cfg.KibanaAgentConfig, cfg.DefaultServiceEnvironment, cfg.AgentAuth.Anonymous.AllowAgent)
 
 	if !cfg.Kibana.Enabled && cfg.AgentConfigs == nil {
 		msg := "Agent remote configuration is disabled. " +
@@ -260,7 +252,7 @@ func rumMiddleware(cfg *config.Config, auth *authorization.Handler, ratelimitSto
 		middleware.ResponseHeadersMiddleware(cfg.ResponseHeaders),
 		middleware.ResponseHeadersMiddleware(cfg.RumConfig.ResponseHeaders),
 		middleware.CORSMiddleware(cfg.RumConfig.AllowOrigins, cfg.RumConfig.AllowHeaders),
-		middleware.AnonymousAuthorizationMiddleware(),
+		middleware.AuthorizationMiddleware(auth, true),
 		middleware.AnonymousRateLimitMiddleware(ratelimitStore),
 	)
 	return append(rumMiddleware, middleware.KillSwitchMiddleware(cfg.RumConfig.Enabled, msg))
@@ -284,28 +276,6 @@ func rootMiddleware(cfg *config.Config, auth *authorization.Handler) []middlewar
 		middleware.AuthorizationMiddleware(auth, false))
 }
 
-// TODO(axw) move this into the authorization package when introducing anonymous auth.
-func batchProcessorWithAllowedServiceNames(p model.BatchProcessor, allowedServiceNames []string) model.BatchProcessor {
-	if len(allowedServiceNames) == 0 {
-		// All service names are allowed.
-		return p
-	}
-	m := make(map[string]bool)
-	for _, name := range allowedServiceNames {
-		m[name] = true
-	}
-	var restrictServiceName modelprocessor.MetadataProcessorFunc = func(ctx context.Context, meta *model.Metadata) error {
-		// Restrict to explicitly allowed service names.
-		// The list of allowed service names is not considered secret,
-		// so we do not use constant time comparison.
-		if !m[meta.Service.Name] {
-			return &stream.InvalidInputError{Message: "service name is not allowed"}
-		}
-		return nil
-	}
-	return modelprocessor.Chained{restrictServiceName, p}
-}
-
 func emptyRequestMetadata(c *request.Context) model.Metadata {
 	return model.Metadata{}
 }

beater/api/mux_intake_rum_test.go

@@ -18,8 +18,6 @@
 package api
 
 import (
-	"bytes"
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -29,6 +27,7 @@ import (
 
 	"github.com/elastic/apm-server/approvaltest"
 	"github.com/elastic/apm-server/beater/api/intake"
+	"github.com/elastic/apm-server/beater/authorization"
 	"github.com/elastic/apm-server/beater/config"
 	"github.com/elastic/apm-server/beater/headers"
 	"github.com/elastic/apm-server/beater/middleware"
@@ -43,12 +42,16 @@ func TestOPTIONS(t *testing.T) {
 
 	cfg := cfgEnabledRUM()
 	cfg.RumConfig.AllowOrigins = []string{"*"}
+	authBuilder, _ := authorization.NewBuilder(cfg.AgentAuth)
+	authHandler := authBuilder.ForPrivilege(authorization.PrivilegeEventWrite.Action)
+
 	h, _ := middleware.Wrap(
 		func(c *request.Context) {
 			requestTaken <- struct{}{}
 			<-done
 		},
-		rumMiddleware(cfg, nil, ratelimitStore, intake.MonitoringMap)...)
+		rumMiddleware(cfg, authHandler, ratelimitStore, intake.MonitoringMap)...,
+	)
 
 	// use this to block the single allowed concurrent requests
 	go func() {
@@ -125,42 +128,10 @@ func TestRumHandler_MonitoringMiddleware(t *testing.T) {
 	}
 }
 
-func TestRUMHandler_AllowedServiceNames(t *testing.T) {
-	payload, err := ioutil.ReadFile("../../testdata/intake-v2/transactions_spans_rum.ndjson")
-	require.NoError(t, err)
-
-	for _, test := range []struct {
-		AllowServiceNames []string
-		Allowed           bool
-	}{{
-		AllowServiceNames: nil,
-		Allowed:           true, // none specified = all allowed
-	}, {
-		AllowServiceNames: []string{"apm-agent-js"}, // matches what's in test data
-		Allowed:           true,
-	}, {
-		AllowServiceNames: []string{"reject_everything"},
-		Allowed:           false,
-	}} {
-		cfg := cfgEnabledRUM()
-		cfg.RumConfig.AllowServiceNames = test.AllowServiceNames
-		h := newTestMux(t, cfg)
-
-		req := httptest.NewRequest(http.MethodPost, "/intake/v2/rum/events", bytes.NewReader(payload))
-		req.Header.Set("Content-Type", "application/x-ndjson")
-		w := httptest.NewRecorder()
-		h.ServeHTTP(w, req)
-		if test.Allowed {
-			assert.Equal(t, http.StatusAccepted, w.Code)
-		} else {
-			assert.Equal(t, http.StatusBadRequest, w.Code)
-		}
-	}
-}
-
 func cfgEnabledRUM() *config.Config {
 	cfg := config.DefaultConfig()
 	cfg.RumConfig.Enabled = true
+	cfg.AgentAuth.Anonymous.Enabled = true
 	return cfg
 }