diff --git a/_meta/beat.yml b/_meta/beat.yml index 7363d568b8b..9dee7ba5fb5 100644 --- a/_meta/beat.yml +++ b/_meta/beat.yml @@ -6,6 +6,19 @@ apm-server: # Defines the host and port the server is listening on. Use "unix:/path/to.sock" to listen on a unix domain socket. host: "{{ .listen_hostport }}" + # Agent authorization configuration. If no methods are defined, all requests will be allowed. + #auth: + # Agent authorization using Elasticsearch API Keys. + #api_key: + #enabled: false + # + # Restrict how many unique API keys are allowed per minute. Should be set to at least the amount of different + # API keys configured in your monitored services. Every unique API key triggers one request to Elasticsearch. + #limit: 100 + + # Define a shared secret token for authorizing agents using the "Bearer" authorization method. + #secret_token: + # Maximum permitted size in bytes of a request's header accepted by the server to be processed. #max_header_size: 1048576 @@ -102,11 +115,17 @@ apm-server: # Agents include the token in the following format: Authorization: Bearer . # It is recommended to use an authorization token in combination with SSL enabled, # and save the token in the apm-server keystore. + # + # WARNING: This configuration is deprecated and replaced with `apm-server.auth.secret_token`, and will be removed + # in the 8.0 release. If that config is defined, this one will be ignored. #secret_token: # Enable API key authorization by setting enabled to true. By default API key support is disabled. # Agents include a valid API key in the following format: Authorization: ApiKey . # The key must be the base64 encoded representation of the API key's "id:key". + # + # WARNING: This configuration is deprecated and replaced with `apm-server.auth.api_key`, and will be removed + # in the 8.0 release. If that config is defined, this one will be ignored. #api_key: #enabled: false diff --git a/apm-server.docker.yml b/apm-server.docker.yml index d131f087a8b..12a6cbf82c6 100644 --- a/apm-server.docker.yml +++ b/apm-server.docker.yml @@ -6,6 +6,19 @@ apm-server: # Defines the host and port the server is listening on. Use "unix:/path/to.sock" to listen on a unix domain socket. host: "0.0.0.0:8200" + # Agent authorization configuration. If no methods are defined, all requests will be allowed. + #auth: + # Agent authorization using Elasticsearch API Keys. + #api_key: + #enabled: false + # + # Restrict how many unique API keys are allowed per minute. Should be set to at least the amount of different + # API keys configured in your monitored services. Every unique API key triggers one request to Elasticsearch. + #limit: 100 + + # Define a shared secret token for authorizing agents using the "Bearer" authorization method. + #secret_token: + # Maximum permitted size in bytes of a request's header accepted by the server to be processed. #max_header_size: 1048576 @@ -102,11 +115,17 @@ apm-server: # Agents include the token in the following format: Authorization: Bearer . # It is recommended to use an authorization token in combination with SSL enabled, # and save the token in the apm-server keystore. + # + # WARNING: This configuration is deprecated and replaced with `apm-server.auth.secret_token`, and will be removed + # in the 8.0 release. If that config is defined, this one will be ignored. #secret_token: # Enable API key authorization by setting enabled to true. By default API key support is disabled. # Agents include a valid API key in the following format: Authorization: ApiKey . # The key must be the base64 encoded representation of the API key's "id:key". + # + # WARNING: This configuration is deprecated and replaced with `apm-server.auth.api_key`, and will be removed + # in the 8.0 release. If that config is defined, this one will be ignored. #api_key: #enabled: false diff --git a/apm-server.yml b/apm-server.yml index e7bb6c3a6ff..7ac9574daad 100644 --- a/apm-server.yml +++ b/apm-server.yml @@ -6,6 +6,19 @@ apm-server: # Defines the host and port the server is listening on. Use "unix:/path/to.sock" to listen on a unix domain socket. host: "localhost:8200" + # Agent authorization configuration. If no methods are defined, all requests will be allowed. + #auth: + # Agent authorization using Elasticsearch API Keys. + #api_key: + #enabled: false + # + # Restrict how many unique API keys are allowed per minute. Should be set to at least the amount of different + # API keys configured in your monitored services. Every unique API key triggers one request to Elasticsearch. + #limit: 100 + + # Define a shared secret token for authorizing agents using the "Bearer" authorization method. + #secret_token: + # Maximum permitted size in bytes of a request's header accepted by the server to be processed. #max_header_size: 1048576 @@ -102,11 +115,17 @@ apm-server: # Agents include the token in the following format: Authorization: Bearer . # It is recommended to use an authorization token in combination with SSL enabled, # and save the token in the apm-server keystore. + # + # WARNING: This configuration is deprecated and replaced with `apm-server.auth.secret_token`, and will be removed + # in the 8.0 release. If that config is defined, this one will be ignored. #secret_token: # Enable API key authorization by setting enabled to true. By default API key support is disabled. # Agents include a valid API key in the following format: Authorization: ApiKey . # The key must be the base64 encoded representation of the API key's "id:key". + # + # WARNING: This configuration is deprecated and replaced with `apm-server.auth.api_key`, and will be removed + # in the 8.0 release. If that config is defined, this one will be ignored. #api_key: #enabled: false diff --git a/apmpackage/apm/agent/input/template.yml.hbs b/apmpackage/apm/agent/input/template.yml.hbs index 62f822b8d5d..ec5a8a1732c 100644 --- a/apmpackage/apm/agent/input/template.yml.hbs +++ b/apmpackage/apm/agent/input/template.yml.hbs @@ -1,6 +1,10 @@ apm-server: host: {{host}} - secret_token: {{secret_token}} + auth: + secret_token: {{secret_token}} + api_key: + enabled: {{api_key_enabled}} + limit: {{api_key_limit}} max_event_size: {{max_event_bytes}} capture_personal_data: {{capture_personal_data}} default_service_environment: {{default_service_environment}} @@ -15,6 +19,3 @@ apm-server: response_headers: {{rum_response_headers}} event_rate.limit: {{rum_event_rate_limit}} event_rate.lru_size: {{rum_event_rate_lru_size}} - api_key: - enabled: {{api_key_enabled}} - limit: {{api_key_limit}} diff --git a/beater/api/mux.go b/beater/api/mux.go index 43300381b4d..ef6e53afe69 100644 --- a/beater/api/mux.go +++ b/beater/api/mux.go @@ -78,7 +78,7 @@ func NewMux( mux := http.NewServeMux() logger := logp.NewLogger(logs.Handler) - auth, err := authorization.NewBuilder(beaterConfig) + auth, err := authorization.NewBuilder(beaterConfig.AgentAuth) if err != nil { return nil, err } diff --git a/beater/api/mux_config_agent_test.go b/beater/api/mux_config_agent_test.go index 0d83afee99f..da39532ac74 100644 --- a/beater/api/mux_config_agent_test.go +++ b/beater/api/mux_config_agent_test.go @@ -33,7 +33,7 @@ import ( func TestConfigAgentHandler_AuthorizationMiddleware(t *testing.T) { t.Run("Unauthorized", func(t *testing.T) { cfg := configEnabledConfigAgent() - cfg.SecretToken = "1234" + cfg.AgentAuth.SecretToken = "1234" rec, err := requestToMuxerWithPattern(cfg, AgentConfigPath) require.NoError(t, err) require.Equal(t, http.StatusUnauthorized, rec.Code) @@ -42,7 +42,7 @@ func TestConfigAgentHandler_AuthorizationMiddleware(t *testing.T) { t.Run("Authorized", func(t *testing.T) { cfg := configEnabledConfigAgent() - cfg.SecretToken = "1234" + cfg.AgentAuth.SecretToken = "1234" header := map[string]string{headers.Authorization: "Bearer 1234"} queryString := map[string]string{"service.name": "service1"} rec, err := requestToMuxerWithHeaderAndQueryString(cfg, AgentConfigPath, http.MethodGet, header, queryString) diff --git a/beater/api/mux_intake_backend_test.go b/beater/api/mux_intake_backend_test.go index b2bdbc27e4d..9a5a03e7451 100644 --- a/beater/api/mux_intake_backend_test.go +++ b/beater/api/mux_intake_backend_test.go @@ -34,7 +34,7 @@ import ( func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) { t.Run("Unauthorized", func(t *testing.T) { cfg := config.DefaultConfig() - cfg.SecretToken = "1234" + cfg.AgentAuth.SecretToken = "1234" rec, err := requestToMuxerWithPattern(cfg, IntakePath) require.NoError(t, err) @@ -44,7 +44,7 @@ func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) { t.Run("Authorized", func(t *testing.T) { cfg := config.DefaultConfig() - cfg.SecretToken = "1234" + cfg.AgentAuth.SecretToken = "1234" h := map[string]string{headers.Authorization: "Bearer 1234"} rec, err := requestToMuxerWithHeader(cfg, IntakePath, http.MethodGet, h) require.NoError(t, err) diff --git a/beater/api/mux_intake_rum_test.go b/beater/api/mux_intake_rum_test.go index a6fb07a01d8..c1d35d25548 100644 --- a/beater/api/mux_intake_rum_test.go +++ b/beater/api/mux_intake_rum_test.go @@ -67,7 +67,7 @@ func TestOPTIONS(t *testing.T) { func TestRUMHandler_NoAuthorizationRequired(t *testing.T) { cfg := cfgEnabledRUM() - cfg.SecretToken = "1234" + cfg.AgentAuth.SecretToken = "1234" rec, err := requestToMuxerWithPattern(cfg, IntakeRUMPath) require.NoError(t, err) assert.NotEqual(t, http.StatusUnauthorized, rec.Code) diff --git a/beater/api/mux_root_test.go b/beater/api/mux_root_test.go index 3dd784856a6..87aeef3ae7c 100644 --- a/beater/api/mux_root_test.go +++ b/beater/api/mux_root_test.go @@ -33,7 +33,7 @@ import ( func TestRootHandler_AuthorizationMiddleware(t *testing.T) { cfg := config.DefaultConfig() - cfg.SecretToken = "1234" + cfg.AgentAuth.SecretToken = "1234" t.Run("No auth", func(t *testing.T) { rec, err := requestToMuxerWithPattern(cfg, RootPath) diff --git a/beater/api/mux_sourcemap_handler_test.go b/beater/api/mux_sourcemap_handler_test.go index 8fc59853d7f..147cd5258dd 100644 --- a/beater/api/mux_sourcemap_handler_test.go +++ b/beater/api/mux_sourcemap_handler_test.go @@ -33,7 +33,7 @@ import ( func TestSourcemapHandler_AuthorizationMiddleware(t *testing.T) { t.Run("Unauthorized", func(t *testing.T) { cfg := cfgEnabledRUM() - cfg.SecretToken = "1234" + cfg.AgentAuth.SecretToken = "1234" rec, err := requestToMuxerWithPattern(cfg, AssetSourcemapPath) require.NoError(t, err) require.Equal(t, http.StatusUnauthorized, rec.Code) @@ -42,7 +42,7 @@ func TestSourcemapHandler_AuthorizationMiddleware(t *testing.T) { t.Run("Authorized", func(t *testing.T) { cfg := cfgEnabledRUM() - cfg.SecretToken = "1234" + cfg.AgentAuth.SecretToken = "1234" h := map[string]string{headers.Authorization: "Bearer 1234"} rec, err := requestToMuxerWithHeader(cfg, AssetSourcemapPath, http.MethodPost, h) require.NoError(t, err) diff --git a/beater/authorization/apikey_test.go b/beater/authorization/apikey_test.go index 20d6e8d193c..df954eff8d1 100644 --- a/beater/authorization/apikey_test.go +++ b/beater/authorization/apikey_test.go @@ -197,11 +197,11 @@ func (tc *apikeyTestcase) setup(t *testing.T) { } cfg := config.DefaultConfig() - cfg.APIKeyConfig.Enabled = true + cfg.AgentAuth.APIKey.Enabled = true if tc.apiKeyLimit > 0 { - cfg.APIKeyConfig.LimitPerMin = tc.apiKeyLimit + cfg.AgentAuth.APIKey.LimitPerMin = tc.apiKeyLimit } - tc.builder, err = NewBuilder(cfg) + tc.builder, err = NewBuilder(cfg.AgentAuth) require.NoError(t, err) tc.builder.apikey.esClient = tc.client tc.cache = tc.builder.apikey.cache diff --git a/beater/authorization/builder.go b/beater/authorization/builder.go index cb316bbb9d1..e7f3cb4d03e 100644 --- a/beater/authorization/builder.go +++ b/beater/authorization/builder.go @@ -19,6 +19,7 @@ package authorization import ( "context" + "errors" "fmt" "time" @@ -27,6 +28,16 @@ import ( "github.com/elastic/apm-server/elasticsearch" ) +// ErrUnauthorized is an error that can be used to indicate that the client is +// unauthorized for some action and resource. This should be wrapped to provide +// a reason, and checked using `errors.Is`. +// +// This is not returned from AuthorizedFor methods; those methods return a +// Result that indicates whether or not an operation is authorized, as +// auth may be optional. The error is used where auth is absolutely required, +// and will be communicated up the stack to set HTTP/gRPC status codes. +var ErrUnauthorized = errors.New("unauthorized") + // Builder creates an authorization Handler depending on configuration options type Builder struct { apikey *apikeyBuilder @@ -86,19 +97,19 @@ const ( // NewBuilder creates authorization builder based off of the given information // if apm-server.api_key is enabled, authorization is granted/denied solely // based on the request Authorization header -func NewBuilder(cfg *config.Config) (*Builder, error) { +func NewBuilder(cfg config.AgentAuth) (*Builder, error) { b := Builder{} - if cfg.APIKeyConfig.Enabled { + if cfg.APIKey.Enabled { // do not use username+password for API Key requests - cfg.APIKeyConfig.ESConfig.Username = "" - cfg.APIKeyConfig.ESConfig.Password = "" - cfg.APIKeyConfig.ESConfig.APIKey = "" - client, err := elasticsearch.NewClient(cfg.APIKeyConfig.ESConfig) + cfg.APIKey.ESConfig.Username = "" + cfg.APIKey.ESConfig.Password = "" + cfg.APIKey.ESConfig.APIKey = "" + client, err := elasticsearch.NewClient(cfg.APIKey.ESConfig) if err != nil { return nil, err } - cache := newPrivilegesCache(cacheTimeoutMinute, cfg.APIKeyConfig.LimitPerMin) + cache := newPrivilegesCache(cacheTimeoutMinute, cfg.APIKey.LimitPerMin) b.apikey = newApikeyBuilder(client, cache, []elasticsearch.PrivilegeAction{}) } if cfg.SecretToken != "" { diff --git a/beater/authorization/builder_test.go b/beater/authorization/builder_test.go index 36e7c6f4202..0c8989ec598 100644 --- a/beater/authorization/builder_test.go +++ b/beater/authorization/builder_test.go @@ -40,13 +40,12 @@ func TestBuilder(t *testing.T) { } { setup := func() *Builder { - cfg := config.DefaultConfig() - + var cfg config.AgentAuth if tc.withBearer { cfg.SecretToken = "xvz" } if tc.withApikey { - cfg.APIKeyConfig = config.APIKeyConfig{ + cfg.APIKey = config.APIKeyAgentAuth{ Enabled: true, LimitPerMin: 100, ESConfig: elasticsearch.DefaultConfig(), } diff --git a/beater/authprocessor.go b/beater/authprocessor.go index ca0090168e6..1ffffd62acb 100644 --- a/beater/authprocessor.go +++ b/beater/authprocessor.go @@ -19,7 +19,7 @@ package beater import ( "context" - "errors" + "fmt" "github.com/elastic/apm-server/beater/authorization" "github.com/elastic/apm-server/model" @@ -38,6 +38,5 @@ func verifyAuthorizedFor(ctx context.Context, meta *model.Metadata) error { if result.Authorized { return nil } - // TODO(axw) specific error type to control response code? - return errors.New(result.Reason) + return fmt.Errorf("%w: %s", authorization.ErrUnauthorized, result.Reason) } diff --git a/beater/config/api_key.go b/beater/config/auth.go similarity index 56% rename from beater/config/api_key.go rename to beater/config/auth.go index b87ed0d9579..aa4e8a0687e 100644 --- a/beater/config/api_key.go +++ b/beater/config/auth.go @@ -20,44 +20,59 @@ package config import ( "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/logp" - "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/apm-server/elasticsearch" ) -const apiKeyLimit = 100 +// AgentAuth holds config related to agent auth. +type AgentAuth struct { + APIKey APIKeyAgentAuth `config:"api_key"` + SecretToken string `config:"secret_token"` +} -// APIKeyConfig can be used for authorizing against the APM Server via API Keys. -type APIKeyConfig struct { +// APIKeyAgentAuth holds config related to API Key auth for agents. +type APIKeyAgentAuth struct { Enabled bool `config:"enabled"` LimitPerMin int `config:"limit"` ESConfig *elasticsearch.Config `config:"elasticsearch"` - esConfigured bool + configured bool // api_key explicitly defined + esConfigured bool // api_key.elasticsearch explicitly defined +} + +func (a *APIKeyAgentAuth) Unpack(in *common.Config) error { + type underlyingAPIKeyAgentAuth APIKeyAgentAuth + if err := in.Unpack((*underlyingAPIKeyAgentAuth)(a)); err != nil { + return errors.Wrap(err, "error unpacking api_key config") + } + a.configured = true + a.esConfigured = in.HasField("elasticsearch") + return nil } -func (c *APIKeyConfig) setup(log *logp.Logger, outputESCfg *common.Config) error { - if !c.Enabled || c.esConfigured || outputESCfg == nil { +func (a *APIKeyAgentAuth) setup(log *logp.Logger, outputESCfg *common.Config) error { + if !a.Enabled || a.esConfigured || outputESCfg == nil { return nil } log.Info("Falling back to elasticsearch output for API Key usage") - if err := outputESCfg.Unpack(c.ESConfig); err != nil { + if err := outputESCfg.Unpack(&a.ESConfig); err != nil { return errors.Wrap(err, "unpacking Elasticsearch config into API key config") } return nil } -func defaultAPIKeyConfig() APIKeyConfig { - return APIKeyConfig{Enabled: false, LimitPerMin: apiKeyLimit, ESConfig: elasticsearch.DefaultConfig()} +func defaultAgentAuth() AgentAuth { + return AgentAuth{ + APIKey: defaultAPIKeyAgentAuth(), + } } -func (c *APIKeyConfig) Unpack(inp *common.Config) error { - type underlyingAPIKeyConfig APIKeyConfig - if err := inp.Unpack((*underlyingAPIKeyConfig)(c)); err != nil { - return errors.Wrap(err, "error unpacking api_key config") +func defaultAPIKeyAgentAuth() APIKeyAgentAuth { + return APIKeyAgentAuth{ + Enabled: false, + LimitPerMin: 100, + ESConfig: elasticsearch.DefaultConfig(), } - c.esConfigured = inp.HasField("elasticsearch") - return nil } diff --git a/beater/config/api_key_test.go b/beater/config/auth_test.go similarity index 60% rename from beater/config/api_key_test.go rename to beater/config/auth_test.go index 5aae0355d47..fdd490e53e4 100644 --- a/beater/config/api_key_test.go +++ b/beater/config/auth_test.go @@ -26,35 +26,33 @@ import ( "github.com/elastic/apm-server/elasticsearch" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" ) -func TestAPIKeyConfig_ESConfig(t *testing.T) { +func TestAPIKeyAgentAuth_ESConfig(t *testing.T) { for name, tc := range map[string]struct { - cfg *common.Config - esCfg *common.Config - - expectedConfig APIKeyConfig - expectedErr error + cfg *common.Config + esCfg *common.Config + expectedConfig APIKeyAgentAuth }{ "default": { - cfg: common.NewConfig(), - expectedConfig: defaultAPIKeyConfig(), + cfg: nil, + expectedConfig: defaultAPIKeyAgentAuth(), }, "ES config missing": { cfg: common.MustNewConfigFrom(`{"enabled": true}`), - expectedConfig: APIKeyConfig{ + expectedConfig: APIKeyAgentAuth{ Enabled: true, - LimitPerMin: apiKeyLimit, + LimitPerMin: 100, ESConfig: elasticsearch.DefaultConfig(), + configured: true, }, }, "ES configured": { cfg: common.MustNewConfigFrom(`{"enabled": true, "elasticsearch.timeout":"7s"}`), esCfg: common.MustNewConfigFrom(`{"hosts":["186.0.0.168:9200"]}`), - expectedConfig: APIKeyConfig{ + expectedConfig: APIKeyAgentAuth{ Enabled: true, - LimitPerMin: apiKeyLimit, + LimitPerMin: 100, ESConfig: &elasticsearch.Config{ Hosts: elasticsearch.Hosts{"localhost:9200"}, Protocol: "http", @@ -62,18 +60,19 @@ func TestAPIKeyConfig_ESConfig(t *testing.T) { MaxRetries: 3, Backoff: elasticsearch.DefaultBackoffConfig, }, + configured: true, esConfigured: true, }, }, "disabled with ES from output": { - cfg: common.NewConfig(), + cfg: nil, esCfg: common.MustNewConfigFrom(`{"hosts":["192.0.0.168:9200"]}`), - expectedConfig: defaultAPIKeyConfig(), + expectedConfig: defaultAPIKeyAgentAuth(), }, "ES from output": { cfg: common.MustNewConfigFrom(`{"enabled": true, "limit": 20}`), esCfg: common.MustNewConfigFrom(`{"hosts":["192.0.0.168:9200"],"username":"foo","password":"bar"}`), - expectedConfig: APIKeyConfig{ + expectedConfig: APIKeyAgentAuth{ Enabled: true, LimitPerMin: 20, ESConfig: &elasticsearch.Config{ @@ -85,19 +84,50 @@ func TestAPIKeyConfig_ESConfig(t *testing.T) { MaxRetries: 3, Backoff: elasticsearch.DefaultBackoffConfig, }, + configured: true, }, }, } { t.Run(name, func(t *testing.T) { - apiKeyConfig := defaultAPIKeyConfig() - require.NoError(t, tc.cfg.Unpack(&apiKeyConfig)) - err := apiKeyConfig.setup(logp.NewLogger("api_key"), tc.esCfg) - if tc.expectedErr == nil { - assert.NoError(t, err) - } else { - assert.Error(t, err) + for _, key := range []string{"api_key", "auth.api_key"} { + input := common.NewConfig() + if tc.cfg != nil { + input.SetChild(key, -1, tc.cfg) + } + cfg, err := NewConfig(input, tc.esCfg) + require.NoError(t, err) + assert.Equal(t, tc.expectedConfig, cfg.AgentAuth.APIKey) } - assert.Equal(t, tc.expectedConfig, apiKeyConfig) + }) + } +} + +func TestSecretTokenAuth(t *testing.T) { + for name, tc := range map[string]struct { + cfg *common.Config + expected string + }{ + "default": { + cfg: common.NewConfig(), + expected: "", + }, + "secret_token_auth": { + cfg: common.MustNewConfigFrom(`{"auth.secret_token":"token-one"}`), + expected: "token-one", + }, + "deprecated_secret_token": { + cfg: common.MustNewConfigFrom(`{"secret_token":"token-two"}`), + expected: "token-two", + }, + "deprecated_secret_token_conflict": { + cfg: common.MustNewConfigFrom(`{"auth.secret_token":"token-one","secret_token":"token-two"}`), + expected: "token-one", + }, + } { + t.Run(name, func(t *testing.T) { + cfg, err := NewConfig(tc.cfg, nil) + require.NoError(t, err) + assert.Equal(t, tc.expected, cfg.AgentAuth.SecretToken) }) } } diff --git a/beater/config/config.go b/beater/config/config.go index 3c09fdf5007..579b57cd8ba 100644 --- a/beater/config/config.go +++ b/beater/config/config.go @@ -43,7 +43,13 @@ var ( // Config holds configuration information nested under the key `apm-server` type Config struct { - Host string `config:"host"` + // Host holds the hostname or address that the server should bind to + // when listening for requests from agents. + Host string `config:"host"` + + // AgentAuth holds agent auth config. + AgentAuth AgentAuth `config:"auth"` + MaxHeaderSize int `config:"max_header_size"` IdleTimeout time.Duration `config:"idle_timeout"` ReadTimeout time.Duration `config:"read_timeout"` @@ -62,8 +68,6 @@ type Config struct { Mode Mode `config:"mode"` Kibana KibanaConfig `config:"kibana"` KibanaAgentConfig KibanaAgentConfig `config:"agent.config"` - SecretToken string `config:"secret_token"` - APIKeyConfig APIKeyConfig `config:"api_key"` JaegerConfig JaegerConfig `config:"jaeger"` Aggregation AggregationConfig `config:"aggregation"` Sampling SamplingConfig `config:"sampling"` @@ -93,11 +97,15 @@ func NewConfig(ucfg *common.Config, outputESCfg *common.Config) (*Config, error) } } + if err := setDeprecatedConfig(c, ucfg, logger); err != nil { + return nil, err + } + if err := c.RumConfig.setup(logger, c.DataStreams.Enabled, outputESCfg); err != nil { return nil, err } - if err := c.APIKeyConfig.setup(logger, outputESCfg); err != nil { + if err := c.AgentAuth.APIKey.setup(logger, outputESCfg); err != nil { return nil, err } @@ -128,6 +136,40 @@ func NewConfig(ucfg *common.Config, outputESCfg *common.Config) (*Config, error) return c, nil } +// setDeprecatedConfig translates deprecated top-level config attributes to the +// current config structure. +func setDeprecatedConfig(out *Config, in *common.Config, logger *logp.Logger) error { + var deprecatedConfig struct { + APIKey APIKeyAgentAuth `config:"api_key"` + SecretToken string `config:"secret_token"` + } + deprecatedConfig.APIKey = defaultAPIKeyAgentAuth() + if err := in.Unpack(&deprecatedConfig); err != nil { + return err + } + + warnIgnored := func(deprecated, replacement string) { + logger.Warnf("ignoring deprecated config %q as %q is defined", deprecated, replacement) + } + if deprecatedConfig.APIKey.configured { + // "apm-server.api_key" -> "apm-server.auth.api_key" + if out.AgentAuth.APIKey.configured { + warnIgnored("apm-server.api_key", "apm-server.auth.api_key") + } else { + out.AgentAuth.APIKey = deprecatedConfig.APIKey + } + } + if deprecatedConfig.SecretToken != "" { + // "apm-server.secret_token" -> "apm-server.auth.secret_token" + if out.AgentAuth.SecretToken != "" { + warnIgnored("apm-server.secret_token", "apm-server.auth.secret_token") + } else { + out.AgentAuth.SecretToken = deprecatedConfig.SecretToken + } + } + return nil +} + // DefaultConfig returns a config with default settings for `apm-server` config options. func DefaultConfig() *Config { return &Config{ @@ -152,10 +194,10 @@ func DefaultConfig() *Config { Kibana: defaultKibanaConfig(), KibanaAgentConfig: defaultKibanaAgentConfig(), Pipeline: defaultAPMPipeline, - APIKeyConfig: defaultAPIKeyConfig(), JaegerConfig: defaultJaeger(), Aggregation: defaultAggregationConfig(), Sampling: defaultSamplingConfig(), DataStreams: defaultDataStreamsConfig(), + AgentAuth: defaultAgentAuth(), } } diff --git a/beater/config/config_test.go b/beater/config/config_test.go index eaf40ee5345..9b391b9cc2e 100644 --- a/beater/config/config_test.go +++ b/beater/config/config_test.go @@ -148,7 +148,22 @@ func TestUnpackConfig(t *testing.T) { ReadTimeout: 3000000000, WriteTimeout: 4000000000, ShutdownTimeout: 9000000000, - SecretToken: "1234random", + AgentAuth: AgentAuth{ + SecretToken: "1234random", + APIKey: APIKeyAgentAuth{ + Enabled: true, + LimitPerMin: 200, + ESConfig: &elasticsearch.Config{ + Hosts: elasticsearch.Hosts{"localhost:9201", "localhost:9202"}, + Protocol: "http", + Timeout: 5 * time.Second, + MaxRetries: 3, + Backoff: elasticsearch.DefaultBackoffConfig, + }, + configured: true, + esConfigured: true, + }, + }, TLS: &tlscommon.ServerConfig{ Enabled: newBool(true), Certificate: testdataCertificateConfig, @@ -233,18 +248,6 @@ func TestUnpackConfig(t *testing.T) { Host: "localhost:6789", }, }, - APIKeyConfig: APIKeyConfig{ - Enabled: true, - LimitPerMin: 200, - ESConfig: &elasticsearch.Config{ - Hosts: elasticsearch.Hosts{"localhost:9201", "localhost:9202"}, - Protocol: "http", - Timeout: 5 * time.Second, - MaxRetries: 3, - Backoff: elasticsearch.DefaultBackoffConfig, - }, - esConfigured: true, - }, Aggregation: AggregationConfig{ Transactions: TransactionAggregationConfig{ Enabled: false, @@ -325,7 +328,15 @@ func TestUnpackConfig(t *testing.T) { ReadTimeout: 30000000000, WriteTimeout: 30000000000, ShutdownTimeout: 5000000000, - SecretToken: "1234random", + AgentAuth: AgentAuth{ + SecretToken: "1234random", + APIKey: APIKeyAgentAuth{ + Enabled: true, + LimitPerMin: 100, + ESConfig: elasticsearch.DefaultConfig(), + configured: true, + }, + }, TLS: &tlscommon.ServerConfig{ Enabled: newBool(true), Certificate: testdataCertificateConfig, @@ -399,7 +410,6 @@ func TestUnpackConfig(t *testing.T) { Host: "localhost:14268", }, }, - APIKeyConfig: APIKeyConfig{Enabled: true, LimitPerMin: 100, ESConfig: elasticsearch.DefaultConfig()}, Aggregation: AggregationConfig{ Transactions: TransactionAggregationConfig{ Enabled: false, @@ -595,7 +605,7 @@ func TestNewConfig_ESConfig(t *testing.T) { cfg, err := NewConfig(ucfg, nil) require.NoError(t, err) assert.Equal(t, elasticsearch.DefaultConfig(), cfg.RumConfig.SourceMapping.ESConfig) - assert.Equal(t, elasticsearch.DefaultConfig(), cfg.APIKeyConfig.ESConfig) + assert.Equal(t, elasticsearch.DefaultConfig(), cfg.AgentAuth.APIKey.ESConfig) assert.Equal(t, elasticsearch.DefaultConfig(), cfg.Sampling.Tail.ESConfig) // with es config @@ -604,8 +614,8 @@ func TestNewConfig_ESConfig(t *testing.T) { require.NoError(t, err) assert.NotNil(t, cfg.RumConfig.SourceMapping.ESConfig) assert.Equal(t, []string{"192.0.0.168:9200"}, []string(cfg.RumConfig.SourceMapping.ESConfig.Hosts)) - assert.NotNil(t, cfg.APIKeyConfig.ESConfig) - assert.Equal(t, []string{"192.0.0.168:9200"}, []string(cfg.APIKeyConfig.ESConfig.Hosts)) + assert.NotNil(t, cfg.AgentAuth.APIKey.ESConfig) + assert.Equal(t, []string{"192.0.0.168:9200"}, []string(cfg.AgentAuth.APIKey.ESConfig.Hosts)) assert.NotNil(t, cfg.Sampling.Tail.ESConfig) assert.Equal(t, []string{"192.0.0.168:9200"}, []string(cfg.Sampling.Tail.ESConfig.Hosts)) } diff --git a/beater/http.go b/beater/http.go index a62e4ef3ee7..994e4b396ed 100644 --- a/beater/http.go +++ b/beater/http.go @@ -133,7 +133,7 @@ func (h *httpServer) start() error { h.logger.Info("SSL enabled.") return h.ServeTLS(lis, "", "") } - if h.cfg.SecretToken != "" { + if h.cfg.AgentAuth.SecretToken != "" { h.logger.Warn("Secret token is set, but SSL is not enabled.") } h.logger.Info("SSL disabled.") diff --git a/beater/jaeger/server.go b/beater/jaeger/server.go index 2a15c9c1d0a..42ef75b226d 100644 --- a/beater/jaeger/server.go +++ b/beater/jaeger/server.go @@ -85,7 +85,7 @@ func NewServer( // must explicitly specify which tag to use. // TODO(axw) share auth builder with beater/api. var err error - authBuilder, err = authorization.NewBuilder(cfg) + authBuilder, err = authorization.NewBuilder(cfg.AgentAuth) if err != nil { return nil, err } diff --git a/beater/jaeger/server_test.go b/beater/jaeger/server_test.go index 9555ed670f1..c51b58f4b41 100644 --- a/beater/jaeger/server_test.go +++ b/beater/jaeger/server_test.go @@ -232,7 +232,7 @@ func TestServerIntegration(t *testing.T) { "secret token set but no auth_tag": { cfg: func() *config.Config { cfg := config.DefaultConfig() - cfg.SecretToken = "hunter2" + cfg.AgentAuth.SecretToken = "hunter2" cfg.JaegerConfig.GRPC.Enabled = true cfg.JaegerConfig.GRPC.Host = "localhost:0" cfg.JaegerConfig.HTTP.Enabled = true @@ -244,7 +244,7 @@ func TestServerIntegration(t *testing.T) { "secret token and auth_tag set, but no auth_tag sent by agent": { cfg: func() *config.Config { cfg := config.DefaultConfig() - cfg.SecretToken = "hunter2" + cfg.AgentAuth.SecretToken = "hunter2" cfg.JaegerConfig.GRPC.Enabled = true cfg.JaegerConfig.GRPC.Host = "localhost:0" cfg.JaegerConfig.GRPC.AuthTag = "authorization" @@ -256,7 +256,7 @@ func TestServerIntegration(t *testing.T) { "secret token and auth_tag set, auth_tag sent by agent": { cfg: func() *config.Config { cfg := config.DefaultConfig() - cfg.SecretToken = "hunter2" + cfg.AgentAuth.SecretToken = "hunter2" cfg.JaegerConfig.GRPC.Enabled = true cfg.JaegerConfig.GRPC.Host = "localhost:0" cfg.JaegerConfig.GRPC.AuthTag = "authorization" diff --git a/beater/middleware/authorization_middleware_test.go b/beater/middleware/authorization_middleware_test.go index d4e5b2bb42f..9a254dcea42 100644 --- a/beater/middleware/authorization_middleware_test.go +++ b/beater/middleware/authorization_middleware_test.go @@ -72,7 +72,7 @@ func TestAuthorizationMiddleware(t *testing.T) { if tc.header != "" { c.Request.Header.Set(headers.Authorization, tc.header) } - builder, err := authorization.NewBuilder(&config.Config{SecretToken: token}) + builder, err := authorization.NewBuilder(config.AgentAuth{SecretToken: token}) require.NoError(t, err) return builder.ForAnyOfPrivileges(authorization.ActionAny), c, rec } diff --git a/beater/server.go b/beater/server.go index 467af756bfd..7d144321402 100644 --- a/beater/server.go +++ b/beater/server.go @@ -108,7 +108,14 @@ type server struct { jaegerServer *jaeger.Server } -func newServer(logger *logp.Logger, info beat.Info, cfg *config.Config, tracer *apm.Tracer, reporter publish.Reporter, batchProcessor model.BatchProcessor) (server, error) { +func newServer( + logger *logp.Logger, + info beat.Info, + cfg *config.Config, + tracer *apm.Tracer, + reporter publish.Reporter, + batchProcessor model.BatchProcessor, +) (server, error) { fetcher := agentcfg.NewFetcher(cfg) httpServer, err := newHTTPServer(logger, info, cfg, tracer, reporter, batchProcessor, fetcher) if err != nil { @@ -140,7 +147,7 @@ func newGRPCServer( fetcher agentcfg.Fetcher, ) (*grpc.Server, error) { // TODO(axw) share auth builder with beater/api. - authBuilder, err := authorization.NewBuilder(cfg) + authBuilder, err := authorization.NewBuilder(cfg.AgentAuth) if err != nil { return nil, err } diff --git a/beater/telemetry.go b/beater/telemetry.go index feb0a8295f5..6f9e3d6e6c0 100644 --- a/beater/telemetry.go +++ b/beater/telemetry.go @@ -93,7 +93,7 @@ func recordRootConfig(info beat.Info, rootCfg *common.Config) error { // This should be called once each time runServer is called. func recordAPMServerConfig(cfg *config.Config) { configMonitors.rumEnabled.Set(cfg.RumConfig.Enabled) - configMonitors.apiKeysEnabled.Set(cfg.APIKeyConfig.Enabled) + configMonitors.apiKeysEnabled.Set(cfg.AgentAuth.APIKey.Enabled) configMonitors.kibanaEnabled.Set(cfg.Kibana.Enabled) configMonitors.jaegerHTTPEnabled.Set(cfg.JaegerConfig.HTTP.Enabled) configMonitors.jaegerGRPCEnabled.Set(cfg.JaegerConfig.GRPC.Enabled) diff --git a/beater/telemetry_test.go b/beater/telemetry_test.go index 0604ac702fd..cae9c025baa 100644 --- a/beater/telemetry_test.go +++ b/beater/telemetry_test.go @@ -34,7 +34,7 @@ func TestRecordConfigs(t *testing.T) { info := beat.Info{Name: "apm-server", Version: "7.x"} apmCfg := config.DefaultConfig() - apmCfg.APIKeyConfig.Enabled = true + apmCfg.AgentAuth.APIKey.Enabled = true apmCfg.Kibana.Enabled = true apmCfg.JaegerConfig.GRPC.Enabled = true apmCfg.JaegerConfig.HTTP.Enabled = true diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc new file mode 100644 index 00000000000..b14315593f3 --- /dev/null +++ b/changelogs/head.asciidoc @@ -0,0 +1,37 @@ +[[release-notes-head]] +== APM Server version HEAD + +https://github.com/elastic/apm-server/compare/7.13\...master[View commits] + +[float] +==== Breaking Changes +* Removed monitoring counters `apm-server.processor.stream.errors.{queue,server,closed}` {pull}5453[5453] + +[float] +==== Bug fixes +* Fix panic due to misaligned 64-bit access on 32-bit architectures {pull}5277[5277] +* Fixed tail-based sampling pubsub to use _seq_no correctly {pull}5126[5126] +* Removed service name from dataset {pull}5451[5451] + +[float] +==== Intake API Changes + +[float] +==== Added +* Support setting agent configuration from apm-server.yml {pull}5177[5177] +* Add metric_type and unit to field metadata of system metrics {pull}5230[5230] +* Display apm-server url in fleet ui's apm-server integration {pull}4895[4895] +* Translate otel messaging.* semantic conventions to ECS {pull}5334[5334] +* Add support for dynamic histogram metrics {pull}5239[5239] +* Tail-sampling processor now resumes subscription from previous position after restart {pull}5350[5350] +* Add support for histograms to metrics intake {pull}5360[5360] +* Upgrade Go to 1.16.5 {pull}5454[5454] +* Add units to metric fields {pull}5395[5395] +* Add support for adjusting OTel event timestamps using `telemetry.sdk.elastic_export_timestamp` {pull}5433[5433] +* Add support for OpenTelemetry labels describing mobile connectivity {pull}5436[5436] +* Introduce `apm-server.auth.*` config {pull}5457[5457] + +[float] +==== Deprecated +* `apm-server.secret_token` is now `apm-server.auth.secret_token` {pull}5457[5457] +* `apm-server.api_key` is now `apm-server.auth.api_key` {pull}5457[5457] diff --git a/cmd/apikey.go b/cmd/apikey.go index b9f1f7bd1c4..f39354d7fe6 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -245,7 +245,7 @@ func bootstrap(settings instance.Settings) (es.Client, *config.Config, error) { return nil, nil, err } - client, err := es.NewClient(beaterConfig.APIKeyConfig.ESConfig) + client, err := es.NewClient(beaterConfig.AgentAuth.APIKey.ESConfig) if err != nil { return nil, nil, err } @@ -422,7 +422,7 @@ func verifyAPIKey(config *config.Config, privileges []es.PrivilegeAction, creden perms := make(es.Permissions) printText, printJSON := printers(asJSON) for _, privilege := range privileges { - builder, err := auth.NewBuilder(config) + builder, err := auth.NewBuilder(config.AgentAuth) if err != nil { return err } diff --git a/docs/configuration-process.asciidoc b/docs/configuration-process.asciidoc index ed4f795a20b..0304be8889e 100644 --- a/docs/configuration-process.asciidoc +++ b/docs/configuration-process.asciidoc @@ -90,7 +90,7 @@ Default value is 0, which means _unlimited_. [[config-secret-token]] [float] -==== `secret_token` +==== `auth.secret_token` Authorization token for sending data to the APM server. If a token is set, the agents must send it in the following format: Authorization: Bearer . @@ -99,6 +99,15 @@ The token is not used for RUM endpoints. By default, no authorization token is s It is recommended to use an authorization token in combination with SSL enabled. Read more about <> and the <>. +[[config-secret-token-legacy]] +[float] +==== `secret_token` + +deprecated::[7.14.0, Replaced by `auth.secret_token`. See <>] + +In versions prior to 7.14.0, secret token authorization was known as `apm-server.secret_token`. In 7.14.0 this was renamed `apm-server.auth.secret_token`. +The old configuration will continue to work until 8.0.0, and the new configuration will take precedence. + [[capture_personal_data]] [float] ==== `capture_personal_data` diff --git a/docs/secure-communication-agents.asciidoc b/docs/secure-communication-agents.asciidoc index 3115a418f63..744924b1831 100644 --- a/docs/secure-communication-agents.asciidoc +++ b/docs/secure-communication-agents.asciidoc @@ -35,7 +35,7 @@ You can configure API keys to authorize requests to the APM Server. NOTE: API keys are sent as plain-text, so they only provide security when used in combination with <>. -By enabling `apm-server.api_key.enabled: true`, you ensure that only agents with a valid API Key +By enabling `apm-server.auth.api_key.enabled: true`, you ensure that only agents with a valid API Key are able to successfully use APM Server's API (except for RUM endpoints). To secure the communication between APM Agents and the APM Server with API keys: @@ -53,16 +53,16 @@ as there is no way to prevent them from being publicly exposed. === Enable and configure API keys API keys are disabled by default. You can change this, and additional settings, -in the `apm-server.api_key` section of the +{beatname_lc}.yml+ configuration file. +in the `apm-server.auth.api_key` section of the +{beatname_lc}.yml+ configuration file. At a minimum, you must enable API keys, and should set a limit on the number of unique API keys that APM Server allows per minute. -Here's an example `apm-server.api_key` config using 50 unique API keys: +Here's an example `apm-server.auth.api_key` config using 50 unique API keys: [source,yaml] ---- -apm-server.api_key.enabled: true <1> -apm-server.api_key.limit: 50 <2> +apm-server.auth.api_key.enabled: true <1> +apm-server.auth.api_key.limit: 50 <2> ---- <1> Enables API keys <2> Restricts the number of unique API keys that {es} allows each minute. @@ -290,10 +290,15 @@ See the relevant agent documentation for additional information: * *Python agent*: {apm-py-ref}/configuration.html#config-api-key[`api_key`] * *Ruby agent*: {apm-ruby-ref}/configuration.html#config-api-key[`api_key`] +[float] [[api-key-settings]] -=== `api_key.*` configuration options +=== API key configuration options + +[float] +[[api-key-auth-settings]] +==== `auth.api_key.*` configuration options -You can specify the following options in the `apm-server.api_key.*` section of the +You can specify the following options in the `apm-server.auth.api_key.*` section of the +{beatname_lc}.yml+ configuration file. They apply to API key communication between the APM Server and APM Agents. @@ -318,9 +323,9 @@ The minimum value for this setting should be the number of API keys configured i The default `limit` is `100`. [float] -=== `api_key.elasticsearch.*` configuration options +==== `auth.api_key.elasticsearch.*` configuration options -All of the `api_key.elasticsearch.*` configurations are optional. +All of the `auth.api_key.elasticsearch.*` configurations are optional. If none are set, configuration settings from the `apm-server.output` section will be reused. [float] @@ -357,7 +362,7 @@ The http request timeout in seconds for the Elasticsearch request. If nothing is configured, configuration settings from the `output` section will be reused. [float] -==== `api_key.elasticsearch.ssl.*` configuration options +==== `auth.api_key.elasticsearch.ssl.*` configuration options SSL is off by default. Set `elasticsearch.protocol` to `https` if you want to enable `https`. @@ -424,6 +429,15 @@ Valid options are `never`, `once`, and `freely`. Default is `never`. * `once` - Allows a remote server to request renegotiation once per connection. * `freely` - Allows a remote server to repeatedly request renegotiation. +[float] +[[api-key-settings-legacy]] +==== `api_key.*` configuration options + +deprecated::[7.14.0, Replaced by `auth.api_key.*`. See <>] + +In versions prior to 7.14.0, API Key authorization was known as `apm-server.api_key`. In 7.14.0 this was renamed `apm-server.auth.api_key`. +The old configuration will continue to work until 8.0.0, and the new configuration will take precedence. + [[secret-token]] === Secret token @@ -453,7 +467,7 @@ Here's how you set the secret token in APM Server: [source,yaml] ---- -apm-server.secret_token: +apm-server.auth.secret_token: ---- We recommend saving the token in the APM Server <>. diff --git a/systemtest/rum_test.go b/systemtest/rum_test.go index 93c7d82f84d..7f72a3ea6d2 100644 --- a/systemtest/rum_test.go +++ b/systemtest/rum_test.go @@ -123,6 +123,7 @@ func TestRUMAuth(t *testing.T) { func TestRUMAllowServiceNames(t *testing.T) { srv := apmservertest.NewUnstartedServer(t) + srv.Config.SecretToken = "abc123" srv.Config.RUM = &apmservertest.RUMConfig{ Enabled: true, AllowServiceNames: []string{"allowed"},