Add /status/overrides/{tenant} endpoint

CHANGELOG.md

@@ -40,6 +40,7 @@
 * [ENHANCEMENT] Added a `parquet-page` cache role for page level caching. [#3196](https://github.com/grafana/tempo/pull/3196) (@joe-elliott)
 * [ENHANCEMENT] Update opentelemetry-collector-contrib dependency to the latest version, v0.89.0 [#3148](https://github.com/grafana/tempo/pull/3148) (@gebn)
 * [ENHANCEMENT] Update memcached default image in jsonnet for multiple CVE [#3310](https://github.com/grafana/tempo/pull/3310) (@zalegrala)
+* [ENHANCEMENT] Add /status/overrides/{tenant} endpoint [#3244](https://github.com/grafana/tempo/pull/3244) (@kvrhdn)
 * [BUGFIX] Prevent building parquet iterators that would loop forever. [#3159](https://github.com/grafana/tempo/pull/3159) (@mapno)
 * [BUGFIX] Sanitize name in mapped dimensions in span-metrics processor [#3171](https://github.com/grafana/tempo/pull/3171) (@mapno)
 * [BUGFIX] Fixed an issue where cached footers were requested then ignored. [#3196](https://github.com/grafana/tempo/pull/3196) (@joe-elliott)

cmd/tempo-cli/cmd-migrate-overrides-config.go

@@ -10,11 +10,11 @@ import (
 	"os"
 
 	"github.com/grafana/dskit/services"
-
-	"github.com/grafana/tempo/modules/overrides"
+	"github.com/prometheus/client_golang/prometheus"
+	"gopkg.in/yaml.v2"
 
 	"github.com/grafana/tempo/cmd/tempo/app"
-	"gopkg.in/yaml.v2"
+	"github.com/grafana/tempo/modules/overrides"
 )
 
 type migrateOverridesConfigCmd struct {
@@ -39,7 +39,7 @@ func (cmd *migrateOverridesConfigCmd) Run(*globalOptions) error {
 		return fmt.Errorf("failed to parse configFile %s: %w", cmd.ConfigFile, err)
 	}
 
-	o, err := overrides.NewOverrides(cfg.Overrides)
+	o, err := overrides.NewOverrides(cfg.Overrides, prometheus.DefaultRegisterer)
 	if err != nil {
 		return fmt.Errorf("failed to load overrides module: %w", err)
 	}

cmd/tempo/app/app.go

@@ -199,6 +199,7 @@ func (t *App) Run() error {
 	t.Server.HTTP().Path("/ready").Handler(t.readyHandler(sm))
 	t.Server.HTTP().Path("/status").Handler(t.statusHandler()).Methods("GET")
 	t.Server.HTTP().Path("/status/{endpoint}").Handler(t.statusHandler()).Methods("GET")
+	t.Server.HTTP().Path("/status/overrides/{tenant}").Handler(t.tenantOverridesHandler()).Methods("GET")
 	grpc_health_v1.RegisterHealthServer(t.Server.GRPC(), grpcutil.NewHealthCheck(sm))
 
 	// Let's listen for events from this manager, and log them.
@@ -372,6 +373,8 @@ func (t *App) statusHandler() http.HandlerFunc {
 
 			switch endpoint {
 			case "runtime_config":
+				fallthrough
+			case "overrides":
 				err := t.writeRuntimeConfig(&msg, r)
 				if err != nil {
 					errs = append(errs, err)
@@ -520,6 +523,25 @@ func (t *App) writeStatusEndpoints(w io.Writer) error {
 	return nil
 }
 
+func (t *App) tenantOverridesHandler() http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		vars := mux.Vars(r)
+
+		tenant, ok := vars["tenant"]
+		if !ok {
+			http.Error(w, "must specify tenant ID", http.StatusBadRequest)
+			return
+		}
+
+		err := t.Overrides.WriteTenantOverrides(w, r, tenant)
+		if err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+			level.Error(log.Logger).Log("msg", "error writing response", "endpoint", r.URL.String(), "err", err)
+			return
+		}
+	}
+}
+
 func (t *App) buildinfoHandler() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "application/json")

cmd/tempo/app/modules.go

@@ -170,7 +170,7 @@ func (t *App) initReadRing(cfg ring.Config, name, key string) (*ring.Ring, error
 }
 
 func (t *App) initOverrides() (services.Service, error) {
-	o, err := overrides.NewOverrides(t.cfg.Overrides)
+	o, err := overrides.NewOverrides(t.cfg.Overrides, prometheus.DefaultRegisterer)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create overrides: %w", err)
 	}

docs/sources/tempo/api_docs/_index.md

@@ -560,6 +560,14 @@ Displays the override configuration.
 Query parameter:
 - `mode = (diff)`: Show the difference between defaults and overrides.
 
+Alias: `/status/overrides`
+
+```
+GET /status/overrides/{tenant}
+```
+
+Displays the override configuration for the specified tenant.
+
 ```
 GET /status/usage-stats
 ```
@@ -634,4 +642,4 @@ message SearchMetrics {
   uint32 totalJobs = 5;
   uint64 totalBlockBytes = 6;
 }
-```
+```

modules/compactor/compactor_test.go

@@ -4,6 +4,7 @@ import (
 	"math"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -23,7 +24,7 @@ func TestCombineLimitsNotHit(t *testing.T) {
 				MaxBytesPerTrace: math.MaxInt,
 			},
 		},
-	})
+	}, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	c := &Compactor{
@@ -57,7 +58,7 @@ func TestCombineLimitsHit(t *testing.T) {
 				MaxBytesPerTrace: 1,
 			},
 		},
-	})
+	}, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	c := &Compactor{
@@ -91,7 +92,7 @@ func TestCombineDoesntEnforceZero(t *testing.T) {
 				MaxBytesPerTrace: math.MaxInt,
 			},
 		},
-	})
+	}, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	c := &Compactor{
@@ -146,7 +147,7 @@ func TestDedicatedColumns(t *testing.T) {
 				},
 			},
 		},
-	})
+	}, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	c := &Compactor{overrides: o}

modules/distributor/distributor_test.go

@@ -1393,7 +1393,7 @@ func prepare(t *testing.T, limits overrides.Config, logger kitlog.Logger) *Distr
 	)
 	flagext.DefaultValues(&clientConfig)
 
-	overrides, err := overrides.NewOverrides(limits)
+	overrides, err := overrides.NewOverrides(limits, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	// Mock the ingesters ring

modules/distributor/forwarder_test.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/go-kit/log"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
@@ -30,7 +31,7 @@ func TestForwarder(t *testing.T) {
 	keys, rebatchedTraces, err := requestsByTraceID([]*v1.ResourceSpans{b}, tenantID, 10)
 	require.NoError(t, err)
 
-	o, err := overrides.NewOverrides(oCfg)
+	o, err := overrides.NewOverrides(oCfg, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	wg := sync.WaitGroup{}
@@ -71,7 +72,7 @@ func TestForwarder_shutdown(t *testing.T) {
 	keys, rebatchedTraces, err := requestsByTraceID([]*v1.ResourceSpans{b}, tenantID, 10)
 	require.NoError(t, err)
 
-	o, err := overrides.NewOverrides(oCfg)
+	o, err := overrides.NewOverrides(oCfg, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	signalCh := make(chan struct{})

modules/distributor/ingestion_rate_strategy_test.go

@@ -4,6 +4,7 @@ import (
 	"testing"
 
 	"github.com/grafana/dskit/limiter"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
@@ -57,7 +58,7 @@ func TestIngestionRateStrategy(t *testing.T) {
 			// Init limits overrides
 			o, err := overrides.NewOverrides(overrides.Config{
 				Defaults: testData.limits,
-			})
+			}, prometheus.DefaultRegisterer)
 			require.NoError(t, err)
 
 			// Instance the strategy

modules/frontend/search_streaming_test.go

@@ -13,12 +13,14 @@ import (
 	"github.com/golang/protobuf/jsonpb" //nolint:all //deprecated
 	"github.com/google/uuid"
 	"github.com/grafana/dskit/user"
-	"github.com/grafana/tempo/modules/overrides"
-	"github.com/grafana/tempo/pkg/tempopb"
-	"github.com/grafana/tempo/tempodb/backend"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/atomic"
 	"google.golang.org/grpc/metadata"
+
+	"github.com/grafana/tempo/modules/overrides"
+	"github.com/grafana/tempo/pkg/tempopb"
+	"github.com/grafana/tempo/tempodb/backend"
 )
 
 type mockStreamingServer struct {
@@ -209,7 +211,7 @@ func TestStreamingSearchHandlerFailsDueToError(t *testing.T) {
 func testHandler(t *testing.T, next http.RoundTripper) streamingSearchHandler {
 	t.Helper()
 
-	o, err := overrides.NewOverrides(overrides.Config{})
+	o, err := overrides.NewOverrides(overrides.Config{}, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	handler := newSearchStreamingGRPCHandler(Config{

modules/frontend/searchsharding_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/golang/protobuf/jsonpb" //nolint:all deprecated
 	"github.com/google/uuid"
 	"github.com/grafana/dskit/user"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -702,7 +703,7 @@ func TestSearchSharderRoundTrip(t *testing.T) {
 				}, nil
 			})
 
-			o, err := overrides.NewOverrides(overrides.Config{})
+			o, err := overrides.NewOverrides(overrides.Config{}, prometheus.DefaultRegisterer)
 			require.NoError(t, err)
 
 			sharder := newSearchSharder(&mockReader{
@@ -765,7 +766,7 @@ func TestTotalJobsIncludesIngester(t *testing.T) {
 		}, nil
 	})
 
-	o, err := overrides.NewOverrides(overrides.Config{})
+	o, err := overrides.NewOverrides(overrides.Config{}, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	now := time.Now().Add(-10 * time.Minute).Unix()
@@ -812,7 +813,7 @@ func TestSearchSharderRoundTripBadRequest(t *testing.T) {
 		return nil, nil
 	})
 
-	o, err := overrides.NewOverrides(overrides.Config{})
+	o, err := overrides.NewOverrides(overrides.Config{}, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	sharder := newSearchSharder(&mockReader{}, o, SearchSharderConfig{
@@ -845,7 +846,7 @@ func TestSearchSharderRoundTripBadRequest(t *testing.T) {
 				MaxSearchDuration: model.Duration(time.Minute),
 			},
 		},
-	})
+	}, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	sharder = newSearchSharder(&mockReader{}, o, SearchSharderConfig{
@@ -884,7 +885,7 @@ func TestSharderAccessesCache(t *testing.T) {
 	// setup mock cache
 	c := cache.NewMockCache()
 
-	o, err := overrides.NewOverrides(overrides.Config{})
+	o, err := overrides.NewOverrides(overrides.Config{}, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	meta := &backend.BlockMeta{
@@ -1003,7 +1004,7 @@ func TestAdjustLimit(t *testing.T) {
 
 func TestMaxDuration(t *testing.T) {
 	//
-	o, err := overrides.NewOverrides(overrides.Config{})
+	o, err := overrides.NewOverrides(overrides.Config{}, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 	sharder := searchSharder{
 		cfg: SearchSharderConfig{
@@ -1020,7 +1021,7 @@ func TestMaxDuration(t *testing.T) {
 				MaxSearchDuration: model.Duration(10 * time.Minute),
 			},
 		},
-	})
+	}, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 	sharder = searchSharder{
 		cfg: SearchSharderConfig{
@@ -1089,7 +1090,7 @@ func TestSubRequestsCancelled(t *testing.T) {
 		return httptest.NewRecorder().Result(), nil
 	})
 
-	o, err := overrides.NewOverrides(overrides.Config{})
+	o, err := overrides.NewOverrides(overrides.Config{}, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	sharder := newSearchSharder(&mockReader{
@@ -1352,7 +1353,7 @@ func benchmarkSearchSharderRoundTrip(b *testing.B, s int32) {
 		}, nil
 	})
 
-	o, err := overrides.NewOverrides(overrides.Config{})
+	o, err := overrides.NewOverrides(overrides.Config{}, prometheus.DefaultRegisterer)
 	require.NoError(b, err)
 
 	totalMetas := 10000

modules/frontend/tracebyidsharding_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/gogo/protobuf/proto"
 	"github.com/grafana/dskit/user"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/atomic"
@@ -188,7 +189,7 @@ func TestShardingWareDoRequest(t *testing.T) {
 		},
 	}
 
-	o, err := overrides.NewOverrides(overrides.Config{})
+	o, err := overrides.NewOverrides(overrides.Config{}, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	for _, tc := range tests {
@@ -272,7 +273,7 @@ func TestShardingWareDoRequest(t *testing.T) {
 func TestConcurrentShards(t *testing.T) {
 	concurrency := 2
 
-	o, err := overrides.NewOverrides(overrides.Config{})
+	o, err := overrides.NewOverrides(overrides.Config{}, prometheus.DefaultRegisterer)
 	require.NoError(t, err)
 
 	sharder := newTraceByIDSharder(&TraceByIDConfig{

modules/ingester/ingester_test.go

@@ -407,7 +407,7 @@ func defaultIngesterModule(t testing.TB, tmpDir string) *Ingester {
 
 func defaultIngesterWithOverrides(t testing.TB, tmpDir string, o overrides.Config) *Ingester {
 	ingesterConfig := defaultIngesterTestConfig()
-	limits, err := overrides.NewOverrides(o)
+	limits, err := overrides.NewOverrides(o, prometheus.DefaultRegisterer)
 	require.NoError(t, err, "unexpected error creating overrides")
 
 	s, err := storage.NewStore(storage.Config{