diff --git a/CHANGELOG.md b/CHANGELOG.md index 71e8a807343..188784bab44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## main / unreleased +* [ENHANCEMENT] Add querier metrics for requests executed [#3524](https://github.com/grafana/tempo/pull/3524) (@electron0zero) * [FEATURE] Added gRPC streaming endpoints for all tag queries. [#3460](https://github.com/grafana/tempo/pull/3460) (@joe-elliott) * [CHANGE] Align metrics query time ranges to the step parameter [#3490](https://github.com/grafana/tempo/pull/3490) (@mdisibio) * [ENHANCEMENT] Add string interning to TraceQL queries [#3411](https://github.com/grafana/tempo/pull/3411) (@mapno) diff --git a/modules/querier/worker/frontend_processor.go b/modules/querier/worker/frontend_processor.go index a5d6b4cea35..1fb995b58da 100644 --- a/modules/querier/worker/frontend_processor.go +++ b/modules/querier/worker/frontend_processor.go @@ -15,6 +15,8 @@ import ( "github.com/grafana/dskit/httpgrpc" "github.com/grafana/tempo/pkg/util/httpgrpcutil" "github.com/opentracing/opentracing-go" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -27,6 +29,12 @@ var processorBackoffConfig = backoff.Config{ MaxBackoff: 1 * time.Second, } +var metricWorkerRequests = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "tempo", + Name: "querier_worker_request_executed_total", + Help: "The total number of requests executed by the querier worker.", +}) + func newFrontendProcessor(cfg Config, handler RequestHandler, log log.Logger) processor { return &frontendProcessor{ log: log, @@ -185,6 +193,8 @@ func (fp *frontendProcessor) runRequest(ctx context.Context, request *httpgrpc.H level.Error(fp.log).Log("msg", "error processing query", "err", errMsg) } + metricWorkerRequests.Inc() + return response } diff --git a/modules/querier/worker/frontend_processor_test.go b/modules/querier/worker/frontend_processor_test.go index 1a06e2f87c3..ef0903e90b8 100644 --- a/modules/querier/worker/frontend_processor_test.go +++ b/modules/querier/worker/frontend_processor_test.go @@ -11,6 +11,7 @@ import ( "github.com/go-kit/log" "github.com/grafana/dskit/grpcclient" "github.com/grafana/dskit/httpgrpc" + dto "github.com/prometheus/client_model/go" "github.com/stretchr/testify/require" ) @@ -46,6 +47,12 @@ func TestRunRequests(t *testing.T) { for i, resp := range resps { require.Equal(t, []byte{byte(i)}, resp.Body) } + + // check that counter metric is working + m := &dto.Metric{} + err := metricWorkerRequests.Write(m) + require.NoError(t, err) + require.Equal(t, float64(totalRequests), m.Counter.GetValue()) } func TestHandleSendError(t *testing.T) { diff --git a/modules/querier/worker/worker.go b/modules/querier/worker/worker.go index e556b593b48..4433bede1af 100644 --- a/modules/querier/worker/worker.go +++ b/modules/querier/worker/worker.go @@ -242,6 +242,8 @@ func (w *querierWorker) resetConcurrency() { if totalConcurrency > w.cfg.MaxConcurrentRequests { level.Warn(w.log).Log("msg", "total worker concurrency is greater than promql max concurrency. Queries may be queued in the querier which reduces QOS") } + + level.Info(w.log).Log("msg", "total worker concurrency updated", "totalConcurrency", totalConcurrency) } func (w *querierWorker) connect(ctx context.Context, address string) (*grpc.ClientConn, error) {