diff --git a/CHANGELOG.md b/CHANGELOG.md index d34d39d7816..98792eb16e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ ## main / unreleased +* [FEATURE] TraceQL support for instrumentation scope [#3967](https://github.com/grafana/tempo/pull/3967) (@ie-pham) * [ENHANCEMENT] Add bytes and spans received to usage stats [#3983](https://github.com/grafana/tempo/pull/3983) (@joe-elliott) # v2.6.0-rc.0 @@ -18,6 +19,7 @@ * [FEATURE] TraceQL support for event:timeSinceStart [#3908](https://github.com/grafana/tempo/pull/3908) (@ie-pham) * [FEATURE] Autocomplete support for events and links [#3846](https://github.com/grafana/tempo/pull/3846) (@ie-pham) * [FEATURE] Flush and query RF1 blocks for TraceQL metric queries [#3628](https://github.com/grafana/tempo/pull/3628) [#3691](https://github.com/grafana/tempo/pull/3691) [#3723](https://github.com/grafana/tempo/pull/3723) (@mapno) +* [FEATURE] Flush and query RF1 blocks for TraceQL metric queries [#3628](https://github.com/grafana/tempo/pull/3628) [#3691](https://github.com/grafana/tempo/pull/3691) [#3723](https://github.com/grafana/tempo/pull/3723) (@mapno) [#3995](https://github.com/grafana/tempo/pull/3995) (@mdisibio) * [FEATURE] Add new compare() metrics function [#3695](https://github.com/grafana/tempo/pull/3695) (@mdisibio) * [FEATURE] Add new api `/api/metrics/query` for instant metrics queries [#3859](https://github.com/grafana/tempo/pull/3859) (@mdisibio) * [FEATURE] Add a `q` parameter to `/api/v2/serach/tags` for tag name filtering [#3822](https://github.com/grafana/tempo/pull/3822) (@joe-elliott) diff --git a/docs/sources/tempo/configuration/_index.md b/docs/sources/tempo/configuration/_index.md index 4f5342a6d47..4f4712213ac 100644 --- a/docs/sources/tempo/configuration/_index.md +++ b/docs/sources/tempo/configuration/_index.md @@ -654,9 +654,6 @@ query_frontend: # If set to a non-zero value, it's value will be used to decide if query is within SLO or not. # Query is within SLO if it returned 200 within duration_slo seconds OR processed throughput_slo bytes/s data. [throughput_bytes_slo: | default = 0 ] - - # If set to true, TraceQL metric queries will use RF1 blocks built and flushed by the metrics-generator. - [rf1_read_path: | default = false] ``` ## Querier diff --git a/integration/e2e/config-query-range.yaml b/integration/e2e/config-query-range.yaml index 763a42f900d..d17f80a77cf 100644 --- a/integration/e2e/config-query-range.yaml +++ b/integration/e2e/config-query-range.yaml @@ -11,7 +11,6 @@ query_frontend: query_ingesters_until: 0 metrics: exemplars: true - rf1_read_path: true distributor: receivers: diff --git a/modules/frontend/combiner/metrics_query_range.go b/modules/frontend/combiner/metrics_query_range.go index 897b54bde41..8288123353d 100644 --- a/modules/frontend/combiner/metrics_query_range.go +++ b/modules/frontend/combiner/metrics_query_range.go @@ -31,22 +31,6 @@ func NewQueryRange(req *tempopb.QueryRangeRequest, trackDiffs bool) (Combiner, e } } - samplingRate := resp.RequestData() - if samplingRate != nil { - fRate := samplingRate.(float64) - - if fRate <= 1.0 { - // Set final sampling rate after integer rounding - // Multiply up the sampling rate - for _, series := range partial.Series { - for i, sample := range series.Samples { - sample.Value *= 1.0 / fRate - series.Samples[i] = sample - } - } - } - } - combiner.Combine(partial) return nil diff --git a/modules/frontend/config.go b/modules/frontend/config.go index 84e96a33b0d..d502da9a94a 100644 --- a/modules/frontend/config.go +++ b/modules/frontend/config.go @@ -90,7 +90,6 @@ func (cfg *Config) RegisterFlagsAndApplyDefaults(string, *flag.FlagSet) { ConcurrentRequests: defaultConcurrentRequests, TargetBytesPerRequest: defaultTargetBytesPerRequest, Interval: 5 * time.Minute, - RF1ReadPath: false, Exemplars: false, // TODO: Remove? MaxExemplars: 100, }, diff --git a/modules/frontend/metrics_query_range_handler_test.go b/modules/frontend/metrics_query_range_handler_test.go index 299ec41f35c..19c50dac201 100644 --- a/modules/frontend/metrics_query_range_handler_test.go +++ b/modules/frontend/metrics_query_range_handler_test.go @@ -47,6 +47,7 @@ func TestQueryRangeHandlerSucceeds(t *testing.T) { }, nil, nil, nil, func(c *Config) { c.Metrics.Sharder.Interval = time.Hour }) + tenant := "foo" httpReq := httptest.NewRequest("GET", api.PathMetricsQueryRange, nil) @@ -66,10 +67,9 @@ func TestQueryRangeHandlerSucceeds(t *testing.T) { require.Equal(t, 200, httpResp.Code) - // for reasons I don't understand, this query turns into 408 jobs. expectedResp := &tempopb.QueryRangeResponse{ Metrics: &tempopb.SearchMetrics{ - CompletedJobs: 4, + CompletedJobs: 4, // 2 blocks, each with 2 row groups that take 1 job InspectedTraces: 4, InspectedBytes: 4, TotalJobs: 4, @@ -101,90 +101,3 @@ func TestQueryRangeHandlerSucceeds(t *testing.T) { require.NoError(t, err) require.Equal(t, expectedResp, actualResp) } - -func TestQueryRangeHandlerRespectsSamplingRate(t *testing.T) { - resp := &tempopb.QueryRangeResponse{ - Metrics: &tempopb.SearchMetrics{ - InspectedTraces: 1, - InspectedBytes: 1, - }, - Series: []*tempopb.TimeSeries{ - { - PromLabels: "foo", - Labels: []v1.KeyValue{ - {Key: "foo", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "bar"}}}, - }, - Samples: []tempopb.Sample{ - { - TimestampMs: 1200_000, - Value: 2, - }, - { - TimestampMs: 1100_000, - Value: 1, - }, - }, - }, - }, - } - - f := frontendWithSettings(t, &mockRoundTripper{ - responseFn: func() proto.Message { - return resp - }, - }, nil, nil, nil, func(c *Config) { - c.Metrics.Sharder.Interval = time.Hour - }) - tenant := "foo" - - httpReq := httptest.NewRequest("GET", api.PathMetricsQueryRange, nil) - httpReq = api.BuildQueryRangeRequest(httpReq, &tempopb.QueryRangeRequest{ - Query: "{} | rate() with (sample=.2)", - Start: uint64(1100 * time.Second), - End: uint64(1200 * time.Second), - Step: uint64(100 * time.Second), - }) - - ctx := user.InjectOrgID(httpReq.Context(), tenant) - httpReq = httpReq.WithContext(ctx) - - httpResp := httptest.NewRecorder() - - f.MetricsQueryRangeHandler.ServeHTTP(httpResp, httpReq) - - require.Equal(t, 200, httpResp.Code) - - expectedResp := &tempopb.QueryRangeResponse{ - Metrics: &tempopb.SearchMetrics{ - CompletedJobs: 1, - InspectedTraces: 1, - InspectedBytes: 1, - TotalJobs: 1, - TotalBlocks: 2, - TotalBlockBytes: 419430400, - }, - Series: []*tempopb.TimeSeries{ - { - PromLabels: "foo", - Labels: []v1.KeyValue{ - {Key: "foo", Value: &v1.AnyValue{Value: &v1.AnyValue_StringValue{StringValue: "bar"}}}, - }, - Samples: []tempopb.Sample{ - { - TimestampMs: 1100_000, - Value: 5, - }, - { - TimestampMs: 1200_000, - Value: 10, - }, - }, - }, - }, - } - - actualResp := &tempopb.QueryRangeResponse{} - err := jsonpb.Unmarshal(httpResp.Body, actualResp) - require.NoError(t, err) - require.Equal(t, expectedResp, actualResp) -} diff --git a/modules/frontend/metrics_query_range_sharder.go b/modules/frontend/metrics_query_range_sharder.go index 99f6143e538..0357b480159 100644 --- a/modules/frontend/metrics_query_range_sharder.go +++ b/modules/frontend/metrics_query_range_sharder.go @@ -27,12 +27,11 @@ import ( ) type queryRangeSharder struct { - next pipeline.AsyncRoundTripper[combiner.PipelineResponse] - reader tempodb.Reader - overrides overrides.Interface - cfg QueryRangeSharderConfig - logger log.Logger - replicationFactor uint8 + next pipeline.AsyncRoundTripper[combiner.PipelineResponse] + reader tempodb.Reader + overrides overrides.Interface + cfg QueryRangeSharderConfig + logger log.Logger } type QueryRangeSharderConfig struct { @@ -41,17 +40,12 @@ type QueryRangeSharderConfig struct { MaxDuration time.Duration `yaml:"max_duration"` QueryBackendAfter time.Duration `yaml:"query_backend_after,omitempty"` Interval time.Duration `yaml:"interval,omitempty"` - RF1ReadPath bool `yaml:"rf1_read_path,omitempty"` Exemplars bool `yaml:"exemplars,omitempty"` MaxExemplars int `yaml:"max_exemplars,omitempty"` } // newAsyncQueryRangeSharder creates a sharding middleware for search func newAsyncQueryRangeSharder(reader tempodb.Reader, o overrides.Interface, cfg QueryRangeSharderConfig, logger log.Logger) pipeline.AsyncMiddleware[combiner.PipelineResponse] { - var replicationFactor uint8 - if cfg.RF1ReadPath { - replicationFactor = 1 - } return pipeline.AsyncMiddlewareFunc[combiner.PipelineResponse](func(next pipeline.AsyncRoundTripper[combiner.PipelineResponse]) pipeline.AsyncRoundTripper[combiner.PipelineResponse] { return queryRangeSharder{ next: next, @@ -60,8 +54,6 @@ func newAsyncQueryRangeSharder(reader tempodb.Reader, o overrides.Interface, cfg cfg: cfg, logger: logger, - - replicationFactor: replicationFactor, } }) } @@ -103,18 +95,10 @@ func (s queryRangeSharder) RoundTrip(pipelineRequest pipeline.Request) (pipeline var ( allowUnsafe = s.overrides.UnsafeQueryHints(tenantID) - samplingRate = s.samplingRate(expr, allowUnsafe) - targetBytesPerRequest = s.jobSize(expr, samplingRate, allowUnsafe) - interval = s.jobInterval(expr, allowUnsafe) + targetBytesPerRequest = s.jobSize(expr, allowUnsafe) cutoff = time.Now().Add(-s.cfg.QueryBackendAfter) ) - // if interval is 0 then the backend requests code will loop forever. technically if we are here with a 0 interval it should mean a bad request - // b/c it was specified using a query hint. we're going to assume that and return 400 - if interval == 0 { - return pipeline.NewBadRequest(errors.New("invalid interval specified: 0")), nil - } - generatorReq := s.generatorRequest(*req, r, tenantID, cutoff) reqCh := make(chan pipeline.Request, 2) // buffer of 2 allows us to insert generatorReq and metrics @@ -122,15 +106,7 @@ func (s queryRangeSharder) RoundTrip(pipelineRequest pipeline.Request) (pipeline reqCh <- pipeline.NewHTTPRequest(generatorReq) } - var ( - totalJobs, totalBlocks uint32 - totalBlockBytes uint64 - ) - if s.cfg.RF1ReadPath { - totalJobs, totalBlocks, totalBlockBytes = s.backendRequests(ctx, tenantID, r, *req, cutoff, samplingRate, targetBytesPerRequest, interval, reqCh) - } else { - totalJobs, totalBlocks, totalBlockBytes = s.shardedBackendRequests(ctx, tenantID, r, *req, cutoff, samplingRate, targetBytesPerRequest, interval, reqCh, nil) - } + totalJobs, totalBlocks, totalBlockBytes := s.backendRequests(ctx, tenantID, r, *req, cutoff, targetBytesPerRequest, reqCh) span.SetTag("totalJobs", totalJobs) span.SetTag("totalBlocks", totalBlocks) @@ -167,7 +143,7 @@ func (s *queryRangeSharder) blockMetas(start, end int64, tenantID string) []*bac for _, m := range allMetas { if m.StartTime.UnixNano() <= end && m.EndTime.UnixNano() >= start && - m.ReplicationFactor == s.replicationFactor { + m.ReplicationFactor == 1 { // We always only query RF1 blocks metas = append(metas, m) } } @@ -175,141 +151,6 @@ func (s *queryRangeSharder) blockMetas(start, end int64, tenantID string) []*bac return metas } -func (s *queryRangeSharder) shardedBackendRequests(ctx context.Context, tenantID string, parent *http.Request, searchReq tempopb.QueryRangeRequest, cutoff time.Time, samplingRate float64, targetBytesPerRequest int, interval time.Duration, reqCh chan pipeline.Request, _ func(error)) (totalJobs, totalBlocks uint32, totalBlockBytes uint64) { - // request without start or end, search only in generator - if searchReq.Start == 0 || searchReq.End == 0 { - close(reqCh) - return - } - - // Make a copy and limit to backend time range. - backendReq := searchReq - traceql.TrimToBefore(&backendReq, cutoff) - - // If empty window then no need to search backend - if backendReq.Start == backendReq.End { - close(reqCh) - return - } - - // Blocks within overall time range. This is just for instrumentation, more precise time - // range is checked for each window. - blocks := s.blockMetas(int64(backendReq.Start), int64(backendReq.End), tenantID) - if len(blocks) == 0 { - // no need to search backend - close(reqCh) - return - } - - // count blocks - totalBlocks = uint32(len(blocks)) - for _, b := range blocks { - totalBlockBytes += b.Size - } - - // count jobs. same loops as below - var ( - start = backendReq.Start - end = backendReq.End - timeWindowSize = uint64(interval.Nanoseconds()) - ) - - for start < end { - thisStart := start - thisEnd := start + timeWindowSize - if thisEnd > end { - thisEnd = end - } - - blocks := s.blockMetas(int64(thisStart), int64(thisEnd), tenantID) - if len(blocks) == 0 { - start = thisEnd - continue - } - - totalBlockSize := uint64(0) - for _, b := range blocks { - totalBlockSize += b.Size - } - - shards := uint32(math.Ceil(float64(totalBlockSize) / float64(targetBytesPerRequest))) - - for i := uint32(1); i <= shards; i++ { - totalJobs++ - } - - start = thisEnd - } - - go func() { - s.buildShardedBackendRequests(ctx, tenantID, parent, backendReq, samplingRate, targetBytesPerRequest, interval, reqCh) - }() - - return -} - -func (s *queryRangeSharder) buildShardedBackendRequests(ctx context.Context, tenantID string, parent *http.Request, searchReq tempopb.QueryRangeRequest, samplingRate float64, targetBytesPerRequest int, interval time.Duration, reqCh chan pipeline.Request) { - defer close(reqCh) - - var ( - start = searchReq.Start - end = searchReq.End - timeWindowSize = uint64(interval.Nanoseconds()) - ) - - for start < end { - thisStart := start - thisEnd := start + timeWindowSize - if thisEnd > end { - thisEnd = end - } - - blocks := s.blockMetas(int64(thisStart), int64(thisEnd), tenantID) - if len(blocks) == 0 { - start = thisEnd - continue - } - - totalBlockSize := uint64(0) - for _, b := range blocks { - totalBlockSize += b.Size - } - - shards := uint32(math.Ceil(float64(totalBlockSize) / float64(targetBytesPerRequest))) - exemplars := max(s.exemplarsPerShard(shards), 1) - - for i := uint32(1); i <= shards; i++ { - - shardR := searchReq - shardR.Start = thisStart - shardR.End = thisEnd - shardR.ShardID = i - shardR.ShardCount = shards - shardR.Exemplars = exemplars - httpReq := s.toUpstreamRequest(ctx, shardR, parent, tenantID) - - pipelineR := pipeline.NewHTTPRequest(httpReq) - if samplingRate != 1.0 { - shardR.ShardID *= uint32(1.0 / samplingRate) - shardR.ShardCount *= uint32(1.0 / samplingRate) - - // Set final sampling rate after integer rounding - samplingRate = float64(shards) / float64(shardR.ShardCount) - - pipelineR.SetResponseData(samplingRate) - } - - select { - case reqCh <- pipelineR: - case <-ctx.Done(): - return - } - } - - start = thisEnd - } -} - func (s *queryRangeSharder) exemplarsPerShard(total uint32) uint32 { if !s.cfg.Exemplars { return 0 @@ -317,7 +158,7 @@ func (s *queryRangeSharder) exemplarsPerShard(total uint32) uint32 { return uint32(math.Ceil(float64(s.cfg.MaxExemplars)*1.2)) / total } -func (s *queryRangeSharder) backendRequests(ctx context.Context, tenantID string, parent *http.Request, searchReq tempopb.QueryRangeRequest, cutoff time.Time, _ float64, targetBytesPerRequest int, _ time.Duration, reqCh chan pipeline.Request) (totalJobs, totalBlocks uint32, totalBlockBytes uint64) { +func (s *queryRangeSharder) backendRequests(ctx context.Context, tenantID string, parent *http.Request, searchReq tempopb.QueryRangeRequest, cutoff time.Time, targetBytesPerRequest int, reqCh chan pipeline.Request) (totalJobs, totalBlocks uint32, totalBlockBytes uint64) { // request without start or end, search only in generator if searchReq.Start == 0 || searchReq.End == 0 { close(reqCh) @@ -406,12 +247,10 @@ func (s *queryRangeSharder) buildBackendRequests(ctx context.Context, tenantID s } queryRangeReq := &tempopb.QueryRangeRequest{ - Query: searchReq.Query, - Start: start, - End: end, - Step: step, - // ShardID: uint32, // No sharding with RF=1 - // ShardCount: uint32, // No sharding with RF=1 + Query: searchReq.Query, + Start: start, + End: end, + Step: step, QueryMode: searchReq.QueryMode, // New RF1 fields BlockID: m.BlockID.String(), @@ -487,17 +326,7 @@ func (s *queryRangeSharder) maxDuration(tenantID string) time.Duration { return s.cfg.MaxDuration } -func (s *queryRangeSharder) samplingRate(expr *traceql.RootExpr, allowUnsafe bool) float64 { - samplingRate := 1.0 - if v, ok := expr.Hints.GetFloat(traceql.HintSample, allowUnsafe); ok { - if v > 0 && v < 1.0 { - samplingRate = v - } - } - return samplingRate -} - -func (s *queryRangeSharder) jobSize(expr *traceql.RootExpr, samplingRate float64, allowUnsafe bool) int { +func (s *queryRangeSharder) jobSize(expr *traceql.RootExpr, allowUnsafe bool) int { // If we have a query hint then use it if v, ok := expr.Hints.GetInt(traceql.HintJobSize, allowUnsafe); ok && v > 0 { return v @@ -506,32 +335,9 @@ func (s *queryRangeSharder) jobSize(expr *traceql.RootExpr, samplingRate float64 // Else use configured value. size := s.cfg.TargetBytesPerRequest - // Automatically scale job size when sampling less than 100% - // This improves performance. - if samplingRate < 1.0 { - factor := 1.0 / samplingRate - - // Keep it within reason - if factor > 10.0 { - factor = 10.0 - } - - size = int(float64(size) * factor) - } - return size } -func (s *queryRangeSharder) jobInterval(expr *traceql.RootExpr, allowUnsafe bool) time.Duration { - // If we have a query hint then use it - if v, ok := expr.Hints.GetDuration(traceql.HintJobInterval, allowUnsafe); ok && v > 0 { - return v - } - - // Else use configured value - return s.cfg.Interval -} - func hashForQueryRangeRequest(req *tempopb.QueryRangeRequest) uint64 { if req.Query == "" { return 0 diff --git a/modules/frontend/search_handlers_test.go b/modules/frontend/search_handlers_test.go index 3337e97f169..1039938ee5e 100644 --- a/modules/frontend/search_handlers_test.go +++ b/modules/frontend/search_handlers_test.go @@ -742,6 +742,23 @@ func frontendWithSettings(t require.TestingT, next http.RoundTripper, rdr tempod TotalRecords: 2, BlockID: uuid.MustParse("00000000-0000-0000-0000-000000000001"), }, + // These are RF1 metrics blocks + { + StartTime: time.Unix(1100, 0), + EndTime: time.Unix(1200, 0), + Size: defaultTargetBytesPerRequest * 2, + TotalRecords: 2, + BlockID: uuid.MustParse("00000000-0000-0000-0000-000000000002"), + ReplicationFactor: 1, + }, + { + StartTime: time.Unix(1100, 0), + EndTime: time.Unix(1200, 0), + Size: defaultTargetBytesPerRequest * 2, + TotalRecords: 2, + BlockID: uuid.MustParse("00000000-0000-0000-0000-000000000003"), + ReplicationFactor: 1, + }, }, } } diff --git a/modules/generator/processor/localblocks/query_range.go b/modules/generator/processor/localblocks/query_range.go index 8806a4f315e..f8b519ef1aa 100644 --- a/modules/generator/processor/localblocks/query_range.go +++ b/modules/generator/processor/localblocks/query_range.go @@ -55,7 +55,7 @@ func (p *Processor) QueryRange(ctx context.Context, req *tempopb.QueryRangeReque // Compile the raw version of the query for wal blocks // These aren't cached and we put them all into the same evaluator // for efficiency. - eval, err := e.CompileMetricsQueryRange(req, false, int(req.Exemplars), timeOverlapCutoff, unsafe) + eval, err := e.CompileMetricsQueryRange(req, int(req.Exemplars), timeOverlapCutoff, unsafe) if err != nil { return nil, err } @@ -192,7 +192,7 @@ func (p *Processor) queryRangeCompleteBlock(ctx context.Context, b *ingester.Loc } // Not in cache or not cacheable, so execute - eval, err := traceql.NewEngine().CompileMetricsQueryRange(&req, false, exemplars, timeOverlapCutoff, unsafe) + eval, err := traceql.NewEngine().CompileMetricsQueryRange(&req, exemplars, timeOverlapCutoff, unsafe) if err != nil { return nil, err } diff --git a/modules/querier/http.go b/modules/querier/http.go index cd40792019a..5831c777b57 100644 --- a/modules/querier/http.go +++ b/modules/querier/http.go @@ -394,8 +394,6 @@ func (q *Querier) QueryRangeHandler(w http.ResponseWriter, r *http.Request) { } span.SetTag("query", req.Query) - span.SetTag("shard", req.ShardID) - span.SetTag("shardCount", req.ShardCount) span.SetTag("step", time.Duration(req.Step)) span.SetTag("interval", time.Unix(0, int64(req.End)).Sub(time.Unix(0, int64(req.Start)))) diff --git a/modules/querier/querier_query_range.go b/modules/querier/querier_query_range.go index b2633abd580..cfc8df2ceba 100644 --- a/modules/querier/querier_query_range.go +++ b/modules/querier/querier_query_range.go @@ -9,15 +9,12 @@ import ( "github.com/google/uuid" "github.com/grafana/dskit/ring" "github.com/grafana/dskit/user" - "github.com/grafana/tempo/pkg/boundedwaitgroup" "github.com/grafana/tempo/pkg/tempopb" v1 "github.com/grafana/tempo/pkg/tempopb/common/v1" "github.com/grafana/tempo/pkg/traceql" "github.com/grafana/tempo/pkg/util/log" "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/encoding/common" - "github.com/opentracing/opentracing-go" - "github.com/uber-go/atomic" ) func (q *Querier) QueryRange(ctx context.Context, req *tempopb.QueryRangeRequest) (*tempopb.QueryRangeResponse, error) { @@ -25,19 +22,14 @@ func (q *Querier) QueryRange(ctx context.Context, req *tempopb.QueryRangeRequest return q.queryRangeRecent(ctx, req) } - if req.BlockID != "" { // RF1 search - return q.queryBlock(ctx, req) - } - - // Backend requests go here - return q.queryBackend(ctx, req) + return q.queryBlock(ctx, req) } func (q *Querier) queryRangeRecent(ctx context.Context, req *tempopb.QueryRangeRequest) (*tempopb.QueryRangeResponse, error) { // // Get results from all generators replicationSet, err := q.generatorRing.GetReplicationSetForOperation(ring.Read) if err != nil { - return nil, fmt.Errorf("error finding generators in Querier.SpanMetricsSummary: %w", err) + return nil, fmt.Errorf("error finding generators in Querier.queryRangeRecent: %w", err) } lookupResults, err := q.forGivenGenerators( ctx, @@ -47,9 +39,9 @@ func (q *Querier) queryRangeRecent(ctx context.Context, req *tempopb.QueryRangeR }, ) if err != nil { - _ = level.Error(log.Logger).Log("error querying generators in Querier.MetricsQueryRange", "err", err) + _ = level.Error(log.Logger).Log("error querying generators in Querier.queryRangeRecent", "err", err) - return nil, fmt.Errorf("error querying generators in Querier.MetricsQueryRange: %w", err) + return nil, fmt.Errorf("error querying generators in Querier.queryRangeRecent: %w", err) } c, err := traceql.QueryRangeCombinerFor(req, traceql.AggregateModeSum, false) @@ -67,7 +59,7 @@ func (q *Querier) queryRangeRecent(ctx context.Context, req *tempopb.QueryRangeR func (q *Querier) queryBlock(ctx context.Context, req *tempopb.QueryRangeRequest) (*tempopb.QueryRangeResponse, error) { tenantID, err := user.ExtractOrgID(ctx) if err != nil { - return nil, fmt.Errorf("error extracting org id in Querier.BackendSearch: %w", err) + return nil, fmt.Errorf("error extracting org id in Querier.queryBlock: %w", err) } blockID, err := uuid.Parse(req.BlockID) @@ -116,7 +108,7 @@ func (q *Querier) queryBlock(ctx context.Context, req *tempopb.QueryRangeRequest timeOverlapCutoff = v } - eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, false, int(req.Exemplars), timeOverlapCutoff, unsafe) + eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, int(req.Exemplars), timeOverlapCutoff, unsafe) if err != nil { return nil, err } @@ -142,103 +134,6 @@ func (q *Querier) queryBlock(ctx context.Context, req *tempopb.QueryRangeRequest }, nil } -func (q *Querier) queryBackend(ctx context.Context, req *tempopb.QueryRangeRequest) (*tempopb.QueryRangeResponse, error) { - ctx, cancel := context.WithCancel(ctx) - defer cancel() - - tenantID, err := user.ExtractOrgID(ctx) - if err != nil { - return nil, err - } - - // Get blocks that overlap this time range - metas := q.store.BlockMetas(tenantID) - withinTimeRange := metas[:0] - for _, m := range metas { - if m.StartTime.UnixNano() <= int64(req.End) && m.EndTime.UnixNano() > int64(req.Start) { - withinTimeRange = append(withinTimeRange, m) - } - } - - if len(withinTimeRange) == 0 { - return nil, nil - } - - unsafe := q.limits.UnsafeQueryHints(tenantID) - - // Optimization - // If there's only 1 block then dedupe not needed. - dedupe := len(withinTimeRange) > 1 - - expr, err := traceql.Parse(req.Query) - if err != nil { - return nil, err - } - - timeOverlapCutoff := q.cfg.Metrics.TimeOverlapCutoff - if v, ok := expr.Hints.GetFloat(traceql.HintTimeOverlapCutoff, unsafe); ok && v >= 0 && v <= 1.0 { - timeOverlapCutoff = v - } - - concurrency := q.cfg.Metrics.ConcurrentBlocks - if v, ok := expr.Hints.GetInt(traceql.HintConcurrentBlocks, unsafe); ok && v > 0 && v < 100 { - concurrency = v - } - - eval, err := traceql.NewEngine().CompileMetricsQueryRange(req, dedupe, int(req.Exemplars), timeOverlapCutoff, unsafe) - if err != nil { - return nil, err - } - - wg := boundedwaitgroup.New(uint(concurrency)) - jobErr := atomic.Error{} - - for _, m := range withinTimeRange { - // If a job errored then quit immediately. - if err := jobErr.Load(); err != nil { - return nil, err - } - - wg.Add(1) - go func(m *backend.BlockMeta) { - defer wg.Done() - - span, ctx := opentracing.StartSpanFromContext(ctx, "querier.queryBackEnd.Block", opentracing.Tags{ - "block": m.BlockID.String(), - "blockSize": m.Size, - }) - defer span.Finish() - - f := traceql.NewSpansetFetcherWrapper(func(ctx context.Context, req traceql.FetchSpansRequest) (traceql.FetchSpansResponse, error) { - return q.store.Fetch(ctx, m, req, common.DefaultSearchOptions()) - }) - - // TODO handle error - err := eval.Do(ctx, f, uint64(m.StartTime.UnixNano()), uint64(m.EndTime.UnixNano())) - if err != nil { - jobErr.Store(err) - } - }(m) - } - - wg.Wait() - if err := jobErr.Load(); err != nil { - return nil, err - } - - res := eval.Results() - - inspectedBytes, spansTotal, _ := eval.Metrics() - - return &tempopb.QueryRangeResponse{ - Series: queryRangeTraceQLToProto(res, req), - Metrics: &tempopb.SearchMetrics{ - InspectedBytes: inspectedBytes, - InspectedSpans: spansTotal, - }, - }, nil -} - func queryRangeTraceQLToProto(set traceql.SeriesSet, req *tempopb.QueryRangeRequest) []*tempopb.TimeSeries { resp := make([]*tempopb.TimeSeries, 0, len(set)) diff --git a/pkg/api/http.go b/pkg/api/http.go index 83029129145..11cd7676e6a 100644 --- a/pkg/api/http.go +++ b/pkg/api/http.go @@ -37,8 +37,6 @@ const ( urlParamEnd = "end" urlParamSpansPerSpanSet = "spss" urlParamStep = "step" - urlParamShard = "shard" - urlParamShardCount = "shardCount" urlParamSince = "since" urlParamExemplars = "exemplars" @@ -382,15 +380,6 @@ func ParseQueryRangeRequest(r *http.Request) (*tempopb.QueryRangeRequest, error) } req.Step = uint64(step.Nanoseconds()) - shardCount, _ := extractQueryParam(vals, urlParamShardCount) - if shardCount, err := strconv.Atoi(shardCount); err == nil { - req.ShardCount = uint32(shardCount) - } - shard, _ := extractQueryParam(vals, urlParamShard) - if shard, err := strconv.Atoi(shard); err == nil { - req.ShardID = uint32(shard) - } - // New RF1 params blockID, _ := extractQueryParam(vals, urlParamBlockID) if blockID, err := uuid.Parse(blockID); err == nil { @@ -475,8 +464,6 @@ func BuildQueryRangeRequest(req *http.Request, searchReq *tempopb.QueryRangeRequ qb.addParam(urlParamStart, strconv.FormatUint(searchReq.Start, 10)) qb.addParam(urlParamEnd, strconv.FormatUint(searchReq.End, 10)) qb.addParam(urlParamStep, time.Duration(searchReq.Step).String()) - qb.addParam(urlParamShard, strconv.FormatUint(uint64(searchReq.ShardID), 10)) - qb.addParam(urlParamShardCount, strconv.FormatUint(uint64(searchReq.ShardCount), 10)) qb.addParam(QueryModeKey, searchReq.QueryMode) // New RF1 params qb.addParam(urlParamBlockID, searchReq.BlockID) diff --git a/pkg/api/http_test.go b/pkg/api/http_test.go index c3387192b01..f4d57db01b5 100644 --- a/pkg/api/http_test.go +++ b/pkg/api/http_test.go @@ -710,13 +710,11 @@ func TestQueryRangeRoundtrip(t *testing.T) { { name: "not empty!", req: &tempopb.QueryRangeRequest{ - Query: "{ foo = `bar` }", - Start: uint64(24 * time.Hour), - End: uint64(25 * time.Hour), - Step: uint64(30 * time.Second), - ShardID: 1, - ShardCount: 2, - QueryMode: "foo", + Query: "{ foo = `bar` }", + Start: uint64(24 * time.Hour), + End: uint64(25 * time.Hour), + Step: uint64(30 * time.Second), + QueryMode: "foo", }, }, } diff --git a/pkg/tempopb/tempo.pb.go b/pkg/tempopb/tempo.pb.go index f885a29b86e..dc1696fa5e0 100644 --- a/pkg/tempopb/tempo.pb.go +++ b/pkg/tempopb/tempo.pb.go @@ -2925,13 +2925,13 @@ func (m *InstantSeries) GetPromLabels() string { } type QueryRangeRequest struct { - Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` - Start uint64 `protobuf:"varint,2,opt,name=start,proto3" json:"start,omitempty"` - End uint64 `protobuf:"varint,3,opt,name=end,proto3" json:"end,omitempty"` - Step uint64 `protobuf:"varint,4,opt,name=step,proto3" json:"step,omitempty"` - ShardID uint32 `protobuf:"varint,5,opt,name=shardID,proto3" json:"shardID,omitempty"` - ShardCount uint32 `protobuf:"varint,6,opt,name=shardCount,proto3" json:"shardCount,omitempty"` - QueryMode string `protobuf:"bytes,7,opt,name=queryMode,proto3" json:"queryMode,omitempty"` + Query string `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` + Start uint64 `protobuf:"varint,2,opt,name=start,proto3" json:"start,omitempty"` + End uint64 `protobuf:"varint,3,opt,name=end,proto3" json:"end,omitempty"` + Step uint64 `protobuf:"varint,4,opt,name=step,proto3" json:"step,omitempty"` + //uint32 shardID = 5; // removed + //uint32 shardCount = 6; // removed + QueryMode string `protobuf:"bytes,7,opt,name=queryMode,proto3" json:"queryMode,omitempty"` // New RF1 fields BlockID string `protobuf:"bytes,8,opt,name=blockID,proto3" json:"blockID,omitempty"` StartPage uint32 `protobuf:"varint,9,opt,name=startPage,proto3" json:"startPage,omitempty"` @@ -3006,20 +3006,6 @@ func (m *QueryRangeRequest) GetStep() uint64 { return 0 } -func (m *QueryRangeRequest) GetShardID() uint32 { - if m != nil { - return m.ShardID - } - return 0 -} - -func (m *QueryRangeRequest) GetShardCount() uint32 { - if m != nil { - return m.ShardCount - } - return 0 -} - func (m *QueryRangeRequest) GetQueryMode() string { if m != nil { return m.QueryMode @@ -3389,104 +3375,104 @@ func init() { func init() { proto.RegisterFile("pkg/tempopb/tempo.proto", fileDescriptor_f22805646f4f62b6) } var fileDescriptor_f22805646f4f62b6 = []byte{ - // 2756 bytes of a gzipped FileDescriptorProto + // 2735 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3a, 0xcd, 0x6f, 0x1b, 0xc7, 0xf5, 0x5a, 0xf1, 0x43, 0xe4, 0x23, 0x29, 0x51, 0x63, 0x45, 0xa1, 0x69, 0x47, 0xd6, 0x6f, 0x63, 0xe4, 0xa7, 0xe6, 0x83, 0x92, 0x19, 0x1b, 0x89, 0x93, 0x36, 0x85, 0x65, 0xb1, 0x8e, 0x12, 0x49, 0x56, 0x86, 0x8a, 0x12, 0x14, 0x01, 0x84, 0x15, 0x39, 0xa6, 0x17, 0x22, 0x77, 0x99, 0xdd, 0xa1, 0x6a, 0xf6, 0x50, 0xa0, 0x05, 0x5a, 0xa0, 0x40, 0x0f, 0x3d, 0xb4, 0x87, 0x1e, 0x7b, 0x2a, 0x7a, - 0xee, 0x9f, 0x50, 0xa0, 0x08, 0x50, 0x34, 0x08, 0xd0, 0x4b, 0xd0, 0x43, 0x50, 0x24, 0x87, 0xa2, - 0xff, 0x45, 0x31, 0x6f, 0x66, 0x76, 0x67, 0xc9, 0x95, 0x14, 0x37, 0x0e, 0x9a, 0x43, 0x4e, 0x9a, - 0xf7, 0xe6, 0xed, 0x9b, 0x37, 0xef, 0xfb, 0x0d, 0x05, 0x4f, 0x0f, 0x4f, 0x7a, 0xeb, 0x9c, 0x0d, - 0x86, 0xfe, 0xf0, 0x58, 0xfe, 0x6d, 0x0c, 0x03, 0x9f, 0xfb, 0x64, 0x4e, 0x21, 0xeb, 0xcb, 0x1d, - 0x7f, 0x30, 0xf0, 0xbd, 0xf5, 0xd3, 0x1b, 0xeb, 0x72, 0x25, 0x09, 0xea, 0x2f, 0xf5, 0x5c, 0xfe, - 0x70, 0x74, 0xdc, 0xe8, 0xf8, 0x83, 0xf5, 0x9e, 0xdf, 0xf3, 0xd7, 0x11, 0x7d, 0x3c, 0x7a, 0x80, - 0x10, 0x02, 0xb8, 0x52, 0xe4, 0x4b, 0x3c, 0x70, 0x3a, 0x4c, 0x70, 0xc1, 0x85, 0xc4, 0xda, 0xbf, - 0xb0, 0xa0, 0x7a, 0x20, 0xe0, 0xcd, 0xf1, 0xf6, 0x16, 0x65, 0x1f, 0x8e, 0x58, 0xc8, 0x49, 0x0d, - 0xe6, 0x90, 0x66, 0x7b, 0xab, 0x66, 0xad, 0x5a, 0x6b, 0x65, 0xaa, 0x41, 0xb2, 0x02, 0x70, 0xdc, - 0xf7, 0x3b, 0x27, 0x6d, 0xee, 0x04, 0xbc, 0x36, 0xbb, 0x6a, 0xad, 0x15, 0xa9, 0x81, 0x21, 0x75, - 0x28, 0x20, 0xd4, 0xf2, 0xba, 0xb5, 0x0c, 0xee, 0x46, 0x30, 0xb9, 0x0a, 0xc5, 0x0f, 0x47, 0x2c, - 0x18, 0xef, 0xfa, 0x5d, 0x56, 0xcb, 0xe1, 0x66, 0x8c, 0xb0, 0x3d, 0x58, 0x34, 0xe4, 0x08, 0x87, - 0xbe, 0x17, 0x32, 0x72, 0x1d, 0x72, 0x78, 0x32, 0x8a, 0x51, 0x6a, 0xce, 0x37, 0x94, 0x4e, 0x1a, - 0x48, 0x4a, 0xe5, 0x26, 0x79, 0x19, 0xe6, 0x06, 0x8c, 0x07, 0x6e, 0x27, 0x44, 0x89, 0x4a, 0xcd, - 0xcb, 0x49, 0x3a, 0xc1, 0x72, 0x57, 0x12, 0x50, 0x4d, 0x69, 0x13, 0xe3, 0xde, 0x6a, 0xd3, 0xfe, - 0x78, 0x16, 0x2a, 0x6d, 0xe6, 0x04, 0x9d, 0x87, 0x5a, 0x13, 0xaf, 0x41, 0xf6, 0xc0, 0xe9, 0x85, - 0x35, 0x6b, 0x35, 0xb3, 0x56, 0x6a, 0xae, 0x46, 0x7c, 0x13, 0x54, 0x0d, 0x41, 0xd2, 0xf2, 0x78, - 0x30, 0xde, 0xcc, 0x7e, 0xf4, 0xd9, 0xb5, 0x19, 0x8a, 0xdf, 0x90, 0xeb, 0x50, 0xd9, 0x75, 0xbd, - 0xad, 0x51, 0xe0, 0x70, 0xd7, 0xf7, 0x76, 0xa5, 0x70, 0x15, 0x9a, 0x44, 0x22, 0x95, 0xf3, 0xc8, - 0xa0, 0xca, 0x28, 0x2a, 0x13, 0x49, 0x96, 0x20, 0xb7, 0xe3, 0x0e, 0x5c, 0x5e, 0xcb, 0xe2, 0xae, - 0x04, 0x04, 0x36, 0x44, 0x43, 0xe4, 0x24, 0x16, 0x01, 0x52, 0x85, 0x0c, 0xf3, 0xba, 0xb5, 0x3c, - 0xe2, 0xc4, 0x52, 0xd0, 0xbd, 0x23, 0x14, 0x5d, 0x2b, 0xa0, 0xd6, 0x25, 0x40, 0xd6, 0x60, 0xa1, - 0x3d, 0x74, 0xbc, 0x70, 0x9f, 0x05, 0xe2, 0x6f, 0x9b, 0xf1, 0x5a, 0x11, 0xbf, 0x99, 0x44, 0xd7, - 0x5f, 0x81, 0x62, 0x74, 0x45, 0xc1, 0xfe, 0x84, 0x8d, 0xd1, 0x22, 0x45, 0x2a, 0x96, 0x82, 0xfd, - 0xa9, 0xd3, 0x1f, 0x31, 0xe5, 0x0f, 0x12, 0x78, 0x6d, 0xf6, 0x55, 0xcb, 0xfe, 0x4b, 0x06, 0x88, - 0x54, 0xd5, 0xa6, 0xf0, 0x02, 0xad, 0xd5, 0x9b, 0x50, 0x0c, 0xb5, 0x02, 0x95, 0x69, 0x97, 0xd3, - 0x55, 0x4b, 0x63, 0x42, 0xe1, 0x95, 0xe8, 0x4b, 0xdb, 0x5b, 0xea, 0x20, 0x0d, 0x0a, 0xcf, 0xc2, - 0xab, 0xef, 0x3b, 0x3d, 0xa6, 0xf4, 0x17, 0x23, 0x84, 0x86, 0x87, 0x4e, 0x8f, 0x85, 0x07, 0xbe, - 0x64, 0xad, 0x74, 0x98, 0x44, 0x0a, 0xcf, 0x65, 0x5e, 0xc7, 0xef, 0xba, 0x5e, 0x4f, 0x39, 0x67, - 0x04, 0x0b, 0x0e, 0xae, 0xd7, 0x65, 0x8f, 0x04, 0xbb, 0xb6, 0xfb, 0x63, 0xa6, 0x74, 0x9b, 0x44, - 0x12, 0x1b, 0xca, 0xdc, 0xe7, 0x4e, 0x9f, 0xb2, 0x8e, 0x1f, 0x74, 0xc3, 0xda, 0x1c, 0x12, 0x25, - 0x70, 0x82, 0xa6, 0xeb, 0x70, 0xa7, 0xa5, 0x4f, 0x92, 0x06, 0x49, 0xe0, 0xc4, 0x3d, 0x4f, 0x59, - 0x10, 0xba, 0xbe, 0x87, 0xf6, 0x28, 0x52, 0x0d, 0x12, 0x02, 0xd9, 0x50, 0x1c, 0x0f, 0xab, 0xd6, - 0x5a, 0x96, 0xe2, 0x5a, 0x44, 0xe4, 0x03, 0xdf, 0xe7, 0x2c, 0x40, 0xc1, 0x4a, 0x78, 0xa6, 0x81, - 0x21, 0x5b, 0x50, 0xed, 0xb2, 0xae, 0xdb, 0x71, 0x38, 0xeb, 0xde, 0xf5, 0xfb, 0xa3, 0x81, 0x17, - 0xd6, 0xca, 0xe8, 0xcd, 0xb5, 0x48, 0xe5, 0x5b, 0x49, 0x02, 0x3a, 0xf5, 0x85, 0xfd, 0x67, 0x0b, - 0x16, 0x26, 0xa8, 0xc8, 0x4d, 0xc8, 0x85, 0x1d, 0x7f, 0x28, 0x35, 0x3e, 0xdf, 0x5c, 0x39, 0x8b, - 0x5d, 0xa3, 0x2d, 0xa8, 0xa8, 0x24, 0x16, 0x77, 0xf0, 0x9c, 0x81, 0xf6, 0x15, 0x5c, 0x93, 0x1b, - 0x90, 0xe5, 0xe3, 0xa1, 0x8c, 0xf2, 0xf9, 0xe6, 0x33, 0x67, 0x32, 0x3a, 0x18, 0x0f, 0x19, 0x45, - 0x52, 0xfb, 0x1a, 0xe4, 0x90, 0x2d, 0x29, 0x40, 0xb6, 0xbd, 0x7f, 0x67, 0xaf, 0x3a, 0x43, 0xca, - 0x50, 0xa0, 0xad, 0xf6, 0xfd, 0x77, 0xe9, 0xdd, 0x56, 0xd5, 0xb2, 0x09, 0x64, 0x05, 0x39, 0x01, - 0xc8, 0xb7, 0x0f, 0xe8, 0xf6, 0xde, 0xbd, 0xea, 0x8c, 0xfd, 0x08, 0xe6, 0xb5, 0x77, 0xa9, 0x04, - 0x73, 0x13, 0xf2, 0x98, 0x43, 0x74, 0x84, 0x5f, 0x4d, 0x66, 0x0e, 0x49, 0xbd, 0xcb, 0xb8, 0x23, - 0x2c, 0x44, 0x15, 0x2d, 0xd9, 0x98, 0x4c, 0x38, 0x93, 0xde, 0x3b, 0x95, 0x6d, 0xfe, 0x9e, 0x81, - 0x4b, 0x29, 0x1c, 0x27, 0x33, 0x6d, 0x31, 0xce, 0xb4, 0x6b, 0xb0, 0x10, 0xf8, 0x3e, 0x6f, 0xb3, - 0xe0, 0xd4, 0xed, 0xb0, 0xbd, 0x58, 0x65, 0x93, 0x68, 0xe1, 0x9d, 0x02, 0x85, 0xec, 0x91, 0x4e, - 0x26, 0xde, 0x24, 0x92, 0xbc, 0x08, 0x8b, 0x18, 0x12, 0x07, 0xee, 0x80, 0xbd, 0xeb, 0xb9, 0x8f, - 0xf6, 0x1c, 0xcf, 0xc7, 0x48, 0xc8, 0xd2, 0xe9, 0x0d, 0xe1, 0x55, 0xdd, 0x38, 0x25, 0xc9, 0xf4, - 0x62, 0x60, 0xc8, 0xf3, 0x30, 0x17, 0xaa, 0x9c, 0x91, 0x47, 0x0d, 0x54, 0x63, 0x0d, 0x48, 0x3c, - 0xd5, 0x04, 0xe4, 0x45, 0x28, 0xa8, 0xa5, 0x88, 0x89, 0x4c, 0x2a, 0x71, 0x44, 0x41, 0x28, 0x94, - 0x43, 0x79, 0xb9, 0x36, 0x77, 0x78, 0x58, 0x2b, 0xe0, 0x17, 0x8d, 0xf3, 0xec, 0xd2, 0x68, 0x1b, - 0x1f, 0x60, 0x92, 0xa2, 0x09, 0x1e, 0xf5, 0x43, 0x58, 0x9c, 0x22, 0x49, 0xc9, 0x63, 0x2f, 0x98, - 0x79, 0xac, 0xd4, 0x7c, 0xca, 0x30, 0x6a, 0xfc, 0xb1, 0x99, 0xde, 0x76, 0xa0, 0x6c, 0x6e, 0x61, - 0x1e, 0x1a, 0x3a, 0xde, 0x5d, 0x7f, 0xe4, 0x71, 0x64, 0x2c, 0xf2, 0x90, 0x46, 0x08, 0x9d, 0xb2, - 0x20, 0xf0, 0x03, 0xb9, 0x2d, 0x8b, 0x81, 0x81, 0xb1, 0x7f, 0x6e, 0xc1, 0x9c, 0xd2, 0x07, 0x79, - 0x16, 0x72, 0xe2, 0x43, 0xed, 0x96, 0x95, 0x84, 0xc2, 0xa8, 0xdc, 0x13, 0xce, 0x33, 0x70, 0x78, - 0xe7, 0x21, 0xeb, 0x2a, 0x6e, 0x1a, 0x24, 0xaf, 0x03, 0x38, 0x9c, 0x07, 0xee, 0xf1, 0x88, 0x33, - 0x51, 0x51, 0x04, 0x8f, 0x2b, 0x11, 0x0f, 0xd5, 0x45, 0x9c, 0xde, 0x68, 0xbc, 0xcd, 0xc6, 0x87, - 0xe2, 0x36, 0xd4, 0x20, 0x17, 0xb1, 0x9e, 0x15, 0xc7, 0x90, 0x65, 0xc8, 0x8b, 0x83, 0x22, 0xdf, - 0x54, 0x50, 0x6a, 0x08, 0xa7, 0xba, 0x57, 0xe6, 0x2c, 0xf7, 0xba, 0x0e, 0x15, 0xed, 0x4c, 0x02, - 0x0e, 0x95, 0x23, 0x26, 0x91, 0x13, 0xb7, 0xc8, 0x3d, 0xde, 0x2d, 0x7e, 0x17, 0xd5, 0x72, 0x15, - 0x8c, 0x22, 0xa2, 0x5c, 0x2f, 0x1c, 0xb2, 0x0e, 0x67, 0xdd, 0x03, 0x1d, 0xf4, 0x58, 0xef, 0x26, - 0xd0, 0xe4, 0x39, 0x98, 0x8f, 0x50, 0x9b, 0x63, 0x71, 0xf8, 0x2c, 0xca, 0x37, 0x81, 0x25, 0xab, - 0x50, 0xc2, 0xec, 0x8e, 0xc5, 0x4d, 0x57, 0x6e, 0x13, 0x25, 0x2e, 0xda, 0xf1, 0x07, 0xc3, 0x3e, - 0xe3, 0xac, 0xfb, 0x96, 0x7f, 0x1c, 0xea, 0xda, 0x93, 0x40, 0x0a, 0xbf, 0xc1, 0x8f, 0x90, 0x42, - 0x06, 0x5b, 0x8c, 0x10, 0x72, 0xc7, 0x2c, 0xa5, 0x38, 0x79, 0x14, 0x67, 0x12, 0x9d, 0x90, 0x1b, - 0x6b, 0x38, 0xd6, 0x20, 0x53, 0x6e, 0xc4, 0xda, 0x3d, 0x11, 0x0f, 0x42, 0x35, 0xa2, 0xaa, 0xeb, - 0xa2, 0xbc, 0xa4, 0xd3, 0xb9, 0x34, 0xb6, 0x4a, 0xd7, 0x4b, 0x90, 0xc3, 0x1e, 0x4d, 0xd7, 0x76, - 0x04, 0xe2, 0xc6, 0x23, 0x93, 0xd2, 0x78, 0x64, 0xa3, 0xc6, 0xc3, 0xfe, 0x38, 0x03, 0xcb, 0xf1, - 0x49, 0x89, 0x1e, 0xe0, 0xd5, 0xe9, 0x1e, 0xa0, 0x3e, 0x91, 0x45, 0x0d, 0xe9, 0xbe, 0xed, 0x03, - 0xbe, 0x19, 0x7d, 0xc0, 0xa7, 0x19, 0xb8, 0x12, 0x19, 0x07, 0x83, 0x2e, 0x69, 0xd5, 0xef, 0x4d, - 0x5b, 0xf5, 0xda, 0xb4, 0x55, 0xe5, 0x87, 0xdf, 0x9a, 0xf6, 0x1b, 0x65, 0xda, 0x0d, 0xdd, 0xaa, - 0xcb, 0xb0, 0x53, 0x0d, 0x52, 0x1d, 0x0a, 0xdc, 0xe9, 0x89, 0x0e, 0x42, 0xd6, 0xa2, 0x22, 0x8d, - 0x60, 0xfb, 0x2d, 0x58, 0x8a, 0xbf, 0x38, 0x6c, 0x46, 0xdf, 0x34, 0x21, 0x8f, 0xc9, 0x43, 0x57, - 0xaf, 0xb4, 0xb8, 0x3e, 0x6c, 0xca, 0xae, 0x50, 0x51, 0xda, 0xaf, 0x9b, 0x29, 0x49, 0x6d, 0x46, - 0x85, 0xc6, 0x32, 0x0a, 0x0d, 0x81, 0x2c, 0x17, 0x13, 0xd9, 0x2c, 0x0a, 0x83, 0x6b, 0x7b, 0x68, - 0x64, 0x99, 0x84, 0x6f, 0x61, 0x7f, 0x25, 0xc5, 0x8d, 0xfa, 0x2b, 0x09, 0x5e, 0x94, 0xd8, 0xb2, - 0x29, 0x89, 0x2d, 0x17, 0x27, 0xb6, 0x57, 0xe0, 0xe9, 0xa9, 0x13, 0xd5, 0xed, 0x45, 0x32, 0xd7, - 0x48, 0xa5, 0xb2, 0x18, 0x61, 0xdf, 0x84, 0x82, 0xfe, 0x04, 0xaf, 0x32, 0x8e, 0x12, 0x2e, 0xae, - 0xd3, 0x67, 0x29, 0x7b, 0x07, 0x2e, 0x4f, 0x1c, 0x67, 0xa8, 0x7b, 0x7d, 0xf2, 0xc0, 0x52, 0x73, - 0x31, 0x6e, 0x97, 0xd4, 0x8e, 0x29, 0xc3, 0x1e, 0xe4, 0xb0, 0xd0, 0x91, 0x16, 0x54, 0x02, 0x16, - 0xfa, 0xa3, 0xa0, 0xc3, 0xda, 0x46, 0xb7, 0x11, 0x47, 0xac, 0x7c, 0x29, 0x38, 0xbd, 0xd1, 0xa0, - 0x26, 0x19, 0x4d, 0x7e, 0x65, 0xef, 0x41, 0x79, 0x7f, 0x14, 0xc6, 0x4d, 0xf5, 0x1b, 0x50, 0xc1, - 0xb6, 0x26, 0xdc, 0x1c, 0x1f, 0xa8, 0xe9, 0x3d, 0xb3, 0x36, 0x6f, 0x38, 0xa3, 0xa0, 0x6e, 0x09, - 0x0a, 0xca, 0x9c, 0xd0, 0xf7, 0x68, 0x92, 0xdc, 0xfe, 0xbd, 0x05, 0x55, 0x41, 0x82, 0x45, 0x4d, - 0x5b, 0xf2, 0xa5, 0xa8, 0x53, 0x17, 0x96, 0x2f, 0x6f, 0x3e, 0x25, 0x26, 0xed, 0x7f, 0x7c, 0x76, - 0xad, 0xb2, 0x1f, 0x30, 0xa7, 0xdf, 0xf7, 0x3b, 0x92, 0x5a, 0xb7, 0xe8, 0xff, 0x0f, 0x19, 0xb7, - 0x2b, 0x5b, 0x9f, 0x33, 0x69, 0x05, 0x05, 0xb9, 0x05, 0x20, 0xf3, 0xcf, 0x96, 0xc3, 0x9d, 0x5a, - 0xf6, 0x3c, 0x7a, 0x83, 0xd0, 0xde, 0x95, 0x22, 0x4a, 0x7d, 0x28, 0x11, 0x6f, 0xc3, 0xdc, 0x31, - 0x36, 0x60, 0x5f, 0x5a, 0x91, 0x9a, 0xde, 0xbe, 0x0e, 0xa0, 0x5e, 0x23, 0x44, 0x1d, 0x5f, 0x4e, - 0x4c, 0x25, 0x65, 0x7d, 0x29, 0xfb, 0x0d, 0x28, 0xee, 0xb8, 0xde, 0x49, 0xbb, 0xef, 0x76, 0xc4, - 0xd0, 0x94, 0xeb, 0xbb, 0xde, 0x89, 0x3e, 0xeb, 0xca, 0xf4, 0x59, 0xe2, 0x8c, 0x86, 0xf8, 0x80, + 0xea, 0xa1, 0x7f, 0x42, 0x81, 0x22, 0x40, 0xd1, 0x20, 0x40, 0x2f, 0x41, 0x0f, 0x41, 0x91, 0x1c, + 0xfa, 0x6f, 0x14, 0xf3, 0x66, 0x66, 0x77, 0x96, 0x5c, 0x49, 0x71, 0xe3, 0xa0, 0x39, 0xe4, 0xa4, + 0x7d, 0x6f, 0xde, 0xbc, 0x79, 0xf3, 0xbe, 0xdf, 0x50, 0xf0, 0xf4, 0xf0, 0xa4, 0xb7, 0xce, 0xd9, + 0x60, 0xe8, 0x0f, 0x8f, 0xe5, 0xdf, 0xc6, 0x30, 0xf0, 0xb9, 0x4f, 0xe6, 0x14, 0xb2, 0xbe, 0xdc, + 0xf1, 0x07, 0x03, 0xdf, 0x5b, 0x3f, 0xbd, 0xb1, 0x2e, 0xbf, 0x24, 0x41, 0xfd, 0xa5, 0x9e, 0xcb, + 0x1f, 0x8e, 0x8e, 0x1b, 0x1d, 0x7f, 0xb0, 0xde, 0xf3, 0x7b, 0xfe, 0x3a, 0xa2, 0x8f, 0x47, 0x0f, + 0x10, 0x42, 0x00, 0xbf, 0x14, 0xf9, 0x12, 0x0f, 0x9c, 0x0e, 0x13, 0x5c, 0xf0, 0x43, 0x62, 0xed, + 0x5f, 0x58, 0x50, 0x3d, 0x10, 0xf0, 0xe6, 0x78, 0x7b, 0x8b, 0xb2, 0x0f, 0x47, 0x2c, 0xe4, 0xa4, + 0x06, 0x73, 0x48, 0xb3, 0xbd, 0x55, 0xb3, 0x56, 0xad, 0xb5, 0x32, 0xd5, 0x20, 0x59, 0x01, 0x38, + 0xee, 0xfb, 0x9d, 0x93, 0x36, 0x77, 0x02, 0x5e, 0x9b, 0x5d, 0xb5, 0xd6, 0x8a, 0xd4, 0xc0, 0x90, + 0x3a, 0x14, 0x10, 0x6a, 0x79, 0xdd, 0x5a, 0x06, 0x57, 0x23, 0x98, 0x5c, 0x85, 0xe2, 0x87, 0x23, + 0x16, 0x8c, 0x77, 0xfd, 0x2e, 0xab, 0xe5, 0x70, 0x31, 0x46, 0xd8, 0x1e, 0x2c, 0x1a, 0x72, 0x84, + 0x43, 0xdf, 0x0b, 0x19, 0xb9, 0x0e, 0x39, 0x3c, 0x19, 0xc5, 0x28, 0x35, 0xe7, 0x1b, 0x4a, 0x27, + 0x0d, 0x24, 0xa5, 0x72, 0x91, 0xbc, 0x0c, 0x73, 0x03, 0xc6, 0x03, 0xb7, 0x13, 0xa2, 0x44, 0xa5, + 0xe6, 0xe5, 0x24, 0x9d, 0x60, 0xb9, 0x2b, 0x09, 0xa8, 0xa6, 0xb4, 0x89, 0x71, 0x6f, 0xb5, 0x68, + 0x7f, 0x3c, 0x0b, 0x95, 0x36, 0x73, 0x82, 0xce, 0x43, 0xad, 0x89, 0xd7, 0x20, 0x7b, 0xe0, 0xf4, + 0xc2, 0x9a, 0xb5, 0x9a, 0x59, 0x2b, 0x35, 0x57, 0x23, 0xbe, 0x09, 0xaa, 0x86, 0x20, 0x69, 0x79, + 0x3c, 0x18, 0x6f, 0x66, 0x3f, 0xfa, 0xec, 0xda, 0x0c, 0xc5, 0x3d, 0xe4, 0x3a, 0x54, 0x76, 0x5d, + 0x6f, 0x6b, 0x14, 0x38, 0xdc, 0xf5, 0xbd, 0x5d, 0x29, 0x5c, 0x85, 0x26, 0x91, 0x48, 0xe5, 0x3c, + 0x32, 0xa8, 0x32, 0x8a, 0xca, 0x44, 0x92, 0x25, 0xc8, 0xed, 0xb8, 0x03, 0x97, 0xd7, 0xb2, 0xb8, + 0x2a, 0x01, 0x81, 0x0d, 0xd1, 0x10, 0x39, 0x89, 0x45, 0x80, 0x54, 0x21, 0xc3, 0xbc, 0x6e, 0x2d, + 0x8f, 0x38, 0xf1, 0x29, 0xe8, 0xde, 0x11, 0x8a, 0xae, 0x15, 0x50, 0xeb, 0x12, 0x20, 0x6b, 0xb0, + 0xd0, 0x1e, 0x3a, 0x5e, 0xb8, 0xcf, 0x02, 0xf1, 0xb7, 0xcd, 0x78, 0xad, 0x88, 0x7b, 0x26, 0xd1, + 0xf5, 0x57, 0xa0, 0x18, 0x5d, 0x51, 0xb0, 0x3f, 0x61, 0x63, 0xb4, 0x48, 0x91, 0x8a, 0x4f, 0xc1, + 0xfe, 0xd4, 0xe9, 0x8f, 0x98, 0xf2, 0x07, 0x09, 0xbc, 0x36, 0xfb, 0xaa, 0x65, 0xff, 0x35, 0x03, + 0x44, 0xaa, 0x6a, 0x53, 0x78, 0x81, 0xd6, 0xea, 0x4d, 0x28, 0x86, 0x5a, 0x81, 0xca, 0xb4, 0xcb, + 0xe9, 0xaa, 0xa5, 0x31, 0xa1, 0xf0, 0x4a, 0xf4, 0xa5, 0xed, 0x2d, 0x75, 0x90, 0x06, 0x85, 0x67, + 0xe1, 0xd5, 0xf7, 0x9d, 0x1e, 0x53, 0xfa, 0x8b, 0x11, 0x42, 0xc3, 0x43, 0xa7, 0xc7, 0xc2, 0x03, + 0x5f, 0xb2, 0x56, 0x3a, 0x4c, 0x22, 0x85, 0xe7, 0x32, 0xaf, 0xe3, 0x77, 0x5d, 0xaf, 0xa7, 0x9c, + 0x33, 0x82, 0x05, 0x07, 0xd7, 0xeb, 0xb2, 0x47, 0x82, 0x5d, 0xdb, 0xfd, 0x31, 0x53, 0xba, 0x4d, + 0x22, 0x89, 0x0d, 0x65, 0xee, 0x73, 0xa7, 0x4f, 0x59, 0xc7, 0x0f, 0xba, 0x61, 0x6d, 0x0e, 0x89, + 0x12, 0x38, 0x41, 0xd3, 0x75, 0xb8, 0xd3, 0xd2, 0x27, 0x49, 0x83, 0x24, 0x70, 0xe2, 0x9e, 0xa7, + 0x2c, 0x08, 0x5d, 0xdf, 0x43, 0x7b, 0x14, 0xa9, 0x06, 0x09, 0x81, 0x6c, 0x28, 0x8e, 0x87, 0x55, + 0x6b, 0x2d, 0x4b, 0xf1, 0x5b, 0x44, 0xe4, 0x03, 0xdf, 0xe7, 0x2c, 0x40, 0xc1, 0x4a, 0x78, 0xa6, + 0x81, 0x21, 0x5b, 0x50, 0xed, 0xb2, 0xae, 0xdb, 0x71, 0x38, 0xeb, 0xde, 0xf5, 0xfb, 0xa3, 0x81, + 0x17, 0xd6, 0xca, 0xe8, 0xcd, 0xb5, 0x48, 0xe5, 0x5b, 0x49, 0x02, 0x3a, 0xb5, 0xc3, 0xfe, 0x8b, + 0x05, 0x0b, 0x13, 0x54, 0xe4, 0x26, 0xe4, 0xc2, 0x8e, 0x3f, 0x94, 0x1a, 0x9f, 0x6f, 0xae, 0x9c, + 0xc5, 0xae, 0xd1, 0x16, 0x54, 0x54, 0x12, 0x8b, 0x3b, 0x78, 0xce, 0x40, 0xfb, 0x0a, 0x7e, 0x93, + 0x1b, 0x90, 0xe5, 0xe3, 0xa1, 0x8c, 0xf2, 0xf9, 0xe6, 0x33, 0x67, 0x32, 0x3a, 0x18, 0x0f, 0x19, + 0x45, 0x52, 0xfb, 0x1a, 0xe4, 0x90, 0x2d, 0x29, 0x40, 0xb6, 0xbd, 0x7f, 0x67, 0xaf, 0x3a, 0x43, + 0xca, 0x50, 0xa0, 0xad, 0xf6, 0xfd, 0x77, 0xe9, 0xdd, 0x56, 0xd5, 0xb2, 0x09, 0x64, 0x05, 0x39, + 0x01, 0xc8, 0xb7, 0x0f, 0xe8, 0xf6, 0xde, 0xbd, 0xea, 0x8c, 0xfd, 0x08, 0xe6, 0xb5, 0x77, 0xa9, + 0x04, 0x73, 0x13, 0xf2, 0x98, 0x43, 0x74, 0x84, 0x5f, 0x4d, 0x66, 0x0e, 0x49, 0xbd, 0xcb, 0xb8, + 0x23, 0x2c, 0x44, 0x15, 0x2d, 0xd9, 0x98, 0x4c, 0x38, 0x93, 0xde, 0x3b, 0x95, 0x6d, 0xfe, 0x91, + 0x81, 0x4b, 0x29, 0x1c, 0x27, 0x33, 0x6d, 0x31, 0xce, 0xb4, 0x6b, 0xb0, 0x10, 0xf8, 0x3e, 0x6f, + 0xb3, 0xe0, 0xd4, 0xed, 0xb0, 0xbd, 0x58, 0x65, 0x93, 0x68, 0xe1, 0x9d, 0x02, 0x85, 0xec, 0x91, + 0x4e, 0x26, 0xde, 0x24, 0x92, 0xbc, 0x08, 0x8b, 0x18, 0x12, 0x07, 0xee, 0x80, 0xbd, 0xeb, 0xb9, + 0x8f, 0xf6, 0x1c, 0xcf, 0xc7, 0x48, 0xc8, 0xd2, 0xe9, 0x05, 0xe1, 0x55, 0xdd, 0x38, 0x25, 0xc9, + 0xf4, 0x62, 0x60, 0xc8, 0xf3, 0x30, 0x17, 0xaa, 0x9c, 0x91, 0x47, 0x0d, 0x54, 0x63, 0x0d, 0x48, + 0x3c, 0xd5, 0x04, 0xe4, 0x45, 0x28, 0xa8, 0x4f, 0x11, 0x13, 0x99, 0x54, 0xe2, 0x88, 0x82, 0x50, + 0x28, 0x87, 0xf2, 0x72, 0x6d, 0xee, 0xf0, 0xb0, 0x56, 0xc0, 0x1d, 0x8d, 0xf3, 0xec, 0xd2, 0x68, + 0x1b, 0x1b, 0x30, 0x49, 0xd1, 0x04, 0x8f, 0xfa, 0x21, 0x2c, 0x4e, 0x91, 0xa4, 0xe4, 0xb1, 0x17, + 0xcc, 0x3c, 0x56, 0x6a, 0x3e, 0x65, 0x18, 0x35, 0xde, 0x6c, 0xa6, 0xb7, 0x1d, 0x28, 0x9b, 0x4b, + 0x98, 0x87, 0x86, 0x8e, 0x77, 0xd7, 0x1f, 0x79, 0x1c, 0x19, 0x8b, 0x3c, 0xa4, 0x11, 0x42, 0xa7, + 0x2c, 0x08, 0xfc, 0x40, 0x2e, 0xcb, 0x62, 0x60, 0x60, 0xec, 0x9f, 0x5b, 0x30, 0xa7, 0xf4, 0x41, + 0x9e, 0x85, 0x9c, 0xd8, 0xa8, 0xdd, 0xb2, 0x92, 0x50, 0x18, 0x95, 0x6b, 0xc2, 0x79, 0x06, 0x0e, + 0xef, 0x3c, 0x64, 0x5d, 0xc5, 0x4d, 0x83, 0xe4, 0x75, 0x00, 0x87, 0xf3, 0xc0, 0x3d, 0x1e, 0x71, + 0x26, 0x2a, 0x8a, 0xe0, 0x71, 0x25, 0xe2, 0xa1, 0xba, 0x88, 0xd3, 0x1b, 0x8d, 0xb7, 0xd9, 0xf8, + 0x50, 0xdc, 0x86, 0x1a, 0xe4, 0x22, 0xd6, 0xb3, 0xe2, 0x18, 0xb2, 0x0c, 0x79, 0x71, 0x50, 0xe4, + 0x9b, 0x0a, 0x4a, 0x0d, 0xe1, 0x54, 0xf7, 0xca, 0x9c, 0xe5, 0x5e, 0xd7, 0xa1, 0xa2, 0x9d, 0x49, + 0xc0, 0xa1, 0x72, 0xc4, 0x24, 0x72, 0xe2, 0x16, 0xb9, 0xc7, 0xbb, 0xc5, 0xef, 0xa2, 0x5a, 0xae, + 0x82, 0x51, 0x44, 0x94, 0xeb, 0x85, 0x43, 0xd6, 0xe1, 0xac, 0x7b, 0xa0, 0x83, 0x1e, 0xeb, 0xdd, + 0x04, 0x9a, 0x3c, 0x07, 0xf3, 0x11, 0x6a, 0x73, 0x2c, 0x0e, 0x9f, 0x45, 0xf9, 0x26, 0xb0, 0x64, + 0x15, 0x4a, 0x98, 0xdd, 0xb1, 0xb8, 0xe9, 0xca, 0x6d, 0xa2, 0xc4, 0x45, 0x3b, 0xfe, 0x60, 0xd8, + 0x67, 0x9c, 0x75, 0xdf, 0xf2, 0x8f, 0x43, 0x5d, 0x7b, 0x12, 0x48, 0xe1, 0x37, 0xb8, 0x09, 0x29, + 0x64, 0xb0, 0xc5, 0x08, 0x21, 0x77, 0xcc, 0x52, 0x8a, 0x93, 0x47, 0x71, 0x26, 0xd1, 0x09, 0xb9, + 0xb1, 0x86, 0x63, 0x0d, 0x32, 0xe5, 0x46, 0xac, 0xdd, 0x13, 0xf1, 0x20, 0x54, 0x23, 0xaa, 0xba, + 0x2e, 0xca, 0x4b, 0x3a, 0x9d, 0x4b, 0x63, 0xab, 0x74, 0xbd, 0x04, 0x39, 0xec, 0xd1, 0x74, 0x6d, + 0x47, 0x20, 0x6e, 0x3c, 0x32, 0x29, 0x8d, 0x47, 0x36, 0x6a, 0x3c, 0xec, 0x8f, 0x33, 0xb0, 0x1c, + 0x9f, 0x94, 0xe8, 0x01, 0x5e, 0x9d, 0xee, 0x01, 0xea, 0x13, 0x59, 0xd4, 0x90, 0xee, 0xdb, 0x3e, + 0xe0, 0x9b, 0xd1, 0x07, 0x7c, 0x9a, 0x81, 0x2b, 0x91, 0x71, 0x30, 0xe8, 0x92, 0x56, 0xfd, 0xde, + 0xb4, 0x55, 0xaf, 0x4d, 0x5b, 0x55, 0x6e, 0xfc, 0xd6, 0xb4, 0xdf, 0x28, 0xd3, 0x6e, 0xe8, 0x56, + 0x5d, 0x86, 0x9d, 0x6a, 0x90, 0xea, 0x50, 0xe0, 0x4e, 0x4f, 0x74, 0x10, 0xb2, 0x16, 0x15, 0x69, + 0x04, 0xdb, 0x6f, 0xc1, 0x52, 0xbc, 0xe3, 0xb0, 0x19, 0xed, 0x69, 0x42, 0x1e, 0x93, 0x87, 0xae, + 0x5e, 0x69, 0x71, 0x7d, 0xd8, 0x94, 0x5d, 0xa1, 0xa2, 0xb4, 0x5f, 0x37, 0x53, 0x92, 0x5a, 0x8c, + 0x0a, 0x8d, 0x65, 0x14, 0x1a, 0x02, 0x59, 0x2e, 0x26, 0xb2, 0x59, 0x14, 0x06, 0xbf, 0xed, 0xa1, + 0x91, 0x65, 0x12, 0xbe, 0x85, 0xfd, 0x95, 0x14, 0x37, 0xea, 0xaf, 0x24, 0x78, 0x51, 0x62, 0xcb, + 0xa6, 0x24, 0xb6, 0x5c, 0x9c, 0xd8, 0x5e, 0x81, 0xa7, 0xa7, 0x4e, 0x54, 0xb7, 0x17, 0xc9, 0x5c, + 0x23, 0x95, 0xca, 0x62, 0x84, 0x7d, 0x13, 0x0a, 0x7a, 0x0b, 0x5e, 0x65, 0x1c, 0x25, 0x5c, 0xfc, + 0x4e, 0x9f, 0xa5, 0xec, 0x1d, 0xb8, 0x3c, 0x71, 0x9c, 0xa1, 0xee, 0xf5, 0xc9, 0x03, 0x4b, 0xcd, + 0xc5, 0xb8, 0x5d, 0x52, 0x2b, 0xa6, 0x0c, 0x7b, 0x90, 0xc3, 0x42, 0x47, 0x5a, 0x50, 0x09, 0x58, + 0xe8, 0x8f, 0x82, 0x0e, 0x6b, 0x1b, 0xdd, 0x46, 0x1c, 0xb1, 0xf2, 0xa5, 0xe0, 0xf4, 0x46, 0x83, + 0x9a, 0x64, 0x34, 0xb9, 0xcb, 0xde, 0x83, 0xf2, 0xfe, 0x28, 0x8c, 0x9b, 0xea, 0x37, 0xa0, 0x82, + 0x6d, 0x4d, 0xb8, 0x39, 0x3e, 0x50, 0xd3, 0x7b, 0x66, 0x6d, 0xde, 0x70, 0x46, 0x41, 0xdd, 0x12, + 0x14, 0x94, 0x39, 0xa1, 0xef, 0xd1, 0x24, 0xb9, 0xfd, 0x7b, 0x0b, 0xaa, 0x82, 0x04, 0x8b, 0x9a, + 0xb6, 0xe4, 0x4b, 0x51, 0xa7, 0x2e, 0x2c, 0x5f, 0xde, 0x7c, 0x4a, 0x4c, 0xda, 0xff, 0xfc, 0xec, + 0x5a, 0x65, 0x3f, 0x60, 0x4e, 0xbf, 0xef, 0x77, 0x24, 0xb5, 0x6e, 0xd1, 0xff, 0x1f, 0x32, 0x6e, + 0x57, 0xb6, 0x3e, 0x67, 0xd2, 0x0a, 0x0a, 0x72, 0x0b, 0x40, 0xe6, 0x9f, 0x2d, 0x87, 0x3b, 0xb5, + 0xec, 0x79, 0xf4, 0x06, 0xa1, 0xbd, 0x2b, 0x45, 0x94, 0xfa, 0x50, 0x22, 0xde, 0x86, 0xb9, 0x63, + 0x6c, 0xc0, 0xbe, 0xb4, 0x22, 0x35, 0xbd, 0x7d, 0x1d, 0x40, 0xbd, 0x46, 0x88, 0x3a, 0xbe, 0x9c, + 0x98, 0x4a, 0xca, 0xfa, 0x52, 0xf6, 0x1b, 0x50, 0xdc, 0x71, 0xbd, 0x93, 0x76, 0xdf, 0xed, 0x88, + 0xa1, 0x29, 0xd7, 0x77, 0xbd, 0x13, 0x7d, 0xd6, 0x95, 0xe9, 0xb3, 0xc4, 0x19, 0x0d, 0xb1, 0x81, 0x4a, 0x4a, 0xfb, 0x67, 0x16, 0x10, 0x81, 0xd4, 0xe3, 0x49, 0x5c, 0xf9, 0x65, 0x28, 0x58, 0x66, 0x28, 0xd4, 0x60, 0xae, 0x17, 0xf8, 0xa3, 0xe1, 0xa6, 0x0e, 0x11, 0x0d, 0x0a, 0xfa, 0x3e, 0x3e, 0x46, 0xc8, 0xfe, 0x4e, 0x02, 0x5f, 0x3a, 0x74, 0x7e, 0x69, 0xc1, 0x65, 0x43, 0x88, 0xf6, 0x68, @@ -3500,69 +3486,67 @@ var fileDescriptor_f22805646f4f62b6 = []byte{ 0x46, 0xbf, 0xb1, 0xa0, 0x64, 0x88, 0x48, 0x36, 0x61, 0xb1, 0xef, 0x70, 0xe6, 0x75, 0xc6, 0x47, 0x0f, 0xb5, 0x78, 0xca, 0x2b, 0xe3, 0x19, 0xca, 0x94, 0x9d, 0x56, 0x15, 0x7d, 0x7c, 0x9b, 0xef, 0x40, 0x3e, 0x64, 0x81, 0xab, 0xc2, 0xdb, 0xcc, 0x60, 0x51, 0x77, 0xaf, 0x08, 0xc4, 0xc5, 0x65, - 0xbe, 0x50, 0x8a, 0x55, 0x90, 0xfd, 0xb7, 0xa4, 0x77, 0x2b, 0xc7, 0x9a, 0x1e, 0xca, 0x2e, 0xb0, + 0xbe, 0x50, 0x8a, 0x55, 0x90, 0xfd, 0xf7, 0xa4, 0x77, 0x2b, 0xc7, 0x9a, 0x1e, 0xca, 0x2e, 0xb0, 0xd6, 0x6c, 0xaa, 0xb5, 0x62, 0xf9, 0x32, 0x17, 0xc9, 0x57, 0x85, 0xcc, 0xf0, 0xf6, 0x6d, 0x35, - 0xd2, 0x88, 0xa5, 0xc4, 0xdc, 0x42, 0xc7, 0x43, 0xcc, 0x2d, 0x89, 0xd9, 0x50, 0x7d, 0xbc, 0x58, - 0x22, 0xe6, 0xd6, 0x86, 0x6a, 0xd8, 0xc5, 0xd2, 0x7e, 0x0f, 0xea, 0x69, 0x71, 0xa2, 0x5c, 0xf4, - 0x36, 0x14, 0x43, 0x44, 0xb9, 0x6c, 0x3a, 0x05, 0xa4, 0x7c, 0x17, 0x53, 0xdb, 0xbf, 0xb5, 0xa0, - 0x92, 0x30, 0x6c, 0xa2, 0x12, 0xe5, 0x54, 0x25, 0x2a, 0x83, 0xe5, 0xa1, 0x32, 0x32, 0xd4, 0xf2, - 0x04, 0xf4, 0x00, 0xf5, 0x6d, 0x51, 0xeb, 0x81, 0x80, 0xe4, 0x28, 0x53, 0xa4, 0x56, 0x28, 0xa0, - 0x63, 0xbc, 0x5c, 0x81, 0x5a, 0xc7, 0x02, 0xea, 0xaa, 0x8b, 0x59, 0x5d, 0x9c, 0x21, 0xb9, 0xc3, - 0x47, 0xb2, 0x57, 0xca, 0x51, 0x05, 0x89, 0x13, 0x4f, 0x5c, 0xaf, 0x8b, 0xdd, 0x51, 0x8e, 0xe2, - 0xda, 0x66, 0xf2, 0x41, 0x52, 0x09, 0x2e, 0xd2, 0xac, 0x68, 0x7d, 0x02, 0x16, 0x8e, 0xfa, 0xfc, - 0x20, 0x2e, 0x94, 0x06, 0x46, 0xb4, 0x1a, 0x12, 0x52, 0x6e, 0x53, 0x4f, 0x8d, 0x21, 0xa4, 0xa0, - 0x8a, 0x52, 0x64, 0xc1, 0xc5, 0xa9, 0x5d, 0xe1, 0x26, 0x7d, 0xe7, 0x98, 0xf5, 0x8d, 0x5e, 0x21, - 0x46, 0x08, 0x39, 0x10, 0x38, 0x34, 0x6a, 0xb3, 0x81, 0x21, 0xeb, 0x30, 0xcb, 0xb5, 0x6b, 0x5c, - 0x3b, 0x5b, 0x86, 0x7d, 0xdf, 0xf5, 0x38, 0x9d, 0xe5, 0xa1, 0x88, 0xa1, 0xe5, 0xf4, 0x6d, 0x34, - 0x86, 0xab, 0x84, 0xa8, 0x50, 0x5c, 0x0b, 0xef, 0x38, 0x75, 0xfa, 0x78, 0xb0, 0x45, 0xc5, 0x52, - 0x4c, 0x85, 0xec, 0x11, 0x1b, 0x0c, 0xfb, 0x4e, 0x70, 0xa0, 0x5e, 0x90, 0x32, 0xf8, 0x56, 0x3f, - 0x89, 0x26, 0xcf, 0x43, 0x55, 0xa3, 0xf4, 0x8b, 0xb2, 0x72, 0xce, 0x29, 0xbc, 0xdd, 0x86, 0x4b, - 0xf8, 0x38, 0xbc, 0xed, 0x85, 0xdc, 0xf1, 0xf8, 0xf9, 0x59, 0x39, 0xca, 0xb2, 0x2a, 0xd3, 0x24, - 0xb2, 0xac, 0x8c, 0x4d, 0xcc, 0xb2, 0x8f, 0x60, 0x29, 0xc9, 0x54, 0xb9, 0x70, 0x23, 0x8a, 0x29, - 0xe9, 0xbf, 0x71, 0xda, 0x51, 0x94, 0x6d, 0xdc, 0x8d, 0x02, 0xeb, 0xf1, 0x9f, 0xdd, 0x7e, 0x6a, - 0x41, 0x25, 0xc1, 0x8b, 0xdc, 0x86, 0x3c, 0x9a, 0x6d, 0x3a, 0x66, 0xa6, 0xdf, 0x13, 0xd4, 0x6b, - 0xbe, 0xfa, 0x20, 0xd9, 0x9a, 0x59, 0x2a, 0x19, 0x92, 0x6b, 0x50, 0x1a, 0x06, 0xfe, 0xe0, 0x48, - 0x71, 0x95, 0x6f, 0x6f, 0x20, 0x50, 0x3b, 0x88, 0xb1, 0xff, 0x9d, 0x81, 0x45, 0xbc, 0x3e, 0x75, - 0xbc, 0x1e, 0x7b, 0x22, 0x1a, 0xc5, 0x29, 0x81, 0xb3, 0xa1, 0x32, 0x23, 0xae, 0x45, 0x8d, 0x0c, - 0x1f, 0x3a, 0x41, 0x77, 0x7b, 0x4b, 0x55, 0x38, 0x0d, 0x0a, 0xe7, 0xc5, 0xa5, 0xcc, 0x6f, 0x72, - 0xb0, 0x31, 0x30, 0xc9, 0x1f, 0x66, 0xe6, 0x26, 0x7e, 0x98, 0x31, 0x67, 0xb2, 0xc2, 0x39, 0x33, - 0x59, 0xf1, 0xc2, 0x99, 0x0c, 0xd2, 0x66, 0x32, 0x63, 0x12, 0x2a, 0x25, 0x27, 0x21, 0x73, 0x5a, - 0x2b, 0x4f, 0x4c, 0x6b, 0x7a, 0x4a, 0xaa, 0x9c, 0x39, 0x25, 0xcd, 0x7f, 0xa9, 0x29, 0x69, 0xe1, - 0x71, 0xa7, 0x24, 0xec, 0x0c, 0x54, 0xd0, 0x84, 0xb5, 0xaa, 0xbc, 0x73, 0x84, 0xb0, 0x43, 0x20, - 0xa6, 0xa9, 0x95, 0x9f, 0xbf, 0x30, 0xe1, 0xe7, 0x97, 0xe2, 0xf2, 0xea, 0x0e, 0xd8, 0x57, 0x76, - 0xf2, 0x9f, 0x40, 0xa1, 0xa5, 0x24, 0x78, 0xf2, 0xee, 0xfd, 0x7f, 0x50, 0x16, 0x09, 0x28, 0xe4, - 0xce, 0x60, 0x78, 0x34, 0x90, 0xfe, 0x9d, 0xa1, 0xa5, 0x08, 0xb7, 0x1b, 0xda, 0x77, 0x20, 0xdf, - 0x76, 0x06, 0xc3, 0xfe, 0x34, 0xf1, 0xec, 0x14, 0x71, 0x7c, 0x8a, 0x65, 0x9c, 0x62, 0x7f, 0x62, - 0x01, 0xc4, 0xba, 0xf8, 0x2a, 0xb7, 0x58, 0x87, 0xb9, 0x10, 0x85, 0xd1, 0x8d, 0xc4, 0x42, 0xac, - 0x3e, 0xc4, 0x2b, 0x7a, 0x4d, 0x75, 0x61, 0xfc, 0x92, 0x5b, 0xa6, 0xc5, 0xb3, 0x13, 0xc5, 0x5f, - 0x2b, 0x5e, 0x71, 0x8d, 0x29, 0x9f, 0xff, 0x00, 0x16, 0x26, 0xc6, 0x1c, 0x52, 0x86, 0xc2, 0xde, - 0xfd, 0xa3, 0x16, 0xa5, 0xf7, 0x69, 0x75, 0x86, 0x5c, 0x82, 0x85, 0xdd, 0x3b, 0xef, 0x1f, 0xed, - 0x6c, 0x1f, 0xb6, 0x8e, 0x0e, 0xe8, 0x9d, 0xbb, 0xad, 0x76, 0xd5, 0x12, 0x48, 0x5c, 0x1f, 0x1d, - 0xdc, 0xbf, 0x7f, 0xb4, 0x73, 0x87, 0xde, 0x6b, 0x55, 0x67, 0xc9, 0x22, 0x54, 0xde, 0xdd, 0x7b, - 0x7b, 0xef, 0xfe, 0x7b, 0x7b, 0xea, 0xe3, 0x4c, 0xf3, 0x57, 0x16, 0xe4, 0x05, 0x7b, 0x16, 0x90, - 0xef, 0x43, 0x31, 0x1a, 0x96, 0xc8, 0xe5, 0xc4, 0x8c, 0x65, 0x0e, 0x50, 0xf5, 0xa7, 0x12, 0x5b, - 0xda, 0x39, 0xed, 0x19, 0x72, 0x07, 0x4a, 0x11, 0xf1, 0x61, 0xf3, 0xbf, 0x61, 0xd1, 0xfc, 0x97, - 0x05, 0x55, 0xe5, 0x97, 0xf7, 0x98, 0xc7, 0x02, 0x87, 0xfb, 0x91, 0x60, 0x38, 0xe9, 0x4c, 0x70, - 0x35, 0xc7, 0xa6, 0xb3, 0x05, 0xdb, 0x06, 0xb8, 0xc7, 0xb8, 0xee, 0x32, 0xaf, 0xa4, 0x97, 0x55, - 0xc9, 0xe3, 0xea, 0x19, 0x35, 0x57, 0xb3, 0xba, 0x07, 0x10, 0x07, 0x26, 0x89, 0xbb, 0x84, 0xa9, - 0xc4, 0x5c, 0xbf, 0x92, 0xba, 0x17, 0xdd, 0xf4, 0x0f, 0x59, 0x98, 0x13, 0x1b, 0x2e, 0x0b, 0xc8, - 0x9b, 0x50, 0xf9, 0x81, 0xeb, 0x75, 0xa3, 0x9f, 0x91, 0x49, 0xca, 0xef, 0xce, 0x9a, 0x6d, 0x3d, - 0x6d, 0xcb, 0x30, 0x41, 0x59, 0xff, 0x30, 0xd5, 0x61, 0x1e, 0x27, 0x67, 0xfc, 0x1a, 0x5a, 0x7f, - 0x7a, 0x0a, 0x1f, 0xb1, 0x68, 0x41, 0xc9, 0xf8, 0xa5, 0xd5, 0xd4, 0xd6, 0xd4, 0xef, 0xaf, 0xe7, - 0xb1, 0xb9, 0x07, 0x10, 0xbf, 0xc3, 0x90, 0x73, 0x5e, 0x64, 0xeb, 0x57, 0x52, 0xf7, 0x22, 0x46, - 0x6f, 0xeb, 0x2b, 0xc9, 0x07, 0x9d, 0x73, 0x59, 0x3d, 0x93, 0xfa, 0x40, 0x64, 0x30, 0x3b, 0x84, - 0x85, 0x89, 0xf7, 0x0f, 0x72, 0xd1, 0xb3, 0x62, 0x7d, 0xf5, 0x6c, 0x82, 0x88, 0xef, 0x0f, 0x8d, - 0x57, 0x27, 0xfd, 0xae, 0x72, 0x31, 0x67, 0xfb, 0x2c, 0x02, 0x53, 0xe6, 0xe6, 0x5f, 0xb3, 0x50, - 0x6d, 0xf3, 0x80, 0x39, 0x03, 0xd7, 0xeb, 0x69, 0x97, 0x79, 0x1d, 0xf2, 0xaa, 0xf0, 0x3d, 0xae, - 0x89, 0x37, 0x2c, 0x11, 0x0f, 0x4f, 0xc4, 0x36, 0x1b, 0x16, 0xd9, 0x7d, 0x82, 0xd6, 0xd9, 0xb0, - 0xc8, 0xfb, 0x5f, 0x8f, 0x7d, 0x36, 0x2c, 0xf2, 0xc1, 0xd7, 0x67, 0xa1, 0x0d, 0x8b, 0xec, 0xc3, - 0xa2, 0xca, 0x15, 0x4f, 0x24, 0x3b, 0x6c, 0x58, 0xe4, 0x10, 0x2e, 0x99, 0x1c, 0x55, 0xf3, 0x49, - 0xae, 0x26, 0xbf, 0x4b, 0xb6, 0xd7, 0x86, 0x86, 0xd3, 0xfa, 0x64, 0xc1, 0xb7, 0xf9, 0x27, 0x0b, - 0xe6, 0x74, 0x26, 0x3c, 0x4a, 0x9d, 0x73, 0xed, 0xf3, 0xa6, 0x3f, 0x75, 0xd0, 0xb3, 0xe7, 0xd2, - 0x3c, 0xf1, 0x6c, 0xb9, 0x59, 0xfb, 0xe8, 0xf3, 0x15, 0xeb, 0x93, 0xcf, 0x57, 0xac, 0x7f, 0x7e, - 0xbe, 0x62, 0xfd, 0xfa, 0x8b, 0x95, 0x99, 0x4f, 0xbe, 0x58, 0x99, 0xf9, 0xf4, 0x8b, 0x95, 0x99, - 0xe3, 0x3c, 0xfe, 0xfb, 0xd1, 0xcb, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, 0x09, 0xb5, 0xc0, 0xda, - 0xff, 0x24, 0x00, 0x00, + 0xd2, 0x88, 0x4f, 0x89, 0xb9, 0x85, 0x8e, 0x87, 0x98, 0x5b, 0x12, 0xb3, 0xa1, 0xfa, 0x78, 0xf1, + 0x89, 0x98, 0x5b, 0x1b, 0xaa, 0x61, 0x17, 0x9f, 0xf6, 0x7b, 0x50, 0x4f, 0x8b, 0x13, 0xe5, 0xa2, + 0xb7, 0xa1, 0x18, 0x22, 0xca, 0x65, 0xd3, 0x29, 0x20, 0x65, 0x5f, 0x4c, 0x6d, 0xff, 0xd6, 0x82, + 0x4a, 0xc2, 0xb0, 0x89, 0x4a, 0x94, 0x53, 0x95, 0xa8, 0x0c, 0x96, 0x87, 0xca, 0xc8, 0x50, 0xcb, + 0x13, 0xd0, 0x03, 0xd4, 0xb7, 0x45, 0xad, 0x07, 0x02, 0x92, 0xa3, 0x4c, 0x91, 0x5a, 0xa1, 0x80, + 0x8e, 0xf1, 0x72, 0x05, 0x6a, 0x1d, 0x0b, 0xa8, 0xab, 0x2e, 0x66, 0x75, 0x71, 0x86, 0xe4, 0x0e, + 0x1f, 0xc9, 0x5e, 0x29, 0x47, 0x15, 0x24, 0x4e, 0x3c, 0x71, 0xbd, 0x2e, 0x76, 0x47, 0x39, 0x8a, + 0xdf, 0x36, 0x93, 0x0f, 0x92, 0x4a, 0x70, 0x91, 0x66, 0x45, 0xeb, 0x13, 0xb0, 0x70, 0xd4, 0xe7, + 0x07, 0x71, 0xa1, 0x34, 0x30, 0xa2, 0xd5, 0x90, 0x90, 0x72, 0x9b, 0x7a, 0x6a, 0x0c, 0x21, 0x05, + 0x55, 0x94, 0x22, 0x0b, 0x2e, 0x4e, 0xad, 0x0a, 0x37, 0xe9, 0x3b, 0xc7, 0xac, 0x6f, 0xf4, 0x0a, + 0x31, 0x42, 0xc8, 0x81, 0xc0, 0xa1, 0x51, 0x9b, 0x0d, 0x0c, 0x59, 0x87, 0x59, 0xae, 0x5d, 0xe3, + 0xda, 0xd9, 0x32, 0xec, 0xfb, 0xae, 0xc7, 0xe9, 0x2c, 0x0f, 0x45, 0x0c, 0x2d, 0xa7, 0x2f, 0xa3, + 0x31, 0x5c, 0x25, 0x44, 0x85, 0xe2, 0xb7, 0xf0, 0x8e, 0x53, 0xa7, 0x8f, 0x07, 0x5b, 0x54, 0x7c, + 0x8a, 0xa9, 0x90, 0x3d, 0x62, 0x83, 0x61, 0xdf, 0x09, 0x0e, 0xd4, 0x0b, 0x52, 0x06, 0xdf, 0xea, + 0x27, 0xd1, 0xe4, 0x79, 0xa8, 0x6a, 0x94, 0x7e, 0x51, 0x56, 0xce, 0x39, 0x85, 0xb7, 0xdb, 0x70, + 0x09, 0x1f, 0x87, 0xb7, 0xbd, 0x90, 0x3b, 0x1e, 0x3f, 0x3f, 0x2b, 0x47, 0x59, 0x56, 0x65, 0x9a, + 0x44, 0x96, 0x95, 0xb1, 0x89, 0x59, 0xf6, 0x11, 0x2c, 0x25, 0x99, 0x2a, 0x17, 0x6e, 0x44, 0x31, + 0x25, 0xfd, 0x37, 0x4e, 0x3b, 0x8a, 0xb2, 0x8d, 0xab, 0x51, 0x60, 0x3d, 0xfe, 0xb3, 0xdb, 0x4f, + 0x2d, 0xa8, 0x24, 0x78, 0x91, 0xdb, 0x90, 0x47, 0xb3, 0x4d, 0xc7, 0xcc, 0xf4, 0x7b, 0x82, 0x7a, + 0xcd, 0x57, 0x1b, 0x92, 0xad, 0x99, 0xa5, 0x92, 0x21, 0xb9, 0x06, 0xa5, 0x61, 0xe0, 0x0f, 0x8e, + 0x14, 0x57, 0xf9, 0xf6, 0x06, 0x02, 0xb5, 0x83, 0x18, 0xfb, 0x4f, 0x19, 0x58, 0xc4, 0xeb, 0x53, + 0xc7, 0xeb, 0xb1, 0x27, 0xa2, 0x51, 0x9c, 0x12, 0x38, 0x1b, 0x2a, 0x33, 0xe2, 0x77, 0xf2, 0xe7, + 0x95, 0xb9, 0x89, 0x9f, 0x57, 0xcc, 0xc9, 0xaa, 0x70, 0xce, 0x64, 0x55, 0xbc, 0x70, 0xb2, 0x82, + 0xb4, 0xc9, 0xca, 0x98, 0x67, 0x4a, 0xc9, 0x79, 0xc6, 0x9c, 0xb9, 0xca, 0x13, 0x33, 0x97, 0x9e, + 0x75, 0x2a, 0x67, 0xce, 0x3a, 0xf3, 0x5f, 0x6a, 0xd6, 0x59, 0x78, 0xdc, 0x59, 0x07, 0xeb, 0xbb, + 0x72, 0xfd, 0xb0, 0x56, 0x95, 0x77, 0x8e, 0x10, 0x76, 0x08, 0xc4, 0x34, 0x98, 0xf2, 0xd6, 0x17, + 0x26, 0xbc, 0xf5, 0x52, 0x5c, 0x24, 0xdd, 0x01, 0xfb, 0xca, 0xae, 0xfa, 0x13, 0x28, 0xb4, 0x94, + 0x04, 0x4f, 0xde, 0x49, 0xff, 0x0f, 0xca, 0x22, 0x8d, 0x84, 0xdc, 0x19, 0x0c, 0x8f, 0x06, 0xd2, + 0x4b, 0x33, 0xb4, 0x14, 0xe1, 0x76, 0x43, 0xfb, 0x0e, 0xe4, 0xdb, 0xce, 0x60, 0xd8, 0x9f, 0x26, + 0x9e, 0x9d, 0x22, 0x8e, 0x4f, 0xb1, 0x8c, 0x53, 0xec, 0x4f, 0x2c, 0x80, 0x58, 0x17, 0x5f, 0xe5, + 0x16, 0xeb, 0x30, 0x17, 0xa2, 0x30, 0xba, 0x1d, 0x58, 0x88, 0xd5, 0x87, 0x78, 0x45, 0xaf, 0xa9, + 0x2e, 0x8c, 0x42, 0x72, 0xcb, 0xb4, 0x78, 0x76, 0xa2, 0x84, 0x6b, 0xc5, 0x2b, 0xae, 0x31, 0xe5, + 0xf3, 0x1f, 0xc0, 0xc2, 0xc4, 0xb0, 0x42, 0xca, 0x50, 0xd8, 0xbb, 0x7f, 0xd4, 0xa2, 0xf4, 0x3e, + 0xad, 0xce, 0x90, 0x4b, 0xb0, 0xb0, 0x7b, 0xe7, 0xfd, 0xa3, 0x9d, 0xed, 0xc3, 0xd6, 0xd1, 0x01, + 0xbd, 0x73, 0xb7, 0xd5, 0xae, 0x5a, 0x02, 0x89, 0xdf, 0x47, 0x07, 0xf7, 0xef, 0x1f, 0xed, 0xdc, + 0xa1, 0xf7, 0x5a, 0xd5, 0x59, 0xb2, 0x08, 0x95, 0x77, 0xf7, 0xde, 0xde, 0xbb, 0xff, 0xde, 0x9e, + 0xda, 0x9c, 0x69, 0xfe, 0xca, 0x82, 0xbc, 0x60, 0xcf, 0x02, 0xf2, 0x7d, 0x28, 0x46, 0x23, 0x0f, + 0xb9, 0x9c, 0x98, 0x94, 0xcc, 0x31, 0xa8, 0xfe, 0x54, 0x62, 0x49, 0x3b, 0xa7, 0x3d, 0x43, 0xee, + 0x40, 0x29, 0x22, 0x3e, 0x6c, 0xfe, 0x37, 0x2c, 0x9a, 0xff, 0xb6, 0xa0, 0xaa, 0xfc, 0xf2, 0x1e, + 0xf3, 0x58, 0xe0, 0x70, 0x3f, 0x12, 0x0c, 0xe7, 0x95, 0x09, 0xae, 0xe6, 0xf0, 0x73, 0xb6, 0x60, + 0xdb, 0x00, 0xf7, 0x18, 0xd7, 0xbd, 0xe2, 0x95, 0xf4, 0xe2, 0x28, 0x79, 0x5c, 0x3d, 0xa3, 0x72, + 0x6a, 0x56, 0xf7, 0x00, 0xe2, 0xc0, 0x24, 0x71, 0xad, 0x9f, 0x4a, 0xaf, 0xf5, 0x2b, 0xa9, 0x6b, + 0xd1, 0x4d, 0xff, 0x90, 0x85, 0x39, 0xb1, 0xe0, 0xb2, 0x80, 0xbc, 0x09, 0x95, 0x1f, 0xb8, 0x5e, + 0x37, 0xfa, 0x31, 0x98, 0xa4, 0xfc, 0x7a, 0xac, 0xd9, 0xd6, 0xd3, 0x96, 0x0c, 0x13, 0x94, 0xf5, + 0xcf, 0x4b, 0x1d, 0xe6, 0x71, 0x72, 0xc6, 0x6f, 0x9a, 0xf5, 0xa7, 0xa7, 0xf0, 0x11, 0x8b, 0x16, + 0x94, 0x8c, 0xdf, 0x4b, 0x4d, 0x6d, 0x4d, 0xfd, 0x8a, 0x7a, 0x1e, 0x9b, 0x7b, 0x00, 0xf1, 0x6b, + 0x0a, 0x39, 0xe7, 0x5d, 0xb5, 0x7e, 0x25, 0x75, 0x2d, 0x62, 0xf4, 0xb6, 0xbe, 0x92, 0x7c, 0x96, + 0x39, 0x97, 0xd5, 0x33, 0xa9, 0xcf, 0x3c, 0x06, 0xb3, 0x43, 0x58, 0x98, 0x78, 0xc5, 0x20, 0x17, + 0x3d, 0x0e, 0xd6, 0x57, 0xcf, 0x26, 0x88, 0xf8, 0xfe, 0xd0, 0x78, 0x3b, 0xd2, 0xaf, 0x23, 0x17, + 0x73, 0xb6, 0xcf, 0x22, 0x30, 0x65, 0x6e, 0xfe, 0x2d, 0x0b, 0xd5, 0x36, 0x0f, 0x98, 0x33, 0x70, + 0xbd, 0x9e, 0x76, 0x99, 0xd7, 0x21, 0xaf, 0x0a, 0xdf, 0xe3, 0x9a, 0x78, 0xc3, 0x12, 0xf1, 0xf0, + 0x44, 0x6c, 0xb3, 0x61, 0x91, 0xdd, 0x27, 0x68, 0x9d, 0x0d, 0x8b, 0xbc, 0xff, 0xf5, 0xd8, 0x67, + 0xc3, 0x22, 0x1f, 0x7c, 0x7d, 0x16, 0xda, 0xb0, 0xc8, 0x3e, 0x2c, 0xaa, 0x5c, 0xf1, 0x44, 0xb2, + 0xc3, 0x86, 0x45, 0x0e, 0xe1, 0x92, 0xc9, 0x51, 0xb5, 0x90, 0xe4, 0x6a, 0x72, 0x5f, 0xb2, 0x49, + 0x36, 0x34, 0x9c, 0xd6, 0xed, 0x0a, 0xbe, 0xcd, 0x3f, 0x5b, 0x30, 0xa7, 0x33, 0xe1, 0x51, 0xea, + 0xb4, 0x6a, 0x9f, 0x37, 0xc3, 0xa9, 0x83, 0x9e, 0x3d, 0x97, 0xe6, 0x89, 0x67, 0xcb, 0xcd, 0xda, + 0x47, 0x9f, 0xaf, 0x58, 0x9f, 0x7c, 0xbe, 0x62, 0xfd, 0xeb, 0xf3, 0x15, 0xeb, 0xd7, 0x5f, 0xac, + 0xcc, 0x7c, 0xf2, 0xc5, 0xca, 0xcc, 0xa7, 0x5f, 0xac, 0xcc, 0x1c, 0xe7, 0xf1, 0x9f, 0x88, 0x5e, + 0xfe, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x64, 0x7a, 0x8f, 0x66, 0xc5, 0x24, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -7019,16 +7003,6 @@ func (m *QueryRangeRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x3a } - if m.ShardCount != 0 { - i = encodeVarintTempo(dAtA, i, uint64(m.ShardCount)) - i-- - dAtA[i] = 0x30 - } - if m.ShardID != 0 { - i = encodeVarintTempo(dAtA, i, uint64(m.ShardID)) - i-- - dAtA[i] = 0x28 - } if m.Step != 0 { i = encodeVarintTempo(dAtA, i, uint64(m.Step)) i-- @@ -8305,12 +8279,6 @@ func (m *QueryRangeRequest) Size() (n int) { if m.Step != 0 { n += 1 + sovTempo(uint64(m.Step)) } - if m.ShardID != 0 { - n += 1 + sovTempo(uint64(m.ShardID)) - } - if m.ShardCount != 0 { - n += 1 + sovTempo(uint64(m.ShardCount)) - } l = len(m.QueryMode) if l > 0 { n += 1 + l + sovTempo(uint64(l)) @@ -15092,44 +15060,6 @@ func (m *QueryRangeRequest) Unmarshal(dAtA []byte) error { break } } - case 5: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ShardID", wireType) - } - m.ShardID = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowTempo - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.ShardID |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 6: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ShardCount", wireType) - } - m.ShardCount = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowTempo - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.ShardCount |= uint32(b&0x7F) << shift - if b < 0x80 { - break - } - } case 7: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field QueryMode", wireType) diff --git a/pkg/tempopb/tempo.proto b/pkg/tempopb/tempo.proto index e16c30911ce..6fa72e927dc 100644 --- a/pkg/tempopb/tempo.proto +++ b/pkg/tempopb/tempo.proto @@ -388,8 +388,8 @@ message QueryRangeRequest { uint64 start = 2; uint64 end = 3; uint64 step = 4; - uint32 shardID = 5; - uint32 shardCount = 6; + //uint32 shardID = 5; // removed + //uint32 shardCount = 6; // removed string queryMode = 7; // New RF1 fields string blockID = 8; diff --git a/pkg/traceql/engine_metrics.go b/pkg/traceql/engine_metrics.go index 6c197bfb759..43f6a44828a 100644 --- a/pkg/traceql/engine_metrics.go +++ b/pkg/traceql/engine_metrics.go @@ -2,11 +2,8 @@ package traceql import ( "context" - "encoding/binary" "errors" "fmt" - "hash" - "hash/fnv" "math" "sort" "sync" @@ -728,7 +725,7 @@ func (e *Engine) CompileMetricsQueryRangeNonRaw(req *tempopb.QueryRangeRequest, // Dedupe spans parameter is an indicator of whether to expect duplicates in the datasource. For // example if the datasource is replication factor=1 or only a single block then we know there // aren't duplicates, and we can make some optimizations. -func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupeSpans bool, exemplars int, timeOverlapCutoff float64, allowUnsafeQueryHints bool) (*MetricsEvalulator, error) { +func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, exemplars int, timeOverlapCutoff float64, allowUnsafeQueryHints bool) (*MetricsEvalulator, error) { if req.Start <= 0 { return nil, fmt.Errorf("start required") } @@ -751,10 +748,6 @@ func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupe return nil, fmt.Errorf("not a metrics query") } - if v, ok := expr.Hints.GetBool(HintDedupe, allowUnsafeQueryHints); ok { - dedupeSpans = v - } - if v, ok := expr.Hints.GetInt(HintExemplars, allowUnsafeQueryHints); ok { exemplars = v } @@ -765,31 +758,11 @@ func (e *Engine) CompileMetricsQueryRange(req *tempopb.QueryRangeRequest, dedupe me := &MetricsEvalulator{ storageReq: storageReq, metricsPipeline: metricsPipeline, - dedupeSpans: dedupeSpans, timeOverlapCutoff: timeOverlapCutoff, maxExemplars: exemplars, exemplarMap: make(map[string]struct{}, exemplars), // TODO: Lazy, use bloom filter, CM sketch or something } - // TraceID (optional) - if req.ShardCount > 1 { - // For sharding it must be in the first pass so that we only evalulate our traces. - storageReq.ShardID = req.ShardID - storageReq.ShardCount = req.ShardCount - if !storageReq.HasAttribute(IntrinsicTraceIDAttribute) { - storageReq.Conditions = append(storageReq.Conditions, Condition{Attribute: IntrinsicTraceIDAttribute}) - } - } - - if dedupeSpans { - // For dedupe we only need the trace ID on matching spans, so it can go in the second pass. - // This is a no-op if we are already sharding and it's in the first pass. - // Finally, this is often optimized back to the first pass when it lets us avoid a second pass altogether. - if !storageReq.HasAttribute(IntrinsicTraceIDAttribute) { - storageReq.SecondPassConditions = append(storageReq.SecondPassConditions, Condition{Attribute: IntrinsicTraceIDAttribute}) - } - } - // Span start time (always required) if !storageReq.HasAttribute(IntrinsicSpanStartTimeAttribute) { // Technically we only need the start time of matching spans, so we add it to the second pass. @@ -905,8 +878,6 @@ func lookup(needles []Attribute, haystack Span) Static { type MetricsEvalulator struct { start, end uint64 checkTime bool - dedupeSpans bool - deduper *SpanDeduper2 maxExemplars, exemplarCount int exemplarMap map[string]struct{} timeOverlapCutoff float64 @@ -962,10 +933,6 @@ func (e *MetricsEvalulator) Do(ctx context.Context, f SpansetFetcher, fetcherSta return err } - if e.dedupeSpans && e.deduper == nil { - e.deduper = NewSpanDeduper2() - } - defer fetch.Results.Close() for { @@ -987,11 +954,6 @@ func (e *MetricsEvalulator) Do(ctx context.Context, f SpansetFetcher, fetcherSta } } - if e.dedupeSpans && e.deduper.Skip(ss.TraceID, s.StartTimeUnixNanos()) { - e.spansDeduped++ - continue - } - e.spansTotal++ e.metricsPipeline.observe(s) @@ -1042,56 +1004,6 @@ func (e *MetricsEvalulator) sampleExemplar(id []byte) bool { return true } -// SpanDeduper2 is EXTREMELY LAZY. It attempts to dedupe spans for metrics -// without requiring any new data fields. It uses trace ID and span start time -// which are already loaded. This of course terrible, but did I mention that -// this is extremely lazy? Additionally it uses sharded maps by the lowest byte -// of the trace ID to reduce the pressure on any single map. Maybe it's good enough. Let's find out! -type SpanDeduper2 struct { - m []map[uint32]struct{} - h hash.Hash32 - buf []byte - traceID Attribute -} - -func NewSpanDeduper2() *SpanDeduper2 { - maps := make([]map[uint32]struct{}, 256) - for i := range maps { - maps[i] = make(map[uint32]struct{}, 1000) - } - return &SpanDeduper2{ - m: maps, - h: fnv.New32a(), - buf: make([]byte, 8), - traceID: NewIntrinsic(IntrinsicTraceID), - } -} - -func (d *SpanDeduper2) Skip(tid []byte, startTime uint64) bool { - d.h.Reset() - d.h.Write(tid) - binary.BigEndian.PutUint64(d.buf, startTime) - d.h.Write(d.buf) - - v := d.h.Sum32() - - // Use last byte of the trace to choose the submap. - // Empty ID uses submap 0. - mapIdx := byte(0) - if len(tid) > 0 { - mapIdx = tid[len(tid)-1] - } - - m := d.m[mapIdx] - - if _, ok := m[v]; ok { - return true - } - - m[v] = struct{}{} - return false -} - // MetricsFrontendEvaluator pipes the sharded job results back into the engine for the rest // of the pipeline. i.e. This evaluator is for the query-frontend. type MetricsFrontendEvaluator struct { diff --git a/pkg/traceql/engine_metrics_test.go b/pkg/traceql/engine_metrics_test.go index dc403d9cb19..a6c5c4d1c14 100644 --- a/pkg/traceql/engine_metrics_test.go +++ b/pkg/traceql/engine_metrics_test.go @@ -219,7 +219,7 @@ func TestCompileMetricsQueryRange(t *testing.T) { Start: c.start, End: c.end, Step: c.step, - }, false, 0, 0, false) + }, 0, 0, false) if c.expectedErr != nil { require.EqualError(t, err, c.expectedErr.Error()) @@ -231,9 +231,6 @@ func TestCompileMetricsQueryRange(t *testing.T) { func TestCompileMetricsQueryRangeFetchSpansRequest(t *testing.T) { tc := map[string]struct { q string - shardID uint32 - shardCount uint32 - dedupe bool expectedReq FetchSpansRequest }{ "minimal": { @@ -249,37 +246,26 @@ func TestCompileMetricsQueryRangeFetchSpansRequest(t *testing.T) { }, }, "dedupe": { - q: "{} | rate()", - dedupe: true, + q: "{} | rate()", expectedReq: FetchSpansRequest{ AllConditions: true, Conditions: []Condition{ { Attribute: IntrinsicSpanStartTimeAttribute, }, - { - Attribute: IntrinsicTraceIDAttribute, // Required for dedupe - }, }, }, }, "secondPass": { - q: "{duration > 10s} | rate() by (resource.cluster)", - shardID: 123, - shardCount: 456, + q: "{duration > 10s} | rate() by (resource.cluster)", expectedReq: FetchSpansRequest{ AllConditions: true, - ShardID: 123, - ShardCount: 456, Conditions: []Condition{ { Attribute: IntrinsicDurationAttribute, Op: OpGreater, Operands: Operands{NewStaticDuration(10 * time.Second)}, }, - { - Attribute: IntrinsicTraceIDAttribute, // Required for sharding - }, }, SecondPassConditions: []Condition{ { @@ -294,22 +280,15 @@ func TestCompileMetricsQueryRangeFetchSpansRequest(t *testing.T) { }, }, "optimizations": { - q: "{duration > 10s} | rate() by (name, resource.service.name)", - shardID: 123, - shardCount: 456, + q: "{duration > 10s} | rate() by (name, resource.service.name)", expectedReq: FetchSpansRequest{ AllConditions: true, - ShardID: 123, - ShardCount: 456, Conditions: []Condition{ { Attribute: IntrinsicDurationAttribute, Op: OpGreater, Operands: Operands{NewStaticDuration(10 * time.Second)}, }, - { - Attribute: IntrinsicTraceIDAttribute, // Required for sharding - }, { // Intrinsic moved to first pass Attribute: IntrinsicNameAttribute, @@ -329,13 +308,11 @@ func TestCompileMetricsQueryRangeFetchSpansRequest(t *testing.T) { for n, tc := range tc { t.Run(n, func(t *testing.T) { eval, err := NewEngine().CompileMetricsQueryRange(&tempopb.QueryRangeRequest{ - Query: tc.q, - ShardID: tc.shardID, - ShardCount: tc.shardCount, - Start: 1, - End: 2, - Step: 3, - }, tc.dedupe, 0, 0, false) + Query: tc.q, + Start: 1, + End: 2, + Step: 3, + }, 0, 0, false) require.NoError(t, err) // Nil out func to Equal works @@ -438,7 +415,7 @@ func TestQuantileOverTime(t *testing.T) { } // 3 layers of processing matches: query-frontend -> queriers -> generators -> blocks - layer1, err := e.CompileMetricsQueryRange(req, false, 0, 0, false) + layer1, err := e.CompileMetricsQueryRange(req, 0, 0, false) require.NoError(t, err) layer2, err := e.CompileMetricsQueryRangeNonRaw(req, AggregateModeSum) @@ -545,7 +522,7 @@ func TestHistogramOverTime(t *testing.T) { } // 3 layers of processing matches: query-frontend -> queriers -> generators -> blocks - layer1, err := e.CompileMetricsQueryRange(req, false, 0, 0, false) + layer1, err := e.CompileMetricsQueryRange(req, 0, 0, false) require.NoError(t, err) layer2, err := e.CompileMetricsQueryRangeNonRaw(req, AggregateModeSum) @@ -574,26 +551,3 @@ func TestHistogramOverTime(t *testing.T) { final := layer3.Results() require.Equal(t, out, final) } - -func TestSpanDeduper(t *testing.T) { - d := NewSpanDeduper2() - - in := []struct { - tid []byte - ts uint64 - }{ - {nil, 0}, - {[]byte{1}, 1}, - {[]byte{1, 1}, 1}, - {[]byte{1, 2}, 2}, - } - - for _, tc := range in { - // First call is always false - require.False(t, d.Skip(tc.tid, tc.ts)) - - // Second call is always true - require.True(t, d.Skip(tc.tid, tc.ts)) - } - d.Skip(nil, 0) -} diff --git a/pkg/traceql/enum_hints.go b/pkg/traceql/enum_hints.go index d456bfadae3..e95de1fda93 100644 --- a/pkg/traceql/enum_hints.go +++ b/pkg/traceql/enum_hints.go @@ -8,8 +8,6 @@ import ( // and not part of the language or engine, we organize them here in one place. const ( HintSample = "sample" - HintDedupe = "dedupe" - HintJobInterval = "job_interval" HintJobSize = "job_size" HintTimeOverlapCutoff = "time_overlap_cutoff" HintConcurrentBlocks = "concurrent_blocks" diff --git a/pkg/traceql/storage.go b/pkg/traceql/storage.go index 03d0667425a..c7f18416c6c 100644 --- a/pkg/traceql/storage.go +++ b/pkg/traceql/storage.go @@ -86,10 +86,6 @@ type FetchSpansRequest struct { EndTimeUnixNanos uint64 Conditions []Condition - // mdisibio - Better to push trace by ID filtering into Conditions with a new between op? - ShardID uint32 - ShardCount uint32 - // Hints // By default, the storage layer fetches spans meeting any of the criteria. diff --git a/pkg/util/traceidboundary/traceidboundary.go b/pkg/util/traceidboundary/traceidboundary.go deleted file mode 100644 index 430ed67ecce..00000000000 --- a/pkg/util/traceidboundary/traceidboundary.go +++ /dev/null @@ -1,136 +0,0 @@ -package traceidboundary - -import ( - "bytes" - "encoding/binary" - "math" -) - -type Boundary struct { - Min, Max []byte -} - -// Pairs returns the boundaries that match trace IDs in that shard. Internally this is -// similar to how queriers divide the block ID-space, but here it's trace IDs instead. -// The inputs are 1-based because it seems more readable: shard 1 of 10. Most boundaries -// are [,) lower inclusive, upper exclusive. However the last boundary that ends in the -// max value 0xFFFF... is [,] inclusive/inclusive and indicated when the return value -// upperInclusive is set. -// Of course there are some caveats: -// - Trace IDs can be 16 or 8 bytes. If we naively sharded only in 16-byte space it would -// be unbalanced because all 8-byte IDs would land in the first shard. Therefore we -// divide in both 16- and 8-byte spaces and a single shard covers a range in each. -// - There are different regimes of 8-byte ID generation and they are not uniformly -// distributed. So there are several sub-shards within 8-byte space to compensate. -func Pairs(shard, of uint32) (boundaries []Boundary, upperInclusive bool) { - boundaries = append(boundaries, complicatedShardingFor8ByteIDs(shard, of)...) - - // Final pair is the normal full precision 16-byte IDs. - b := bounds(of, 0, math.MaxUint64, 0) - - // Avoid overlap with the 64-bit precision ones. I.e. a minimum of 0x0000.... would - // unintentionally include all 64-bit IDs. The first 65-bit ID starts here: - b[0] = []byte{0, 0, 0, 0, 0, 0, 0, 0x01, 0, 0, 0, 0, 0, 0, 0, 0} - - // Adjust max to be full 16-byte max - b[of] = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF} - - boundaries = append(boundaries, Boundary{ - Min: b[shard-1], - Max: b[shard], - }) - - // Top most 0xFFFFF... boundary is inclusive - upperInclusive = shard == of - - return -} - -// Funcs returns helper functions that match trace IDs in the given shard. -func Funcs(shard, of uint32) (testSingle func([]byte) bool, testRange func([]byte, []byte) bool) { - pairs, upperInclusive := Pairs(shard, of) - - upper := -1 - if upperInclusive { - upper = 0 - } - - isMatch := func(id []byte) bool { - for _, p := range pairs { - if bytes.Compare(p.Min, id) <= 0 && bytes.Compare(id, p.Max) <= upper { - return true - } - } - return false - } - - withinRange := func(min []byte, max []byte) bool { - for _, p := range pairs { - if bytes.Compare(p.Min, max) <= 0 && bytes.Compare(min, p.Max) <= upper { - return true - } - } - return false - } - - return isMatch, withinRange -} - -// complicatedShardingFor8ByteIDs generates a list of trace ID boundaries that is subdividing -// the 64-bit space. Not optimal or universal rules. This seems like overkill but in practice -// 8-byte IDs are unevenly weighted towards lower values starting with zeros. The benefit of -// this approach is better fairness across shards, and also *invariance* across workloads, -// no matter if your instrumentation is generating 8-byte or 16-byte trace IDs. -func complicatedShardingFor8ByteIDs(shard, of uint32) []Boundary { - var results []Boundary - - regions := []struct { - min, max uint64 - }{ - {0x0000000000000000, 0x00FFFFFFFFFFFFFF}, // Region with upper byte = 0 - {0x0100000000000000, 0x0FFFFFFFFFFFFFFF}, // Region with upper nibble = 0 - {0x1000000000000000, 0x7FFFFFFFFFFFFFFF}, // Region for 63-bit IDs (upper bit = 0) - {0x8000000000000000, 0xFFFFFFFFFFFFFFFF}, // Region for true 64-bit IDs - } - - for _, r := range regions { - b := bounds(of, r.min, r.max, 8) - results = append(results, Boundary{ - Min: b[shard-1], - Max: b[shard], - }) - } - - return results -} - -func bounds(shards uint32, min, max uint64, dest int) [][]byte { - if shards == 0 { - return nil - } - - bounds := make([][]byte, shards+1) - for i := uint32(0); i < shards+1; i++ { - bounds[i] = make([]byte, 16) - } - - bucketSz := (max - min) / uint64(shards) - - // numLarger is the number of buckets that have to be bumped by 1 - numLarger := max % bucketSz - - boundary := min - for i := uint32(0); i < shards; i++ { - binary.BigEndian.PutUint64(bounds[i][dest:], boundary) - - boundary += bucketSz - if numLarger != 0 { - numLarger-- - boundary++ - } - } - - binary.BigEndian.PutUint64(bounds[shards][dest:], max) - - return bounds -} diff --git a/pkg/util/traceidboundary/traceidboundary_test.go b/pkg/util/traceidboundary/traceidboundary_test.go deleted file mode 100644 index 9dd722f662f..00000000000 --- a/pkg/util/traceidboundary/traceidboundary_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package traceidboundary - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/require" -) - -func TestPairs(t *testing.T) { - testCases := []struct { - shard, of uint32 - expectedPairs []Boundary - expectedUpper bool - }{ - //--------------------------------------------- - // Simplest case, 2 shards - //--------------------------------------------- - {1, 2, []Boundary{ - { - // First half of upper-byte = 0 (between 0x0000 and 0x00FF) - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0, 0, 0, 0, 0, 0}, - }, - { - // First half of upper-nibble = 0 (between 0x01 and 0x0F) - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0x01, 0, 0, 0, 0, 0, 0, 0}, - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x80, 0, 0, 0, 0, 0, 0}, - }, - { - // First half of upper-bit = 0 (between 0x10 and 0x7F) - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0x10, 0, 0, 0, 0, 0, 0, 0}, - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0x48, 0, 0, 0, 0, 0, 0, 0}, - }, - { - // First half of full 8-byte ids (between 0x80 and 0xFF) - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0, 0, 0, 0, 0, 0, 0}, - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0xC0, 0, 0, 0, 0, 0, 0, 0}, - }, - { - // First half of full 16-byte ids (between 0x00 and 0x80) - []byte{0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0}, - []byte{0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - }, - }, false}, - {2, 2, []Boundary{ - { - // Second half of upper-byte = 0 (between 0x0000 and 0x00FF) - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0, 0, 0, 0, 0, 0}, - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, - }, - { - // Second half of upper-nibble = 0 (between 0x01 and 0x0F) - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0x80, 0, 0, 0, 0, 0, 0}, - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0x0F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, - }, - { - // Second half of upper-bit = 0 (between 0x10 and 0x7F) - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0x48, 0, 0, 0, 0, 0, 0, 0}, - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0x7F, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, - }, - { - // Second half of full 8-byte ids (between 0x80 and 0xFF) - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0xC0, 0, 0, 0, 0, 0, 0, 0}, - []byte{0, 0, 0, 0, 0, 0, 0, 0, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, - }, - { - // Second half of full 16-byte ids (between 0x80 and 0xFF) - []byte{0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, - []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, - }, - }, true}, - } - - for _, tc := range testCases { - t.Run(fmt.Sprintf("%d of %d", tc.shard, tc.of), func(t *testing.T) { - pairs, upper := Pairs(tc.shard, tc.of) - require.Equal(t, tc.expectedPairs, pairs) - require.Equal(t, tc.expectedUpper, upper) - }) - } -} diff --git a/tempodb/encoding/vparquet2/block_traceql.go b/tempodb/encoding/vparquet2/block_traceql.go index 9ae9f0ea1aa..4bb056c1918 100644 --- a/tempodb/encoding/vparquet2/block_traceql.go +++ b/tempodb/encoding/vparquet2/block_traceql.go @@ -19,7 +19,6 @@ import ( v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1" "github.com/grafana/tempo/pkg/traceql" "github.com/grafana/tempo/pkg/util" - "github.com/grafana/tempo/pkg/util/traceidboundary" "github.com/grafana/tempo/tempodb/encoding/common" ) @@ -1200,7 +1199,7 @@ func (i *mergeSpansetIterator) Close() { // V func fetch(ctx context.Context, req traceql.FetchSpansRequest, pf *parquet.File, rowGroups []parquet.RowGroup) (*spansetIterator, error) { - iter, err := createAllIterator(ctx, nil, req.Conditions, req.AllConditions, req.StartTimeUnixNanos, req.EndTimeUnixNanos, req.ShardID, req.ShardCount, rowGroups, pf) + iter, err := createAllIterator(ctx, nil, req.Conditions, req.AllConditions, req.StartTimeUnixNanos, req.EndTimeUnixNanos, rowGroups, pf) if err != nil { return nil, fmt.Errorf("error creating iterator: %w", err) } @@ -1208,7 +1207,7 @@ func fetch(ctx context.Context, req traceql.FetchSpansRequest, pf *parquet.File, if req.SecondPass != nil { iter = newBridgeIterator(newRebatchIterator(iter), req.SecondPass) - iter, err = createAllIterator(ctx, iter, req.SecondPassConditions, false, 0, 0, req.ShardID, req.ShardCount, rowGroups, pf) + iter, err = createAllIterator(ctx, iter, req.SecondPassConditions, false, 0, 0, rowGroups, pf) if err != nil { return nil, fmt.Errorf("error creating second pass iterator: %w", err) } @@ -1218,7 +1217,7 @@ func fetch(ctx context.Context, req traceql.FetchSpansRequest, pf *parquet.File, } func createAllIterator(ctx context.Context, primaryIter parquetquery.Iterator, conds []traceql.Condition, allConditions bool, start, end uint64, - shardID, shardCount uint32, rgs []parquet.RowGroup, pf *parquet.File, + rgs []parquet.RowGroup, pf *parquet.File, ) (parquetquery.Iterator, error) { // Categorize conditions into span-level or resource-level var ( @@ -1292,7 +1291,7 @@ func createAllIterator(ctx context.Context, primaryIter parquetquery.Iterator, c return nil, fmt.Errorf("creating resource iterator: %w", err) } - return createTraceIterator(makeIter, resourceIter, traceConditions, start, end, shardID, shardCount, allConditions) + return createTraceIterator(makeIter, resourceIter, traceConditions, start, end, allConditions) } // createSpanIterator iterates through all span-level columns, groups them into rows representing @@ -1560,7 +1559,7 @@ func createResourceIterator(makeIter makeIterFn, spanIterator parquetquery.Itera required, iters, batchCol) } -func createTraceIterator(makeIter makeIterFn, resourceIter parquetquery.Iterator, conds []traceql.Condition, start, end uint64, shardID, shardCount uint32, allConditions bool) (parquetquery.Iterator, error) { +func createTraceIterator(makeIter makeIterFn, resourceIter parquetquery.Iterator, conds []traceql.Condition, start, end uint64, allConditions bool) (parquetquery.Iterator, error) { iters := make([]parquetquery.Iterator, 0, 3) // add conditional iterators first. this way if someone searches for { traceDuration > 1s && span.foo = "bar"} the query will @@ -2276,16 +2275,3 @@ func orIfNeeded(preds []parquetquery.Predicate) parquetquery.Predicate { return parquetquery.NewOrPredicate(preds...) } } - -// NewTraceIDShardingPredicate creates a predicate for the TraceID column to match only IDs -// within the shard. If sharding isn't present, returns nil meaning no predicate. -func NewTraceIDShardingPredicate(shardID, shardCount uint32) parquetquery.Predicate { - if shardCount <= 1 || shardID <= 0 { - return nil - } - - isMatch, withinRange := traceidboundary.Funcs(shardID, shardCount) - extract := func(v parquet.Value) []byte { return v.ByteArray() } - - return parquetquery.NewGenericPredicate(isMatch, withinRange, extract) -} diff --git a/tempodb/encoding/vparquet3/block_traceql.go b/tempodb/encoding/vparquet3/block_traceql.go index 95f744a29c7..5d849533628 100644 --- a/tempodb/encoding/vparquet3/block_traceql.go +++ b/tempodb/encoding/vparquet3/block_traceql.go @@ -13,15 +13,12 @@ import ( "time" "unsafe" - "github.com/opentracing/opentracing-go" "github.com/parquet-go/parquet-go" - "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/pkg/parquetquery" v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1" "github.com/grafana/tempo/pkg/traceql" "github.com/grafana/tempo/pkg/util" - "github.com/grafana/tempo/pkg/util/traceidboundary" "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/encoding/common" ) @@ -893,15 +890,7 @@ func (b *backendBlock) Fetch(ctx context.Context, req traceql.FetchSpansRequest, return traceql.FetchSpansResponse{}, err } - var rgs []parquet.RowGroup - if req.ShardCount > 0 { - rgs, err = b.rowGroupsForShard(ctx, pf, *b.meta, req.ShardID, req.ShardCount) - if err != nil { - return traceql.FetchSpansResponse{}, err - } - } else { - rgs = rowGroupsFromFile(pf, opts) - } + rgs := rowGroupsFromFile(pf, opts) iter, err := fetch(ctx, req, pf, rgs, b.meta.DedicatedColumns) if err != nil { @@ -1350,7 +1339,7 @@ func (i *mergeSpansetIterator) Close() { // V func fetch(ctx context.Context, req traceql.FetchSpansRequest, pf *parquet.File, rowGroups []parquet.RowGroup, dc backend.DedicatedColumns) (*spansetIterator, error) { - iter, err := createAllIterator(ctx, nil, req.Conditions, req.AllConditions, req.StartTimeUnixNanos, req.EndTimeUnixNanos, req.ShardID, req.ShardCount, rowGroups, pf, dc, false) + iter, err := createAllIterator(ctx, nil, req.Conditions, req.AllConditions, req.StartTimeUnixNanos, req.EndTimeUnixNanos, rowGroups, pf, dc, false) if err != nil { return nil, fmt.Errorf("error creating iterator: %w", err) } @@ -1358,7 +1347,7 @@ func fetch(ctx context.Context, req traceql.FetchSpansRequest, pf *parquet.File, if req.SecondPass != nil { iter = newBridgeIterator(newRebatchIterator(iter), req.SecondPass) - iter, err = createAllIterator(ctx, iter, req.SecondPassConditions, false, 0, 0, req.ShardID, req.ShardCount, rowGroups, pf, dc, req.SecondPassSelectAll) + iter, err = createAllIterator(ctx, iter, req.SecondPassConditions, false, 0, 0, rowGroups, pf, dc, req.SecondPassSelectAll) if err != nil { return nil, fmt.Errorf("error creating second pass iterator: %w", err) } @@ -1406,7 +1395,7 @@ func categorizeConditions(conditions []traceql.Condition) (mingled bool, spanCon } func createAllIterator(ctx context.Context, primaryIter parquetquery.Iterator, conds []traceql.Condition, allConditions bool, start, end uint64, - shardID, shardCount uint32, rgs []parquet.RowGroup, pf *parquet.File, dc backend.DedicatedColumns, selectAll bool, + rgs []parquet.RowGroup, pf *parquet.File, dc backend.DedicatedColumns, selectAll bool, ) (parquetquery.Iterator, error) { // categorizeConditions conditions into span-level or resource-level mingledConditions, spanConditions, resourceConditions, traceConditions, err := categorizeConditions(conds) @@ -1445,7 +1434,7 @@ func createAllIterator(ctx context.Context, primaryIter parquetquery.Iterator, c return nil, fmt.Errorf("creating resource iterator: %w", err) } - return createTraceIterator(makeIter, resourceIter, traceConditions, start, end, shardID, shardCount, allConditions, selectAll) + return createTraceIterator(makeIter, resourceIter, traceConditions, start, end, allConditions, selectAll) } // createSpanIterator iterates through all span-level columns, groups them into rows representing @@ -1858,7 +1847,7 @@ func createResourceIterator(makeIter makeIterFn, spanIterator parquetquery.Itera return parquetquery.NewLeftJoinIterator(DefinitionLevelResourceSpans, required, iters, batchCol, parquetquery.WithPool(pqSpansetPool)) } -func createTraceIterator(makeIter makeIterFn, resourceIter parquetquery.Iterator, conds []traceql.Condition, start, end uint64, _, _ uint32, allConditions bool, selectAll bool) (parquetquery.Iterator, error) { +func createTraceIterator(makeIter makeIterFn, resourceIter parquetquery.Iterator, conds []traceql.Condition, start, end uint64, allConditions bool, selectAll bool) (parquetquery.Iterator, error) { iters := make([]parquetquery.Iterator, 0, 3) // add conditional iterators first. this way if someone searches for { traceDuration > 1s && span.foo = "bar"} the query will @@ -2610,65 +2599,3 @@ func unsafeToString(b []byte) string { } return unsafe.String(unsafe.SliceData(b), len(b)) } - -// NewTraceIDShardingPredicate creates a predicate for the TraceID column to match only IDs -// within the shard. If sharding isn't present, returns nil meaning no predicate. -func NewTraceIDShardingPredicate(shardID, shardCount uint32) parquetquery.Predicate { - if shardCount <= 1 || shardID <= 0 { - return nil - } - - isMatch, withinRange := traceidboundary.Funcs(shardID, shardCount) - extract := func(v parquet.Value) []byte { return v.ByteArray() } - - return parquetquery.NewGenericPredicate(isMatch, withinRange, extract) -} - -// rowGroupsForShard uses the block trace ID index to more efficiently find the row -// groups that contain trace IDs within the given shard. Reading the trace ID index -// is a single read and typically comes from cache. Without this we have to test every -// row group in the file which would be N reads. -func (b *backendBlock) rowGroupsForShard(ctx context.Context, pf *parquet.File, m backend.BlockMeta, shardID, shardCount uint32) ([]parquet.RowGroup, error) { - span, _ := opentracing.StartSpanFromContext(ctx, "parquet.rowGroupsForShard") - defer span.Finish() - - cacheInfo := &backend.CacheInfo{ - Meta: &m, - Role: cache.RoleTraceIDIdx, - } - - indexBytes, err := b.r.Read(ctx, common.NameIndex, b.meta.BlockID, b.meta.TenantID, cacheInfo) - if errors.Is(err, backend.ErrDoesNotExist) { - // No index, check all groups - return pf.RowGroups(), nil - } - if err != nil { - return nil, err - } - - index, err := unmarshalIndex(indexBytes) - if err != nil { - return nil, fmt.Errorf("error parsing index (%s, %s): %w", b.meta.TenantID, b.meta.BlockID, err) - } - - _, testRange := traceidboundary.Funcs(shardID, shardCount) - - rgs := pf.RowGroups() - matches := []parquet.RowGroup{} - for i := 0; i < len(index.RowGroups); i++ { - if i == 0 { - if testRange([]byte{}, index.RowGroups[i]) { - matches = append(matches, rgs[i]) - } - } else { - if testRange(index.RowGroups[i-1], index.RowGroups[i]) { - matches = append(matches, rgs[i]) - } - } - } - - span.SetTag("totalRowGroups", len(rgs)) - span.SetTag("matchedRowGroups", len(matches)) - - return matches, nil -} diff --git a/tempodb/encoding/vparquet3/block_traceql_test.go b/tempodb/encoding/vparquet3/block_traceql_test.go index 3e2f2c00034..6b9f074ed5c 100644 --- a/tempodb/encoding/vparquet3/block_traceql_test.go +++ b/tempodb/encoding/vparquet3/block_traceql_test.go @@ -4,9 +4,7 @@ import ( "bytes" "context" "fmt" - "math" "math/rand" - "os" "path" "strconv" "testing" @@ -21,7 +19,6 @@ import ( "github.com/grafana/tempo/pkg/traceql" "github.com/grafana/tempo/pkg/traceqlmetrics" "github.com/grafana/tempo/pkg/util/test" - "github.com/grafana/tempo/pkg/util/traceidboundary" "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/backend/local" "github.com/grafana/tempo/tempodb/encoding/common" @@ -756,15 +753,13 @@ func BenchmarkBackendBlockQueryRange(b *testing.B) { } req := &tempopb.QueryRangeRequest{ - Query: tc, - Step: uint64(time.Minute), - Start: uint64(st.UnixNano()), - End: uint64(end.UnixNano()), - ShardID: 30, - ShardCount: 65, + Query: tc, + Step: uint64(time.Minute), + Start: uint64(st.UnixNano()), + End: uint64(end.UnixNano()), } - eval, err := e.CompileMetricsQueryRange(req, false, 4, 0, false) + eval, err := e.CompileMetricsQueryRange(req, 4, 0, false) require.NoError(b, err) b.ResetTimer() @@ -783,124 +778,6 @@ func BenchmarkBackendBlockQueryRange(b *testing.B) { } } -func TestTraceIDShardingQuality(t *testing.T) { - // Use debug=1 go test -v -run=TestTraceIDShardingQuality - if os.Getenv("debug") != "1" { - t.Skip() - } - - var ( - ctx = context.TODO() - opts = common.DefaultSearchOptions() - tenantID = "1" - // blockID = uuid.MustParse("06ebd383-8d4e-4289-b0e9-cf2197d611d5") - blockID = uuid.MustParse("18364616-f80d-45a6-b2a3-cb63e203edff") - path = "/Users/marty/src/tmp/" - ) - - r, _, _, err := local.New(&local.Config{ - Path: path, - }) - require.NoError(t, err) - - rr := backend.NewReader(r) - meta, err := rr.BlockMeta(ctx, blockID, tenantID) - require.NoError(t, err) - require.Equal(t, VersionString, meta.Version) - - block := newBackendBlock(meta, rr) - pf, _, err := block.openForSearch(ctx, opts) - require.NoError(t, err) - - fetchReq := traceql.FetchSpansRequest{ - AllConditions: true, - Conditions: []traceql.Condition{ - {Attribute: traceql.IntrinsicTraceIDAttribute}, - }, - } - - f := traceql.NewSpansetFetcherWrapper(func(ctx context.Context, req traceql.FetchSpansRequest) (traceql.FetchSpansResponse, error) { - return block.Fetch(ctx, req, opts) - }) - - summarizeCounts := func(prefix string, cs []int) { - count := len(cs) - sum := 0 - l := math.MaxInt - h := math.MinInt - for _, v := range cs { - sum += v - l = min(l, v) - h = max(h, v) - } - fmt.Printf("Shards:%d %s: Min:%d Max:%d Total:%d Avg:%.1f Quality:%.1f %%\n", - count, prefix, l, h, sum, float64(sum)/float64(count), float64(l)/float64(h)*100.0) - } - - shardsToTest := []int{10, 100, 1000} - - for _, shards := range shardsToTest { - t.Run(strconv.Itoa(shards), func(t *testing.T) { - var ( - rgCounts = make([]int, shards) - trCounts = make([]int, shards) - pairs = make([][]traceidboundary.Boundary, shards) - funcs = make([]func([]byte) bool, shards) - ) - - for s := 1; s <= shards; s++ { - pairs[s-1], _ = traceidboundary.Pairs(uint32(s), uint32(shards)) - funcs[s-1], _ = traceidboundary.Funcs(uint32(s), uint32(shards)) - - rgs, err := block.rowGroupsForShard(ctx, pf, *meta, uint32(s), uint32(shards)) - require.NoError(t, err) - rgCounts[s-1] = len(rgs) - } - - resp, err := f.Fetch(ctx, fetchReq) - require.NoError(t, err) - defer resp.Results.Close() - - for { - ss, err := resp.Results.Next(ctx) - require.NoError(t, err) - - if ss == nil { - break - } - - // Match the trace ID against every shard - matched := []int{} - for i := 0; i < shards; i++ { - if funcs[i](ss.TraceID) { - trCounts[i]++ - matched = append(matched, i) - } - } - - // Check for missing or overlapping ranges - if len(matched) > 1 { - fmt.Printf("TraceID %X matched %d shards\n", ss.TraceID, len(matched)) - for _, s := range matched { - for i, b := range pairs[s] { - fmt.Printf(" Bucket %d %d: %X %X\n", s, i, b.Min, b.Max) - } - } - panic("trace matched multiple shards") - } else if len(matched) == 0 { - fmt.Println("TraceID not matched by any shard:", ss.TraceID) - panic("trace id not matched by any shard") - } - - ss.Release() - } - - summarizeCounts("Traces", trCounts) - summarizeCounts("RowGroups", rgCounts) - }) - } -} - func TestDescendantOf(t *testing.T) { ancestor1 := &span{nestedSetLeft: 3, nestedSetRight: 8} descendant1a := &span{nestedSetLeft: 4, nestedSetRight: 5} diff --git a/tempodb/encoding/vparquet4/block_traceql.go b/tempodb/encoding/vparquet4/block_traceql.go index 75c8353abf0..50580d707f6 100644 --- a/tempodb/encoding/vparquet4/block_traceql.go +++ b/tempodb/encoding/vparquet4/block_traceql.go @@ -13,15 +13,12 @@ import ( "time" "unsafe" - "github.com/opentracing/opentracing-go" "github.com/parquet-go/parquet-go" - "github.com/grafana/tempo/pkg/cache" "github.com/grafana/tempo/pkg/parquetquery" v1 "github.com/grafana/tempo/pkg/tempopb/trace/v1" "github.com/grafana/tempo/pkg/traceql" "github.com/grafana/tempo/pkg/util" - "github.com/grafana/tempo/pkg/util/traceidboundary" "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/encoding/common" ) @@ -979,15 +976,7 @@ func (b *backendBlock) Fetch(ctx context.Context, req traceql.FetchSpansRequest, return traceql.FetchSpansResponse{}, err } - var rgs []parquet.RowGroup - if req.ShardCount > 0 { - rgs, err = b.rowGroupsForShard(ctx, pf, *b.meta, req.ShardID, req.ShardCount) - if err != nil { - return traceql.FetchSpansResponse{}, err - } - } else { - rgs = rowGroupsFromFile(pf, opts) - } + rgs := rowGroupsFromFile(pf, opts) iter, err := fetch(ctx, req, pf, rgs, b.meta.DedicatedColumns) if err != nil { @@ -3118,68 +3107,6 @@ func unsafeToString(b []byte) string { return unsafe.String(unsafe.SliceData(b), len(b)) } -// NewTraceIDShardingPredicate creates a predicate for the TraceID column to match only IDs -// within the shard. If sharding isn't present, returns nil meaning no predicate. -func NewTraceIDShardingPredicate(shardID, shardCount uint32) parquetquery.Predicate { - if shardCount <= 1 || shardID <= 0 { - return nil - } - - isMatch, withinRange := traceidboundary.Funcs(shardID, shardCount) - extract := func(v parquet.Value) []byte { return v.ByteArray() } - - return parquetquery.NewGenericPredicate(isMatch, withinRange, extract) -} - -// rowGroupsForShard uses the block trace ID index to more efficiently find the row -// groups that contain trace IDs within the given shard. Reading the trace ID index -// is a single read and typically comes from cache. Without this we have to test every -// row group in the file which would be N reads. -func (b *backendBlock) rowGroupsForShard(ctx context.Context, pf *parquet.File, m backend.BlockMeta, shardID, shardCount uint32) ([]parquet.RowGroup, error) { - span, _ := opentracing.StartSpanFromContext(ctx, "parquet.rowGroupsForShard") - defer span.Finish() - - cacheInfo := &backend.CacheInfo{ - Meta: &m, - Role: cache.RoleTraceIDIdx, - } - - indexBytes, err := b.r.Read(ctx, common.NameIndex, b.meta.BlockID, b.meta.TenantID, cacheInfo) - if errors.Is(err, backend.ErrDoesNotExist) { - // No index, check all groups - return pf.RowGroups(), nil - } - if err != nil { - return nil, err - } - - index, err := unmarshalIndex(indexBytes) - if err != nil { - return nil, fmt.Errorf("error parsing index (%s, %s): %w", b.meta.TenantID, b.meta.BlockID, err) - } - - _, testRange := traceidboundary.Funcs(shardID, shardCount) - - rgs := pf.RowGroups() - matches := []parquet.RowGroup{} - for i := 0; i < len(index.RowGroups); i++ { - if i == 0 { - if testRange([]byte{}, index.RowGroups[i]) { - matches = append(matches, rgs[i]) - } - } else { - if testRange(index.RowGroups[i-1], index.RowGroups[i]) { - matches = append(matches, rgs[i]) - } - } - } - - span.SetTag("totalRowGroups", len(rgs)) - span.SetTag("matchedRowGroups", len(matches)) - - return matches, nil -} - func otlpStatusToTraceqlStatus(v uint64) traceql.Status { // Map OTLP status code back to TraceQL enum. // For other values, use the raw integer. diff --git a/tempodb/encoding/vparquet4/block_traceql_test.go b/tempodb/encoding/vparquet4/block_traceql_test.go index 87eb6113d3e..3284d655650 100644 --- a/tempodb/encoding/vparquet4/block_traceql_test.go +++ b/tempodb/encoding/vparquet4/block_traceql_test.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "fmt" - "math" "math/rand" "os" "path" @@ -24,7 +23,6 @@ import ( "github.com/grafana/tempo/pkg/traceqlmetrics" "github.com/grafana/tempo/pkg/util" "github.com/grafana/tempo/pkg/util/test" - "github.com/grafana/tempo/pkg/util/traceidboundary" "github.com/grafana/tempo/tempodb/backend" "github.com/grafana/tempo/tempodb/backend/local" "github.com/grafana/tempo/tempodb/encoding/common" @@ -1075,7 +1073,7 @@ func BenchmarkBackendBlockQueryRange(b *testing.B) { End: uint64(end.UnixNano()), } - eval, err := e.CompileMetricsQueryRange(req, false, 2, 0, false) + eval, err := e.CompileMetricsQueryRange(req, 2, 0, false) require.NoError(b, err) b.ResetTimer() @@ -1162,7 +1160,7 @@ func TestBackendBlockQueryRange(t *testing.T) { End: uint64(end.UnixNano()), } - eval, err := e.CompileMetricsQueryRange(req, false, 1, 0, false) + eval, err := e.CompileMetricsQueryRange(req, 1, 0, false) require.NoError(t, err) require.NoError(t, eval.Do(ctx, f, uint64(block.meta.StartTime.UnixNano()), uint64(block.meta.EndTime.UnixNano()))) @@ -1180,124 +1178,6 @@ func TestBackendBlockQueryRange(t *testing.T) { } } -func TestTraceIDShardingQuality(t *testing.T) { - // Use debug=1 go test -v -run=TestTraceIDShardingQuality - if os.Getenv("debug") != "1" { - t.Skip() - } - - var ( - ctx = context.TODO() - opts = common.DefaultSearchOptions() - tenantID = "1" - // blockID = uuid.MustParse("06ebd383-8d4e-4289-b0e9-cf2197d611d5") - blockID = uuid.MustParse("18364616-f80d-45a6-b2a3-cb63e203edff") - path = "/Users/marty/src/tmp/" - ) - - r, _, _, err := local.New(&local.Config{ - Path: path, - }) - require.NoError(t, err) - - rr := backend.NewReader(r) - meta, err := rr.BlockMeta(ctx, blockID, tenantID) - require.NoError(t, err) - require.Equal(t, VersionString, meta.Version) - - block := newBackendBlock(meta, rr) - pf, _, err := block.openForSearch(ctx, opts) - require.NoError(t, err) - - fetchReq := traceql.FetchSpansRequest{ - AllConditions: true, - Conditions: []traceql.Condition{ - {Attribute: traceql.IntrinsicTraceIDAttribute}, - }, - } - - f := traceql.NewSpansetFetcherWrapper(func(ctx context.Context, req traceql.FetchSpansRequest) (traceql.FetchSpansResponse, error) { - return block.Fetch(ctx, req, opts) - }) - - summarizeCounts := func(prefix string, cs []int) { - count := len(cs) - sum := 0 - l := math.MaxInt - h := math.MinInt - for _, v := range cs { - sum += v - l = min(l, v) - h = max(h, v) - } - fmt.Printf("Shards:%d %s: Min:%d Max:%d Total:%d Avg:%.1f Quality:%.1f %%\n", - count, prefix, l, h, sum, float64(sum)/float64(count), float64(l)/float64(h)*100.0) - } - - shardsToTest := []int{10, 100, 1000} - - for _, shards := range shardsToTest { - t.Run(strconv.Itoa(shards), func(t *testing.T) { - var ( - rgCounts = make([]int, shards) - trCounts = make([]int, shards) - pairs = make([][]traceidboundary.Boundary, shards) - funcs = make([]func([]byte) bool, shards) - ) - - for s := 1; s <= shards; s++ { - pairs[s-1], _ = traceidboundary.Pairs(uint32(s), uint32(shards)) - funcs[s-1], _ = traceidboundary.Funcs(uint32(s), uint32(shards)) - - rgs, err := block.rowGroupsForShard(ctx, pf, *meta, uint32(s), uint32(shards)) - require.NoError(t, err) - rgCounts[s-1] = len(rgs) - } - - resp, err := f.Fetch(ctx, fetchReq) - require.NoError(t, err) - defer resp.Results.Close() - - for { - ss, err := resp.Results.Next(ctx) - require.NoError(t, err) - - if ss == nil { - break - } - - // Match the trace ID against every shard - matched := []int{} - for i := 0; i < shards; i++ { - if funcs[i](ss.TraceID) { - trCounts[i]++ - matched = append(matched, i) - } - } - - // Check for missing or overlapping ranges - if len(matched) > 1 { - fmt.Printf("TraceID %X matched %d shards\n", ss.TraceID, len(matched)) - for _, s := range matched { - for i, b := range pairs[s] { - fmt.Printf(" Bucket %d %d: %X %X\n", s, i, b.Min, b.Max) - } - } - panic("trace matched multiple shards") - } else if len(matched) == 0 { - fmt.Println("TraceID not matched by any shard:", ss.TraceID) - panic("trace id not matched by any shard") - } - - ss.Release() - } - - summarizeCounts("Traces", trCounts) - summarizeCounts("RowGroups", rgCounts) - }) - } -} - func ptr[T any](v T) *T { return &v }