From 9305535806a76443a3db31b93ec7428644ce0e29 Mon Sep 17 00:00:00 2001 From: joey Date: Mon, 4 Dec 2023 14:53:18 +0800 Subject: [PATCH] fix slow sql --- api/proto/oap/entity/entity.proto | 2 ++ .../clickhouse/entities_ddl_create_default_tables.sql | 9 +++++++-- .../apm/exception/query/source/elasticsearch_source.go | 4 ++-- internal/tools/monitor/core/entity/entity.go | 2 +- .../tools/monitor/core/entity/query/entity.service.go | 2 ++ .../monitor/core/entity/storage/clickhouse/storage.go | 6 +++++- internal/tools/monitor/core/entity/storage/storage.go | 2 ++ 7 files changed, 21 insertions(+), 6 deletions(-) diff --git a/api/proto/oap/entity/entity.proto b/api/proto/oap/entity/entity.proto index 8a341ad2755..f40d712b0e3 100644 --- a/api/proto/oap/entity/entity.proto +++ b/api/proto/oap/entity/entity.proto @@ -79,6 +79,8 @@ message ListEntitiesRequest { int64 updateTimeUnixNanoMin = 4; int64 updateTimeUnixNanoMax = 5; bool debug = 6; + int64 createTimeUnixNanoMin = 7; + int64 createTimeUnixNanoMax = 8; } message ListEntitiesResponse { diff --git a/cmd/monitor/streaming/conf/clickhouse/entities_ddl_create_default_tables.sql b/cmd/monitor/streaming/conf/clickhouse/entities_ddl_create_default_tables.sql index 7db0a6f828e..b364290597b 100644 --- a/cmd/monitor/streaming/conf/clickhouse/entities_ddl_create_default_tables.sql +++ b/cmd/monitor/streaming/conf/clickhouse/entities_ddl_create_default_tables.sql @@ -7,11 +7,16 @@ CREATE TABLE IF NOT EXISTS .entities ON CLUSTER '{cluster}' `type` LowCardinality(String), `key` String, `values` Map(String,String), - `labels` Map(String,String) + `labels` Map(String,String), + `labels.terminusKey` LowCardinality(String) MATERIALIZED labels['terminusKey'], + + INDEX idx_type(type) TYPE bloom_filter GRANULARITY 1, + INDEX idx_key(key) TYPE bloom_filter GRANULARITY 1, + INDEX idx_terminus_key(labels.terminusKey) TYPE bloom_filter GRANULARITY 1 ) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{cluster}--{shard}/entities', '{replica}') PARTITION BY toYYYYMMDD(timestamp) -ORDER BY (timestamp) +ORDER BY (timestamp, type, key) TTL toDateTime(timestamp) + INTERVAL DAY; // create distributed table diff --git a/internal/apps/msp/apm/exception/query/source/elasticsearch_source.go b/internal/apps/msp/apm/exception/query/source/elasticsearch_source.go index 6a9b029e557..2e87b9ccb7f 100644 --- a/internal/apps/msp/apm/exception/query/source/elasticsearch_source.go +++ b/internal/apps/msp/apm/exception/query/source/elasticsearch_source.go @@ -46,8 +46,8 @@ func (source *ElasticsearchSource) GetExceptions(ctx context.Context, req *pb.Ge Type: "error_exception", Labels: conditions, Limit: int64(1000), - UpdateTimeUnixNanoMin: req.StartTime * 1e6, - UpdateTimeUnixNanoMax: req.EndTime * 1e6, + CreateTimeUnixNanoMin: req.StartTime * 1e6, + CreateTimeUnixNanoMax: req.EndTime * 1e6, Debug: req.Debug, } exceptions, err := source.fetchErdaErrorFromES(ctx, entityReq, req.StartTime, req.EndTime) diff --git a/internal/tools/monitor/core/entity/entity.go b/internal/tools/monitor/core/entity/entity.go index 838b9dae856..d121e5b4b3d 100644 --- a/internal/tools/monitor/core/entity/entity.go +++ b/internal/tools/monitor/core/entity/entity.go @@ -29,7 +29,7 @@ type Entity struct { } type GroupedEntity struct { - Timestamp time.Time `ch:"timestamp"` + Timestamp time.Time `ch:"_timestamp"` UpdateTimestamp time.Time `ch:"_update_timestamp"` ID string `ch:"id"` Type string `ch:"_type"` diff --git a/internal/tools/monitor/core/entity/query/entity.service.go b/internal/tools/monitor/core/entity/query/entity.service.go index d930eaf1256..4ffd9893722 100644 --- a/internal/tools/monitor/core/entity/query/entity.service.go +++ b/internal/tools/monitor/core/entity/query/entity.service.go @@ -70,6 +70,8 @@ func (s *entityService) ListEntities(ctx context.Context, req *pb.ListEntitiesRe Limit: int(req.Limit), UpdateTimeUnixNanoMax: req.UpdateTimeUnixNanoMax, UpdateTimeUnixNanoMin: req.UpdateTimeUnixNanoMin, + CreateTimeUnixNanoMax: req.CreateTimeUnixNanoMax, + CreateTimeUnixNanoMin: req.CreateTimeUnixNanoMin, Debug: req.Debug, }) if err != nil { diff --git a/internal/tools/monitor/core/entity/storage/clickhouse/storage.go b/internal/tools/monitor/core/entity/storage/clickhouse/storage.go index 635851c0dfb..f299eb8e3ab 100644 --- a/internal/tools/monitor/core/entity/storage/clickhouse/storage.go +++ b/internal/tools/monitor/core/entity/storage/clickhouse/storage.go @@ -101,7 +101,7 @@ func (p *provider) GetEntity(ctx context.Context, typ, key string) (*entity.Enti func (p *provider) ListEntities(ctx context.Context, opts *storage.ListOptions) ([]*entity.Entity, int64, error) { table, _ := p.Loader.GetSearchTable("") sql := goqu.From(table).Select( - goqu.L("min(timestamp)").As("timestamp"), + goqu.L("min(timestamp)").As("_timestamp"), goqu.L("max(update_timestamp)").As("_update_timestamp"), goqu.L("any(id)").As("id"), goqu.L("any(type)").As("_type"), @@ -118,6 +118,10 @@ func (p *provider) ListEntities(ctx context.Context, opts *storage.ListOptions) for k, v := range opts.Labels { sql = sql.Where(goqu.L("labels[?]", k).Eq(v)) } + if opts.CreateTimeUnixNanoMin > 0 || opts.CreateTimeUnixNanoMax > 0 { + sql = sql.Where(goqu.C("timestamp").Gte(goqu.L("fromUnixTimestamp64Nano(cast(?,'Int64'))", opts.CreateTimeUnixNanoMin)), + goqu.C("timestamp").Lt(goqu.L("fromUnixTimestamp64Nano(cast(?,'Int64'))", opts.CreateTimeUnixNanoMax))) + } if opts.UpdateTimeUnixNanoMin > 0 || opts.UpdateTimeUnixNanoMax > 0 { sql = sql.Where(goqu.C("update_timestamp").Gte(goqu.L("fromUnixTimestamp64Nano(cast(?,'Int64'))", opts.UpdateTimeUnixNanoMin)), goqu.C("update_timestamp").Lt(goqu.L("fromUnixTimestamp64Nano(cast(?,'Int64'))", opts.UpdateTimeUnixNanoMax))) diff --git a/internal/tools/monitor/core/entity/storage/storage.go b/internal/tools/monitor/core/entity/storage/storage.go index f89f0ded2ef..5facc37878e 100644 --- a/internal/tools/monitor/core/entity/storage/storage.go +++ b/internal/tools/monitor/core/entity/storage/storage.go @@ -29,6 +29,8 @@ type ( Limit int UpdateTimeUnixNanoMin int64 UpdateTimeUnixNanoMax int64 + CreateTimeUnixNanoMin int64 + CreateTimeUnixNanoMax int64 Debug bool } // Storage .