fix slow sql

chengjoey · Dec 4, 2023 · 9305535 · 9305535
1 parent 4c7b77d
commit 9305535
Show file tree

Hide file tree

Showing 7 changed files with 21 additions and 6 deletions.
diff --git a/api/proto/oap/entity/entity.proto b/api/proto/oap/entity/entity.proto
@@ -79,6 +79,8 @@ message ListEntitiesRequest {
   int64               updateTimeUnixNanoMin = 4;
   int64               updateTimeUnixNanoMax = 5;
   bool                debug                 = 6;
+  int64               createTimeUnixNanoMin = 7;
+  int64               createTimeUnixNanoMax = 8;
 }
 
 message ListEntitiesResponse {

diff --git a/cmd/monitor/streaming/conf/clickhouse/entities_ddl_create_default_tables.sql b/cmd/monitor/streaming/conf/clickhouse/entities_ddl_create_default_tables.sql
@@ -7,11 +7,16 @@ CREATE TABLE IF NOT EXISTS <database>.entities ON CLUSTER '{cluster}'
     `type` LowCardinality(String),
     `key` String,
     `values` Map(String,String),
-    `labels` Map(String,String)
+    `labels` Map(String,String),
+    `labels.terminusKey` LowCardinality(String) MATERIALIZED labels['terminusKey'],
+
+    INDEX idx_type(type) TYPE bloom_filter GRANULARITY 1,
+    INDEX idx_key(key) TYPE bloom_filter GRANULARITY 1,
+    INDEX idx_terminus_key(labels.terminusKey) TYPE bloom_filter GRANULARITY 1
 )
 ENGINE = ReplicatedMergeTree('/clickhouse/tables/{cluster}--{shard}/entities', '{replica}')
 PARTITION BY toYYYYMMDD(timestamp)
-ORDER BY (timestamp)
+ORDER BY (timestamp, type, key)
 TTL toDateTime(timestamp) + INTERVAL <ttl_in_days> DAY;
 
 // create distributed table

diff --git a/internal/apps/msp/apm/exception/query/source/elasticsearch_source.go b/internal/apps/msp/apm/exception/query/source/elasticsearch_source.go
@@ -46,8 +46,8 @@ func (source *ElasticsearchSource) GetExceptions(ctx context.Context, req *pb.Ge
 		Type:                  "error_exception",
 		Labels:                conditions,
 		Limit:                 int64(1000),
-		UpdateTimeUnixNanoMin: req.StartTime * 1e6,
-		UpdateTimeUnixNanoMax: req.EndTime * 1e6,
+		CreateTimeUnixNanoMin: req.StartTime * 1e6,
+		CreateTimeUnixNanoMax: req.EndTime * 1e6,
 		Debug:                 req.Debug,
 	}
 	exceptions, err := source.fetchErdaErrorFromES(ctx, entityReq, req.StartTime, req.EndTime)

diff --git a/internal/tools/monitor/core/entity/entity.go b/internal/tools/monitor/core/entity/entity.go
@@ -29,7 +29,7 @@ type Entity struct {
 }
 
 type GroupedEntity struct {
-	Timestamp       time.Time         `ch:"timestamp"`
+	Timestamp       time.Time         `ch:"_timestamp"`
 	UpdateTimestamp time.Time         `ch:"_update_timestamp"`
 	ID              string            `ch:"id"`
 	Type            string            `ch:"_type"`

diff --git a/internal/tools/monitor/core/entity/query/entity.service.go b/internal/tools/monitor/core/entity/query/entity.service.go
@@ -70,6 +70,8 @@ func (s *entityService) ListEntities(ctx context.Context, req *pb.ListEntitiesRe
 		Limit:                 int(req.Limit),
 		UpdateTimeUnixNanoMax: req.UpdateTimeUnixNanoMax,
 		UpdateTimeUnixNanoMin: req.UpdateTimeUnixNanoMin,
+		CreateTimeUnixNanoMax: req.CreateTimeUnixNanoMax,
+		CreateTimeUnixNanoMin: req.CreateTimeUnixNanoMin,
 		Debug:                 req.Debug,
 	})
 	if err != nil {

diff --git a/internal/tools/monitor/core/entity/storage/clickhouse/storage.go b/internal/tools/monitor/core/entity/storage/clickhouse/storage.go
@@ -101,7 +101,7 @@ func (p *provider) GetEntity(ctx context.Context, typ, key string) (*entity.Enti
 func (p *provider) ListEntities(ctx context.Context, opts *storage.ListOptions) ([]*entity.Entity, int64, error) {
 	table, _ := p.Loader.GetSearchTable("")
 	sql := goqu.From(table).Select(
-		goqu.L("min(timestamp)").As("timestamp"),
+		goqu.L("min(timestamp)").As("_timestamp"),
 		goqu.L("max(update_timestamp)").As("_update_timestamp"),
 		goqu.L("any(id)").As("id"),
 		goqu.L("any(type)").As("_type"),
@@ -118,6 +118,10 @@ func (p *provider) ListEntities(ctx context.Context, opts *storage.ListOptions)
 		for k, v := range opts.Labels {
 			sql = sql.Where(goqu.L("labels[?]", k).Eq(v))
 		}
+		if opts.CreateTimeUnixNanoMin > 0 || opts.CreateTimeUnixNanoMax > 0 {
+			sql = sql.Where(goqu.C("timestamp").Gte(goqu.L("fromUnixTimestamp64Nano(cast(?,'Int64'))", opts.CreateTimeUnixNanoMin)),
+				goqu.C("timestamp").Lt(goqu.L("fromUnixTimestamp64Nano(cast(?,'Int64'))", opts.CreateTimeUnixNanoMax)))
+		}
 		if opts.UpdateTimeUnixNanoMin > 0 || opts.UpdateTimeUnixNanoMax > 0 {
 			sql = sql.Where(goqu.C("update_timestamp").Gte(goqu.L("fromUnixTimestamp64Nano(cast(?,'Int64'))", opts.UpdateTimeUnixNanoMin)),
 				goqu.C("update_timestamp").Lt(goqu.L("fromUnixTimestamp64Nano(cast(?,'Int64'))", opts.UpdateTimeUnixNanoMax)))

diff --git a/internal/tools/monitor/core/entity/storage/storage.go b/internal/tools/monitor/core/entity/storage/storage.go
@@ -29,6 +29,8 @@ type (
 		Limit                 int
 		UpdateTimeUnixNanoMin int64
 		UpdateTimeUnixNanoMax int64
+		CreateTimeUnixNanoMin int64
+		CreateTimeUnixNanoMax int64
 		Debug                 bool
 	}
 	// Storage .