Support tracing trace query(SkyWalking and Zipkin) for debugging. (#1…

…2378)
apache · Jun 27, 2024 · 05373d6 · 05373d6
1 parent b04fd3c
commit 05373d6
Show file tree

Hide file tree

Showing 49 changed files with 1,260 additions and 619 deletions.
diff --git a/docs/en/api/query-protocol.md b/docs/en/api/query-protocol.md
@@ -149,11 +149,12 @@ full log text fuzzy queries, while others do not due to considerations related t
 
 ### Trace
 ```graphql
+# Param, if debug is true will enable the query tracing and return DebuggingTrace in the result.
 extend type Query {
     # Search segment list with given conditions
-    queryBasicTraces(condition: TraceQueryCondition): TraceBrief
+    queryBasicTraces(condition: TraceQueryCondition, debug: Boolean): TraceBrief
     # Read the specific trace ID with given trace ID
-    queryTrace(traceId: ID!): Trace
+    queryTrace(traceId: ID!, debug: Boolean): Trace
     # Read the list of searchable keys
     queryTraceTagAutocompleteKeys(duration: Duration!):[String!]
     # Search the available value options of the given key.

diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md
@@ -20,6 +20,7 @@
 * Make more proper histogram buckets for the `persistence_timer_bulk_prepare_latency`,
   `persistence_timer_bulk_execute_latency` and `persistence_timer_bulk_all_latency` metrics in PersistenceTimer.
 * [Break Change] Update Nacos version to 2.3.2. Nacos 1.x server can't serve as cluster coordinator and configuration server.
+* Support tracing trace query(SkyWalking and Zipkin) for debugging.
 
 #### UI
 

diff --git a/docs/en/debugging/query-tracing.md b/docs/en/debugging/query-tracing.md
diff --git a/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/MQEVisitorBase.java b/oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/MQEVisitorBase.java
@@ -47,7 +47,7 @@
 import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext;
 import org.apache.skywalking.oap.server.library.util.StringUtil;
 
-import static org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTrace.TRACE_CONTEXT;
+import static org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext.TRACE_CONTEXT;
 
 @Slf4j
 public abstract class MQEVisitorBase extends MQEParserBaseVisitor<ExpressionResult> {

diff --git a/...-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRule.java b/...-plugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/AlarmRule.java
@@ -42,7 +42,7 @@
 import org.apache.skywalking.oap.server.core.storage.annotation.ValueColumnMetadata;
 import org.apache.skywalking.oap.server.library.util.StringUtil;
 
-import static org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTrace.TRACE_CONTEXT;
+import static org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext.TRACE_CONTEXT;
 
 @Data
 @ToString

diff --git a/...lugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java b/...lugin/src/main/java/org/apache/skywalking/oap/server/core/alarm/provider/RunningRule.java
@@ -60,7 +60,7 @@
 import org.joda.time.format.DateTimeFormat;
 import org.joda.time.format.DateTimeFormatter;
 
-import static org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTrace.TRACE_CONTEXT;
+import static org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext.TRACE_CONTEXT;
 
 /**
  * RunningRule represents each rule in running status. Based on the {@link AlarmRule} definition,

diff --git a/...re/src/main/java/org/apache/skywalking/oap/server/core/query/AggregationQueryService.java b/...re/src/main/java/org/apache/skywalking/oap/server/core/query/AggregationQueryService.java
@@ -22,7 +22,6 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
-import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTrace;
 import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingSpan;
 import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext;
 import org.apache.skywalking.oap.server.library.util.StringUtil;
@@ -70,7 +69,7 @@ private List<SelectedRecord> invokeSortMetrics(TopNCondition condition, Duration
             final String serviceId = IDManager.ServiceID.buildId(condition.getParentService(), condition.getNormal());
             additionalConditions.add(new KeyValue(InstanceTraffic.SERVICE_ID, serviceId));
         }
-        final List<SelectedRecord> selectedRecords = getAggregationQueryDAO().sortMetrics(
+        final List<SelectedRecord> selectedRecords = getAggregationQueryDAO().sortMetricsDebuggable(
             condition, valueCName, duration, additionalConditions);
         selectedRecords.forEach(selectedRecord -> {
             switch (condition.getScope()) {
@@ -115,7 +114,7 @@ private List<SelectedRecord> invokeSortMetrics(TopNCondition condition, Duration
 
     public List<SelectedRecord> sortMetrics(TopNCondition condition,
                                             Duration duration) throws IOException {
-        DebuggingTraceContext traceContext = DebuggingTrace.TRACE_CONTEXT.get();
+        DebuggingTraceContext traceContext = DebuggingTraceContext.TRACE_CONTEXT.get();
         DebuggingSpan span = null;
         try {
             if (traceContext != null) {

diff --git a/...r-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetricsQueryService.java b/...r-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetricsQueryService.java
@@ -38,7 +38,7 @@
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 import org.apache.skywalking.oap.server.library.module.Service;
 
-import static org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTrace.TRACE_CONTEXT;
+import static org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext.TRACE_CONTEXT;
 
 @Slf4j
 public class MetricsQueryService implements Service {
@@ -80,7 +80,7 @@ private MetricsValues invokeReadMetricsValues(MetricsCondition condition, Durati
         if (!condition.senseScope() || !condition.getEntity().isValid()) {
             return new MetricsValues();
         }
-        return getMetricQueryDAO().readMetricsValues(
+        return getMetricQueryDAO().readMetricsValuesDebuggable(
             condition, ValueColumnMetadata.INSTANCE.getValueCName(condition.getName()), duration);
     }
 
@@ -109,7 +109,7 @@ private List<MetricsValues> invokeReadLabeledMetricsValues(MetricsCondition cond
         if (!condition.senseScope() || !condition.getEntity().isValid()) {
             return Collections.emptyList();
         }
-        return getMetricQueryDAO().readLabeledMetricsValues(
+        return getMetricQueryDAO().readLabeledMetricsValuesDebuggable(
             condition, ValueColumnMetadata.INSTANCE.getValueCName(condition.getName()), labels, duration);
     }
 

diff --git a/...er-core/src/main/java/org/apache/skywalking/oap/server/core/query/RecordQueryService.java b/...er-core/src/main/java/org/apache/skywalking/oap/server/core/query/RecordQueryService.java
@@ -33,7 +33,7 @@
 import java.io.IOException;
 import java.util.List;
 
-import static org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTrace.TRACE_CONTEXT;
+import static org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext.TRACE_CONTEXT;
 
 public class RecordQueryService implements Service {
     private final ModuleManager moduleManager;
@@ -56,7 +56,7 @@ private List<Record> invokeReadRecords(RecordCondition condition, Duration durat
         if (!condition.senseScope() || !condition.getParentEntity().isValid()) {
             return Collections.emptyList();
         }
-        return getRecordsQueryDAO().readRecords(
+        return getRecordsQueryDAO().readRecordsDebuggable(
             condition, ValueColumnMetadata.INSTANCE.getValueCName(condition.getName()), duration);
     }
 

diff --git a/...ver-core/src/main/java/org/apache/skywalking/oap/server/core/query/TraceQueryService.java b/...ver-core/src/main/java/org/apache/skywalking/oap/server/core/query/TraceQueryService.java
@@ -53,6 +53,8 @@
 import org.apache.skywalking.oap.server.core.query.type.Trace;
 import org.apache.skywalking.oap.server.core.query.type.TraceBrief;
 import org.apache.skywalking.oap.server.core.query.type.TraceState;
+import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingSpan;
+import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext;
 import org.apache.skywalking.oap.server.core.storage.StorageModule;
 import org.apache.skywalking.oap.server.core.storage.query.ISpanAttachedEventQueryDAO;
 import org.apache.skywalking.oap.server.core.storage.query.ITraceQueryDAO;
@@ -61,6 +63,7 @@
 import org.apache.skywalking.oap.server.library.util.CollectionUtils;
 
 import static java.util.Objects.nonNull;
+import static org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext.TRACE_CONTEXT;
 
 public class TraceQueryService implements Service {
 
@@ -107,18 +110,59 @@ public TraceBrief queryBasicTraces(final String serviceId,
                                        final Pagination paging,
                                        final Duration duration,
                                        final List<Tag> tags) throws IOException {
-        PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(paging);
-
-        return getTraceQueryDAO().queryBasicTraces(
-            duration, minTraceDuration, maxTraceDuration, serviceId, serviceInstanceId, endpointId,
-            traceId, page.getLimit(), page.getFrom(), traceState, queryOrder, tags
-        );
+        DebuggingTraceContext traceContext = TRACE_CONTEXT.get();
+        DebuggingSpan span = null;
+        try {
+            if (traceContext != null) {
+                StringBuilder msg = new StringBuilder();
+                span = traceContext.createSpan("Query Service: queryBasicTraces");
+                msg.append("Condition: ServiceId: ").append(serviceId)
+                   .append(", ServiceInstanceId: ").append(serviceInstanceId)
+                   .append(", EndpointId: ").append(endpointId)
+                   .append(", TraceId: ").append(traceId)
+                   .append(", MinTraceDuration: ").append(minTraceDuration)
+                   .append(", MaxTraceDuration: ").append(maxTraceDuration)
+                   .append(", TraceState: ").append(traceState)
+                   .append(", QueryOrder: ").append(queryOrder)
+                   .append(", Pagination: ").append(paging)
+                   .append(", Duration: ").append(duration)
+                   .append(", Tags: ").append(tags);
+                span.setMsg(msg.toString());
+            }
+            PaginationUtils.Page page = PaginationUtils.INSTANCE.exchange(paging);
+
+            return getTraceQueryDAO().queryBasicTracesDebuggable(
+                duration, minTraceDuration, maxTraceDuration, serviceId, serviceInstanceId, endpointId,
+                traceId, page.getLimit(), page.getFrom(), traceState, queryOrder, tags
+            );
+        } finally {
+            if (traceContext != null && span != null) {
+                traceContext.stopSpan(span);
+            }
+        }
     }
 
     public Trace queryTrace(final String traceId) throws IOException {
+        DebuggingTraceContext traceContext = TRACE_CONTEXT.get();
+        DebuggingSpan span = null;
+        try {
+            if (traceContext != null) {
+                StringBuilder msg = new StringBuilder();
+                span = traceContext.createSpan("Query Service: queryTrace");
+                msg.append("Condition: TraceId: ").append(traceId);
+            }
+            return invokeQueryTrace(traceId);
+        } finally {
+            if (traceContext != null && span != null) {
+                traceContext.stopSpan(span);
+            }
+        }
+    }
+
+    private Trace invokeQueryTrace(final String traceId) throws IOException {
         Trace trace = new Trace();
 
-        List<SegmentRecord> segmentRecords = getTraceQueryDAO().queryByTraceId(traceId);
+        List<SegmentRecord> segmentRecords = getTraceQueryDAO().queryByTraceIdDebuggable(traceId);
         if (segmentRecords.isEmpty()) {
             trace.getSpans().addAll(getTraceQueryDAO().doFlexibleTraceQuery(traceId));
         } else {
@@ -147,8 +191,8 @@ public Trace queryTrace(final String traceId) throws IOException {
 
         if (CollectionUtils.isNotEmpty(sortedSpans)) {
             final List<SpanAttachedEventRecord> spanAttachedEvents = getSpanAttachedEventQueryDAO().
-                querySpanAttachedEvents(SpanAttachedEventTraceType.SKYWALKING, Arrays.asList(traceId));
-            appendAttachedEventsToSpan(sortedSpans, spanAttachedEvents);
+                querySpanAttachedEventsDebuggable(SpanAttachedEventTraceType.SKYWALKING, Arrays.asList(traceId));
+            appendAttachedEventsToSpanDebuggable(sortedSpans, spanAttachedEvents);
         }
 
         trace.getSpans().clear();
@@ -273,6 +317,22 @@ private void findChildren(List<Span> spans, Span parentSpan, List<Span> children
         });
     }
 
+    private void appendAttachedEventsToSpanDebuggable(List<Span> spans, List<SpanAttachedEventRecord> events) throws InvalidProtocolBufferException {
+        DebuggingTraceContext traceContext = DebuggingTraceContext.TRACE_CONTEXT.get();
+        DebuggingSpan debuggingSpan = null;
+        try {
+            if (traceContext != null) {
+                debuggingSpan = traceContext.createSpan("Query Service : appendAttachedEventsToSpan");
+            }
+            appendAttachedEventsToSpan(spans, events);
+        } finally {
+            if (traceContext != null && debuggingSpan != null) {
+                traceContext.stopSpan(debuggingSpan);
+
+            }
+        }
+    }
+
     private void appendAttachedEventsToSpan(List<Span> spans, List<SpanAttachedEventRecord> events) throws InvalidProtocolBufferException {
         if (CollectionUtils.isEmpty(events)) {
             return;

diff --git a/.../src/main/java/org/apache/skywalking/oap/server/core/query/input/TraceQueryCondition.java b/.../src/main/java/org/apache/skywalking/oap/server/core/query/input/TraceQueryCondition.java
@@ -21,13 +21,15 @@
 import java.util.List;
 import lombok.Getter;
 import lombok.Setter;
+import lombok.ToString;
 import org.apache.skywalking.oap.server.core.analysis.manual.searchtag.Tag;
 import org.apache.skywalking.oap.server.core.query.type.Pagination;
 import org.apache.skywalking.oap.server.core.query.type.QueryOrder;
 import org.apache.skywalking.oap.server.core.query.type.TraceState;
 
 @Getter
 @Setter
+@ToString
 public class TraceQueryCondition {
     private String serviceId;
     private String serviceInstanceId;

diff --git a/...erver-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Pagination.java b/...erver-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Pagination.java
@@ -22,11 +22,13 @@
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.Setter;
+import lombok.ToString;
 
 @Getter
 @Setter
 @NoArgsConstructor
 @AllArgsConstructor
+@ToString
 public class Pagination {
     private int pageNum;
     private int pageSize;

diff --git a/...ver/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Trace.java b/...ver/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/Trace.java
@@ -21,10 +21,15 @@
 import java.util.ArrayList;
 import java.util.List;
 import lombok.Getter;
+import lombok.Setter;
+import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTrace;
 
 @Getter
 public class Trace {
     private final List<Span> spans;
+    //For OAP internal query debugging
+    @Setter
+    private DebuggingTrace debuggingTrace;
 
     public Trace() {
         this.spans = new ArrayList<>();

diff --git a/...erver-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/TraceBrief.java b/...erver-core/src/main/java/org/apache/skywalking/oap/server/core/query/type/TraceBrief.java
@@ -22,10 +22,15 @@
 
 import java.util.ArrayList;
 import java.util.List;
+import lombok.Setter;
+import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTrace;
 
 @Getter
 public class TraceBrief {
     private final List<BasicTrace> traces;
+    //For OAP internal query debugging
+    @Setter
+    private DebuggingTrace debuggingTrace;
 
     public TraceBrief() {
         this.traces = new ArrayList<>();

diff --git a/.../main/java/org/apache/skywalking/oap/server/core/query/type/debugging/DebuggingTrace.java b/.../main/java/org/apache/skywalking/oap/server/core/query/type/debugging/DebuggingTrace.java
@@ -25,7 +25,6 @@
 
 @Getter
 public class DebuggingTrace {
-    public final static ThreadLocal<DebuggingTraceContext> TRACE_CONTEXT = new ThreadLocal<>();
     private final String traceId;
     private final String condition;
     private final long startTime;

diff --git a/...ava/org/apache/skywalking/oap/server/core/query/type/debugging/DebuggingTraceContext.java b/...ava/org/apache/skywalking/oap/server/core/query/type/debugging/DebuggingTraceContext.java
@@ -23,6 +23,7 @@
 
 @Getter
 public class DebuggingTraceContext {
+    public final static ThreadLocal<DebuggingTraceContext> TRACE_CONTEXT = new ThreadLocal<>();
     private final DebuggingTrace execTrace;
     private final Stack<DebuggingSpan> spanStack = new Stack<>();
     private int spanIdGenerator;

diff --git a/...c/main/java/org/apache/skywalking/oap/server/core/storage/query/IAggregationQueryDAO.java b/...c/main/java/org/apache/skywalking/oap/server/core/storage/query/IAggregationQueryDAO.java
@@ -24,6 +24,8 @@
 import org.apache.skywalking.oap.server.core.query.input.TopNCondition;
 import org.apache.skywalking.oap.server.core.query.type.KeyValue;
 import org.apache.skywalking.oap.server.core.query.type.SelectedRecord;
+import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingSpan;
+import org.apache.skywalking.oap.server.core.query.type.debugging.DebuggingTraceContext;
 import org.apache.skywalking.oap.server.core.storage.DAO;
 
 /**
@@ -32,6 +34,25 @@
  * @since 8.0.0
  */
 public interface IAggregationQueryDAO extends DAO {
+    default List<SelectedRecord> sortMetricsDebuggable(final TopNCondition condition,
+                                                       final String valueColumnName,
+                                                       final Duration duration,
+                                                       final List<KeyValue> additionalConditions) throws IOException {
+        DebuggingTraceContext traceContext = DebuggingTraceContext.TRACE_CONTEXT.get();
+        DebuggingSpan span = null;
+        try {
+            if (traceContext != null) {
+                span = traceContext.createSpan("Query Dao: sortMetrics");
+                span.setMsg("Condition: TopNCondition: " + condition + ", ValueColumnName: " + valueColumnName + ", Duration: " + duration + ", AdditionalConditions: " + additionalConditions);
+            }
+            return sortMetrics(condition, valueColumnName, duration, additionalConditions);
+        } finally {
+            if (traceContext != null && span != null) {
+                traceContext.stopSpan(span);
+            }
+        }
+    }
+
     List<SelectedRecord> sortMetrics(TopNCondition condition,
                                      String valueColumnName,
                                      Duration duration,