add candidate loop state panel

libp2p · Mar 22, 2023 · 99249d0 · 99249d0
1 parent 9f17ffb
commit 99249d0
Show file tree

Hide file tree

Showing 4 changed files with 230 additions and 31 deletions.
diff --git a/dashboards/autorelay/autorelay.json b/dashboards/autorelay/autorelay.json
@@ -91,19 +91,6 @@
         "x": 0,
         "y": 0
       },
-      "id": 24,
-      "panels": [],
-      "title": "Scheduled Work",
-      "type": "row"
-    },
-    {
-      "collapsed": false,
-      "gridPos": {
-        "h": 1,
-        "w": 24,
-        "x": 0,
-        "y": 1
-      },
       "id": 16,
       "panels": [],
       "title": "Status",
@@ -157,7 +144,7 @@
         "h": 5,
         "w": 24,
         "x": 0,
-        "y": 2
+        "y": 1
       },
       "id": 2,
       "options": {
@@ -197,7 +184,7 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 7
+        "y": 6
       },
       "id": 18,
       "panels": [],
@@ -234,7 +221,7 @@
         "h": 8,
         "w": 6,
         "x": 0,
-        "y": 8
+        "y": 7
       },
       "id": 4,
       "options": {
@@ -393,7 +380,7 @@
         "h": 8,
         "w": 18,
         "x": 6,
-        "y": 8
+        "y": 7
       },
       "id": 10,
       "options": {
@@ -434,13 +421,107 @@
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 16
+        "y": 15
       },
       "id": 20,
       "panels": [],
       "title": "Candidates",
       "type": "row"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "fixed"
+          },
+          "custom": {
+            "fillOpacity": 70,
+            "lineWidth": 0,
+            "spanNulls": false
+          },
+          "mappings": [
+            {
+              "options": {
+                "0": {
+                  "color": "purple",
+                  "index": 0,
+                  "text": "peer source rate limited"
+                },
+                "1": {
+                  "color": "blue",
+                  "index": 1,
+                  "text": "waiting on peer chan"
+                },
+                "2": {
+                  "color": "green",
+                  "index": 2,
+                  "text": "waiting for trigger"
+                },
+                "3": {
+                  "color": "light-yellow",
+                  "index": 3,
+                  "text": "stopped"
+                }
+              },
+              "type": "value"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 24,
+        "x": 0,
+        "y": 16
+      },
+      "id": 28,
+      "options": {
+        "alignValue": "left",
+        "legend": {
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "mergeValues": true,
+        "rowHeight": 0.9,
+        "showValue": "auto",
+        "tooltip": {
+          "mode": "single",
+          "sort": "none"
+        }
+      },
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "exemplar": true,
+          "expr": "libp2p_autorelay_candidate_loop_state",
+          "instant": false,
+          "legendFormat": "state",
+          "range": true,
+          "refId": "A"
+        }
+      ],
+      "title": "Candidate Loop State",
+      "type": "state-timeline"
+    },
     {
       "datasource": {
         "type": "prometheus",
@@ -467,9 +548,9 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 6,
-        "x": 2,
-        "y": 17
+        "w": 5,
+        "x": 1,
+        "y": 22
       },
       "id": 6,
       "options": {
@@ -557,9 +638,9 @@
       },
       "gridPos": {
         "h": 8,
-        "w": 6,
-        "x": 8,
-        "y": 17
+        "w": 5,
+        "x": 6,
+        "y": 22
       },
       "id": 12,
       "options": {
@@ -629,8 +710,8 @@
       "gridPos": {
         "h": 8,
         "w": 6,
-        "x": 14,
-        "y": 17
+        "x": 11,
+        "y": 22
       },
       "id": 26,
       "options": {
@@ -664,13 +745,91 @@
       "title": "Next Old Candidate Check",
       "type": "stat"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "fixedColor": "purple",
+            "mode": "fixed"
+          },
+          "mappings": [
+            {
+              "options": {
+                "from": null,
+                "result": {
+                  "color": "blue",
+                  "index": 1,
+                  "text": "immediately"
+                },
+                "to": 0
+              },
+              "type": "range"
+            }
+          ],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 6,
+        "x": 17,
+        "y": 22
+      },
+      "id": 30,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "area",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": [
+            "lastNotNull"
+          ],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "pluginVersion": "9.4.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "round(libp2p_autorelay_scheduled_work_time{work_type=\"allowed peer source call\"} - time()) ",
+          "instant": true,
+          "legendFormat": "__auto",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "title": "Next Allowed Call to peer source",
+      "type": "stat"
+    },
     {
       "collapsed": false,
       "gridPos": {
         "h": 1,
         "w": 24,
         "x": 0,
-        "y": 25
+        "y": 30
       },
       "id": 22,
       "panels": [],
@@ -705,7 +864,7 @@
         "h": 8,
         "w": 6,
         "x": 0,
-        "y": 26
+        "y": 31
       },
       "id": 8,
       "options": {
@@ -816,7 +975,7 @@
         "h": 8,
         "w": 18,
         "x": 6,
-        "y": 26
+        "y": 31
       },
       "id": 14,
       "options": {
@@ -857,13 +1016,13 @@
     "list": []
   },
   "time": {
-    "from": "now-1m",
+    "from": "now-1h",
     "to": "now"
   },
   "timepicker": {},
   "timezone": "",
   "title": "libp2p Autorelay",
   "uid": "deQ_uf-4k",
-  "version": 3,
+  "version": 7,
   "weekStart": ""
 }
diff --git a/p2p/host/autorelay/metrics.go b/p2p/host/autorelay/metrics.go
@@ -71,6 +71,13 @@ var (
 		},
 		[]string{"type"},
 	)
+	candLoopState = prometheus.NewGauge(
+		prometheus.GaugeOpts{
+			Namespace: metricNamespace,
+			Name:      "candidate_loop_state",
+			Help:      "Candidate Loop State",
+		},
+	)
 
 	scheduledWorkTime = prometheus.NewGaugeVec(
 		prometheus.GaugeOpts{
@@ -98,11 +105,21 @@ var (
 		relayAddressesCount,
 		candidatesCircuitV2SupportTotal,
 		candidatesTotal,
+		candLoopState,
 		scheduledWorkTime,
 		desiredReservations,
 	}
 )
 
+type candidateLoopState int
+
+const (
+	peerSourceRateLimited candidateLoopState = iota
+	waitingOnPeerChan
+	waitingForTrigger
+	stopped
+)
+
 // MetricsTracer is the interface for tracking metrics for autorelay
 type MetricsTracer interface {
 	RelayFinderStatus(isActive bool)
@@ -116,6 +133,7 @@ type MetricsTracer interface {
 	CandidateChecked(supportsCircuitV2 bool)
 	CandidateAdded()
 	CandidateRemoved()
+	CandidateLoopState(state candidateLoopState)
 
 	ScheduledWorkUpdated(scheduledWork *scheduledWorkTimes)
 
@@ -218,6 +236,10 @@ func (mt *metricsTracer) CandidateRemoved() {
 	candidatesTotal.WithLabelValues(*tags...).Inc()
 }
 
+func (mt *metricsTracer) CandidateLoopState(state candidateLoopState) {
+	candLoopState.Set(float64(state))
+}
+
 func (mt *metricsTracer) ScheduledWorkUpdated(scheduledWork *scheduledWorkTimes) {
 	tags := metricshelper.GetStringSlice()
 	defer metricshelper.PutStringSlice(tags)
@@ -332,3 +354,9 @@ func (mt *wrappedMetricsTracer) DesiredReservations(cnt int) {
 		mt.mt.DesiredReservations(cnt)
 	}
 }
+
+func (mt *wrappedMetricsTracer) CandidateLoopState(state candidateLoopState) {
+	if mt.mt != nil {
+		mt.mt.CandidateLoopState(state)
+	}
+}