Fix karton metrics generation (#44)

* Fix karton metrics generation
* Bump prometheus

karton/dashboard/app.py

@@ -4,6 +4,7 @@
 import textwrap
 from collections import defaultdict
 from datetime import datetime
+from itertools import product
 from operator import itemgetter
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Tuple
@@ -189,17 +190,24 @@ def varz():
     """ Update and get prometheus metrics """
 
     state = KartonState(karton.backend)
-    for _key, gauge in karton_tasks._metrics.items():
-        gauge.set(0)
+
+    # Clear the metrics completely to account for disappearing queues
+    karton_tasks.clear()
+    karton_replicas.clear()
 
     for queue in state.queues.values():
         safe_name = re.sub("[^a-z0-9]", "_", queue.bind.identity.lower())
         task_infos = defaultdict(int)
         for task in queue.tasks:
             task_infos[(safe_name, task.priority, task.status)] += 1
 
+        # set the default of active queues to 0 to avoid gaps in graphs
+        for (priority, status) in product(TaskPriority, TaskState):
+            karton_tasks.labels(safe_name, priority.value, status.value).set(0)
+
         for (name, priority, status), count in task_infos.items():
             karton_tasks.labels(name, priority.value, status.value).set(count)
+
         replicas = len(state.replicas[queue.bind.identity])
         karton_replicas.labels(safe_name, queue.bind.version).set(replicas)
 

requirements.txt

@@ -1,4 +1,4 @@
 Flask==1.1.1
 karton-core>=4.2.0,<5.0.0
 mistune==0.8.4
-prometheus_client==0.9.0
+prometheus_client==0.11.0