Fix queue metrics when there are no jobs in it

Signed-off-by: Xuzheng Chang <changxuzheng@huawei.com>
Monokaix · May 13, 2024 · 84ec14c · 84ec14c
1 parent 7605027
commit 84ec14c
Show file tree

Hide file tree

Showing 2 changed files with 47 additions and 14 deletions.
diff --git a/pkg/scheduler/plugins/capacity/capacity.go b/pkg/scheduler/plugins/capacity/capacity.go
@@ -172,6 +172,33 @@ func (cp *capacityPlugin) OnSessionOpen(ssn *framework.Session) {
 			attr.name, attr.deserved, attr.realCapability, attr.allocated, attr.request, attr.elastic, attr.share)
 	}
 
+	// Record metrics
+	for queueID, queueInfo := range ssn.Queues {
+		queue := ssn.Queues[queueID]
+		if attr, ok := cp.queueOpts[queueID]; ok {
+			metrics.UpdateQueueDeserved(attr.name, attr.deserved.MilliCPU, attr.deserved.Memory)
+			metrics.UpdateQueueAllocated(attr.name, attr.allocated.MilliCPU, attr.allocated.Memory)
+			metrics.UpdateQueueRequest(attr.name, attr.request.MilliCPU, attr.request.Memory)
+			metrics.UpdateQueuePodGroupInqueueCount(attr.name, queue.Queue.Status.Inqueue)
+			metrics.UpdateQueuePodGroupPendingCount(attr.name, queue.Queue.Status.Pending)
+			metrics.UpdateQueuePodGroupRunningCount(attr.name, queue.Queue.Status.Running)
+			metrics.UpdateQueuePodGroupUnknownCount(attr.name, queue.Queue.Status.Unknown)
+			continue
+		}
+		deservedCPU, deservedMem := 0.0, 0.0
+		if queue.Queue.Spec.Deserved != nil {
+			deservedCPU = float64(queue.Queue.Spec.Deserved.Cpu().MilliValue())
+			deservedMem = float64(queue.Queue.Spec.Deserved.Memory().Value())
+		}
+		metrics.UpdateQueueDeserved(queueInfo.Name, deservedCPU, deservedMem)
+		metrics.UpdateQueueAllocated(queueInfo.Name, 0, 0)
+		metrics.UpdateQueueRequest(queueInfo.Name, 0, 0)
+		metrics.UpdateQueuePodGroupInqueueCount(queueInfo.Name, 0)
+		metrics.UpdateQueuePodGroupPendingCount(queueInfo.Name, 0)
+		metrics.UpdateQueuePodGroupRunningCount(queueInfo.Name, 0)
+		metrics.UpdateQueuePodGroupUnknownCount(queueInfo.Name, 0)
+	}
+
 	ssn.AddQueueOrderFn(cp.Name(), func(l, r interface{}) int {
 		lv := l.(*api.QueueInfo)
 		rv := r.(*api.QueueInfo)
@@ -223,6 +250,7 @@ func (cp *capacityPlugin) OnSessionOpen(ssn *framework.Session) {
 		attr := cp.queueOpts[queue.UID]
 
 		overused := attr.deserved.LessEqual(attr.allocated, api.Zero)
+		metrics.UpdateQueueOverused(attr.name, overused)
 		if overused {
 			klog.V(3).Infof("Queue <%v> can not reclaim, deserved <%v>, allocated <%v>, share <%v>",
 				queue.Name, attr.deserved, attr.allocated, attr.share)
@@ -290,6 +318,7 @@ func (cp *capacityPlugin) OnSessionOpen(ssn *framework.Session) {
 			job := ssn.Jobs[event.Task.Job]
 			attr := cp.queueOpts[job.Queue]
 			attr.allocated.Add(event.Task.Resreq)
+			metrics.UpdateQueueAllocated(attr.name, attr.allocated.MilliCPU, attr.allocated.Memory)
 
 			cp.updateShare(attr)
 
@@ -300,6 +329,7 @@ func (cp *capacityPlugin) OnSessionOpen(ssn *framework.Session) {
 			job := ssn.Jobs[event.Task.Job]
 			attr := cp.queueOpts[job.Queue]
 			attr.allocated.Sub(event.Task.Resreq)
+			metrics.UpdateQueueAllocated(attr.name, attr.allocated.MilliCPU, attr.allocated.Memory)
 
 			cp.updateShare(attr)
 

diff --git a/pkg/scheduler/plugins/proportion/proportion.go b/pkg/scheduler/plugins/proportion/proportion.go
@@ -160,22 +160,25 @@ func (pp *proportionPlugin) OnSessionOpen(ssn *framework.Session) {
 			attr.name, attr.allocated.String(), attr.request.String(), attr.inqueue.String(), attr.elastic.String())
 	}
 
+	// Record metrics
 	for queueID, queueInfo := range ssn.Queues {
-		if _, ok := pp.queueOpts[queueID]; !ok {
-			metrics.UpdateQueueAllocated(queueInfo.Name, 0, 0)
+		if attr, ok := pp.queueOpts[queueID]; ok {
+			metrics.UpdateQueueAllocated(attr.name, attr.allocated.MilliCPU, attr.allocated.Memory)
+			metrics.UpdateQueueRequest(attr.name, attr.request.MilliCPU, attr.request.Memory)
+			metrics.UpdateQueueWeight(attr.name, attr.weight)
+			queue := ssn.Queues[attr.queueID]
+			metrics.UpdateQueuePodGroupInqueueCount(attr.name, queue.Queue.Status.Inqueue)
+			metrics.UpdateQueuePodGroupPendingCount(attr.name, queue.Queue.Status.Pending)
+			metrics.UpdateQueuePodGroupRunningCount(attr.name, queue.Queue.Status.Running)
+			metrics.UpdateQueuePodGroupUnknownCount(attr.name, queue.Queue.Status.Unknown)
+			continue
 		}
-	}
-
-	// Record metrics
-	for _, attr := range pp.queueOpts {
-		metrics.UpdateQueueAllocated(attr.name, attr.allocated.MilliCPU, attr.allocated.Memory)
-		metrics.UpdateQueueRequest(attr.name, attr.request.MilliCPU, attr.request.Memory)
-		metrics.UpdateQueueWeight(attr.name, attr.weight)
-		queue := ssn.Queues[attr.queueID]
-		metrics.UpdateQueuePodGroupInqueueCount(attr.name, queue.Queue.Status.Inqueue)
-		metrics.UpdateQueuePodGroupPendingCount(attr.name, queue.Queue.Status.Pending)
-		metrics.UpdateQueuePodGroupRunningCount(attr.name, queue.Queue.Status.Running)
-		metrics.UpdateQueuePodGroupUnknownCount(attr.name, queue.Queue.Status.Unknown)
+		metrics.UpdateQueueAllocated(queueInfo.Name, 0, 0)
+		metrics.UpdateQueueRequest(queueInfo.Name, 0, 0)
+		metrics.UpdateQueuePodGroupInqueueCount(queueInfo.Name, 0)
+		metrics.UpdateQueuePodGroupPendingCount(queueInfo.Name, 0)
+		metrics.UpdateQueuePodGroupRunningCount(queueInfo.Name, 0)
+		metrics.UpdateQueuePodGroupUnknownCount(queueInfo.Name, 0)
 	}
 
 	remaining := pp.totalResource.Clone()