Add more Prometheus metrics

vllm/core/scheduler.py

@@ -136,7 +136,7 @@ def abort_seq_group(self, request_id: Union[str, Iterable[str]]) -> None:
             for seq_group in state_queue:
                 if not request_ids:
                     # Using 'break' here may add two extra iterations,
-                    # but is acceptable to reduce complexity .
+                    # but is acceptable to reduce complexity.
                     break
                 if seq_group.request_id in request_ids:
                     # Appending aborted group into pending list.

vllm/engine/llm_engine.py

@@ -1,5 +1,5 @@
 import copy
-from collections import defaultdict
+from collections import defaultdict, Counter as CollectionsCounter
 import os
 import time
 from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Tuple,
@@ -845,19 +845,44 @@ def _get_stats(self,
         # Iteration stats if we have scheduler output.
         num_prompt_tokens = 0
         num_generation_tokens = 0
+        num_prompt_tokens_lst = []
+        num_generation_tokens_lst = []
+        max_tokens = []
+        request_n = []
         time_to_first_tokens = []
         time_per_output_tokens = []
         time_e2e_requests = []
+        finished_reason_counter = CollectionsCounter()
         if scheduler_outputs is not None:
             prompt_run = scheduler_outputs.prompt_run
 
-            # Number of Tokens.
+            # Number of Tokens
             if prompt_run:
                 num_prompt_tokens = scheduler_outputs.num_batched_tokens
+                num_prompt_tokens_lst = [
+                    len(seq_group.prompt_token_ids)
+                    for seq_group in scheduler_outputs.scheduled_seq_groups
+                ]
             else:
                 num_generation_tokens = scheduler_outputs.num_batched_tokens
+                num_generation_tokens_lst = [
+                    seq.get_output_len()
+                    for seq_group in scheduler_outputs.scheduled_seq_groups
+                    for seq in seq_group.get_finished_seqs()
+                ]
 
-            # Latency Timings.
+            # Sampling Params
+            if prompt_run:
+                max_tokens = [
+                    seq_group.sampling_params.max_tokens
+                    for seq_group in scheduler_outputs.scheduled_seq_groups
+                ]
+                request_n = [
+                    seq_group.sampling_params.n
+                    for seq_group in scheduler_outputs.scheduled_seq_groups
+                ]
+
+            # Latency Timings
             time_last_iters = []
             for seq_group in scheduler_outputs.scheduled_seq_groups:
                 # Time since last token. (n.b. updates seq_group.last_token_time)
@@ -869,15 +894,29 @@ def _get_stats(self,
             time_to_first_tokens = time_last_iters if prompt_run else []
             time_per_output_tokens = [] if prompt_run else time_last_iters
 
+            # Finished Requests
+            for seq_group in scheduler_outputs.scheduled_seq_groups:
+                if not seq_group.is_finished():
+                    continue
+                finished_reason_counter += CollectionsCounter([
+                    SequenceStatus.get_finished_reason(seq.status)
+                    for seq in seq_group.get_finished_seqs()
+                ])
+
         return Stats(
             now=now,
             num_running=num_running,
             num_swapped=num_swapped,
             num_waiting=num_waiting,
             gpu_cache_usage=gpu_cache_usage,
             cpu_cache_usage=cpu_cache_usage,
+            finished_reason_counter=finished_reason_counter,
             num_prompt_tokens=num_prompt_tokens,
             num_generation_tokens=num_generation_tokens,
+            num_prompt_tokens_lst=num_prompt_tokens_lst,
+            num_generation_tokens_lst=num_generation_tokens_lst,
+            max_tokens=max_tokens,
+            request_n=request_n,
             time_to_first_tokens=time_to_first_tokens,
             time_per_output_tokens=time_per_output_tokens,
             time_e2e_requests=time_e2e_requests,

vllm/engine/metrics.py

@@ -5,6 +5,7 @@
 import numpy as np
 from typing import List
 from dataclasses import dataclass
+from collections import Counter as CollectionsCounter
 
 logger = init_logger(__name__)
 
@@ -28,6 +29,8 @@ def add_global_metrics_labels(**kwargs):
                                 "Number of prefill tokens processed.")
 counter_generation_tokens = Counter("vllm:generation_tokens_total",
                                     "Number of generation tokens processed.")
+counter_request_success = Counter("vllm:request_success",
+                                  "Count of successfully processed requests.")
 
 gauge_scheduler_running = Gauge(
     "vllm:num_requests_running",
@@ -44,6 +47,22 @@ def add_global_metrics_labels(**kwargs):
     "vllm:cpu_cache_usage_perc",
     "CPU KV-cache usage. 1 means 100 percent usage.")
 
+histogram_request_prompt_tokens = Histogram(
+    "vllm:request_prompt_tokens",
+    "Number of prefill tokens processed.",
+    buckets=[
+        1, 2, 5, 10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000,
+        20_000, 50_000, 100_000
+    ],
+)
+histogram_request_generation_tokens = Histogram(
+    "vllm:request_generation_tokens",
+    "Number of generation tokens processed.",
+    buckets=[
+        1, 2, 5, 10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000,
+        20_000, 50_000, 100_000
+    ],
+)
 histogram_time_to_first_token = Histogram(
     "vllm:time_to_first_token_seconds",
     "Histogram of time to first token in seconds.",
@@ -61,6 +80,20 @@ def add_global_metrics_labels(**kwargs):
     "vllm:e2e_request_latency_seconds",
     "Histogram of end to end request latency in seconds.",
     buckets=[1.0, 2.5, 5.0, 10.0, 15.0, 20.0, 30.0, 40.0, 50.0, 60.0])
+
+histogram_max_tokens = Histogram(
+    "vllm:request_max_tokens",
+    "Histogram of the max_tokens request parameter.",
+    buckets=[
+        1, 2, 5, 10, 20, 50, 100, 200, 500, 1_000, 2_000, 5_000, 10_000,
+        20_000, 50_000, 100_000
+    ],
+)
+histogram_request_n = Histogram(
+    "vllm:request_n",
+    "Histogram of the n request parameter.",
+    buckets=[1, 2, 5, 10, 20],
+)
 # end-metrics-definitions
 
 
@@ -77,8 +110,13 @@ class Stats:
     cpu_cache_usage: float
 
     # Raw stats from last model iteration.
+    finished_reason_counter: CollectionsCounter[str, int]
     num_prompt_tokens: int
     num_generation_tokens: int
+    num_prompt_tokens_lst: List[int]
+    num_generation_tokens_lst: List[int]
+    max_tokens: List[int]
+    request_n: List[int]
     time_to_first_tokens: List[float]
     time_per_output_tokens: List[float]
     time_e2e_requests: List[float]
@@ -115,6 +153,26 @@ def _log_prometheus(self, stats: Stats) -> None:
         counter_prompt_tokens.add(labels, stats.num_prompt_tokens)
         counter_generation_tokens.add(labels, stats.num_generation_tokens)
 
+        # Add to request counters.
+        for finished_reason, count in stats.finished_reason_counter.items():
+            counter_request_success.add(
+                {
+                    **labels,
+                    "finished_reason": finished_reason,
+                }, count)
+
+        # Observe number of tokens in histograms.
+        for val in stats.num_prompt_tokens_lst:
+            histogram_request_prompt_tokens.observe(labels, val)
+        for val in stats.num_generation_tokens_lst:
+            histogram_request_generation_tokens.observe(labels, val)
+
+        # Observe sampling params in histograms.
+        for val in stats.max_tokens:
+            histogram_max_tokens.observe(labels, val)
+        for n in stats.request_n:
+            histogram_request_n.observe(labels, n)
+
         # Observe request level latencies in histograms.
         for ttft in stats.time_to_first_tokens:
             histogram_time_to_first_token.observe(labels, ttft)