Fix error when tp > 1 (vllm-project#2644)

Co-authored-by: zhaoyang-star <zhao.yang16@zte.com.cn>
deepinfra · Jan 31, 2024 · 5c1a1f8 · 5c1a1f8
1 parent deed1ff
commit 5c1a1f8
Showing 1 changed file with 2 additions and 3 deletions.
diff --git a/vllm/engine/llm_engine.py b/vllm/engine/llm_engine.py
@@ -240,7 +240,6 @@ def _init_workers_ray(self, placement_group: "PlacementGroup",
         model_config = copy.deepcopy(self.model_config)
         parallel_config = copy.deepcopy(self.parallel_config)
         scheduler_config = copy.deepcopy(self.scheduler_config)
-        cache_config = copy.deepcopy(self.cache_config)
 
         for rank, (worker, (node_id,
                             _)) in enumerate(zip(self.workers,
@@ -256,7 +255,7 @@ def _init_workers_ray(self, placement_group: "PlacementGroup",
                     rank,
                     distributed_init_method,
                     lora_config=self.lora_config,
-                    cache_config=cache_config,
+                    kv_cache_dtype=self.cache_config.cache_dtype,
                 ))
 
         driver_rank = 0
@@ -269,7 +268,7 @@ def _init_workers_ray(self, placement_group: "PlacementGroup",
             driver_rank,
             distributed_init_method,
             lora_config=self.lora_config,
-            cache_config=cache_config,
+            kv_cache_dtype=self.cache_config.cache_dtype,
             is_driver_worker=True,
         )