udpate example

Xu-Kai · Nov 16, 2023 · 2a493b6 · 2a493b6
1 parent 5446fb7
commit 2a493b6
Show file tree

Hide file tree

Showing 20 changed files with 161 additions and 548 deletions.
diff --git a/colossalai/inference/hybridengine/engine.py b/colossalai/inference/hybridengine/engine.py
@@ -14,7 +14,14 @@
 
 PP_AXIS, TP_AXIS = 0, 1
 
-_supported_models = ["LlamaForCausalLM", "BloomForCausalLM", "LlamaGPTQForCausalLM", "SmoothLlamaForCausalLM", "ChatGLMForConditionalGeneration"]
+_supported_models = [
+    "LlamaForCausalLM",
+    "BloomForCausalLM",
+    "LlamaGPTQForCausalLM",
+    "SmoothLlamaForCausalLM",
+    "ChatGLMForConditionalGeneration",
+]
+
 
 class CaiInferEngine:
     """
@@ -161,7 +168,7 @@ def _shardformer(self, model, model_policy, stage_manager, tp_group):
             enable_flash_attention=False,
             enable_jit_fused=False,
             enable_sequence_parallelism=False,
-            quant=self.quant,
+            extra_kwargs={"quant": self.quant},
         )
         shardformer = ShardFormer(shard_config=shardconfig)
         shard_model, _ = shardformer.optimize(model, model_policy)

diff --git a/.../ray_serve/Colossal_Inference_rayserve.py → .../ray_serve/Colossal_Inference_rayserve.py b/.../ray_serve/Colossal_Inference_rayserve.py → .../ray_serve/Colossal_Inference_rayserve.py
diff --git a/...les/inference/serving/ray_serve/README.md → ...acy/inference/serving/ray_serve/README.md b/...les/inference/serving/ray_serve/README.md → ...acy/inference/serving/ray_serve/README.md
diff --git a/...ference/serving/ray_serve/send_request.py → ...ference/serving/ray_serve/send_request.py b/...ference/serving/ray_serve/send_request.py → ...ference/serving/ray_serve/send_request.py
diff --git a/...erence/serving/ray_serve/send_requests.py → ...erence/serving/ray_serve/send_requests.py b/...erence/serving/ray_serve/send_requests.py → ...erence/serving/ray_serve/send_requests.py
diff --git a/examples/inference/serving/test_ci.sh → ...salai/legacy/inference/serving/test_ci.sh b/examples/inference/serving/test_ci.sh → ...salai/legacy/inference/serving/test_ci.sh
diff --git a/...torch_serve/Colossal_Inference_Handler.py → ...torch_serve/Colossal_Inference_Handler.py b/...torch_serve/Colossal_Inference_Handler.py → ...torch_serve/Colossal_Inference_Handler.py
diff --git a/...s/inference/serving/torch_serve/README.md → ...y/inference/serving/torch_serve/README.md b/...s/inference/serving/torch_serve/README.md → ...y/inference/serving/torch_serve/README.md
diff --git a/...nce/serving/torch_serve/config.properties → ...nce/serving/torch_serve/config.properties b/...nce/serving/torch_serve/config.properties → ...nce/serving/torch_serve/config.properties
diff --git a/...nce/serving/torch_serve/docker/Dockerfile → ...nce/serving/torch_serve/docker/Dockerfile b/...nce/serving/torch_serve/docker/Dockerfile → ...nce/serving/torch_serve/docker/Dockerfile
diff --git a/...nce/serving/torch_serve/model-config.yaml → ...nce/serving/torch_serve/model-config.yaml b/...nce/serving/torch_serve/model-config.yaml → ...nce/serving/torch_serve/model-config.yaml
diff --git a/...rence/serving/torch_serve/sample_text.txt → ...rence/serving/torch_serve/sample_text.txt b/...rence/serving/torch_serve/sample_text.txt → ...rence/serving/torch_serve/sample_text.txt
diff --git a/examples/inference/bench_bloom.py b/examples/inference/bench_bloom.py
diff --git a/examples/inference/bench_chatglm2.py b/examples/inference/bench_chatglm2.py
diff --git a/examples/inference/bench_llama.py b/examples/inference/bench_llama.py