Merge pull request #9 from khazic/main

Update openai_api_server.py
2024-06-05 12:43:02 +08:00 · 2024-06-05 12:43:02 +08:00 · 44a55818ef
parent d95f131b03 205b7db3cc
commit 44a55818ef
1 changed files with 1 additions and 1 deletions
--- a/basic_demo/openai_api_server.py
+++ b/basic_demo/openai_api_server.py
@ -533,7 +533,7 @@ if __name__ == "__main__":
        tensor_parallel_size=1,
        dtype="bfloat16",
        trust_remote_code=True,
-        gpu_memory_utilization=0.3,
+        gpu_memory_utilization=0.9,
        enforce_eager=True,
        worker_use_ray=True,
        engine_use_ray=False,