efort
/
glm4
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix #60 and #51

This commit is contained in:
zR 2024-06-06 10:14:15 +08:00
parent 3f26ccc208
commit 1683d673d2
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
basic_demo/openai_api_server.py
View File

@ -314,7 +314,7 @@ async def create_chat_completion(request: ChatCompletionRequest):
if request.stream: if request.stream:
predict_stream_generator = predict_stream(request.model, gen_params) predict_stream_generator = predict_stream(request.model, gen_params)
output = await anext(predict_stream_generator) output = await anext(predict_stream_generator)
if not output and 'get_' in output: if output:
return EventSourceResponse(predict_stream_generator, media_type="text/event-stream") return EventSourceResponse(predict_stream_generator, media_type="text/event-stream")
logger.debug(f"First result output：\n{output}") logger.debug(f"First result output：\n{output}")
@ -535,7 +535,6 @@ if __name__ == "__main__":
engine_args = AsyncEngineArgs( engine_args = AsyncEngineArgs(
model=MODEL_PATH, model=MODEL_PATH,
tokenizer=MODEL_PATH, tokenizer=MODEL_PATH,
tokenizer_mode="slow",
tensor_parallel_size=1, tensor_parallel_size=1,
dtype="bfloat16", dtype="bfloat16",
trust_remote_code=True, trust_remote_code=True,