efort
/
glm4
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request from sixsixcoder/main 

support vllm 0.6.3
This commit is contained in:
Yuxuan.Zhang 2024-11-14 18:41:32 +08:00 committed by GitHub
parent 4446f60195 476a066830
commit b052da77b1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 1 additions and 4 deletions

5
basic_demo/vllm_cli_vision_demo.py
View File

@ -46,9 +46,6 @@ async def vllm_gen(messages: List[Dict[str, str]], top_p: float, temperature: fl
"temperature": temperature, "temperature": temperature,
"top_p": top_p, "top_p": top_p,
"top_k": -1, "top_k": -1,
"use_beam_search": False,
"length_penalty": 1,
"early_stopping": False,
"ignore_eos": False, "ignore_eos": False,
"max_tokens": max_dec_len, "max_tokens": max_dec_len,
"logprobs": None, "logprobs": None,
@ -58,7 +55,7 @@ async def vllm_gen(messages: List[Dict[str, str]], top_p: float, temperature: fl
} }
sampling_params = SamplingParams(**params_dict) sampling_params = SamplingParams(**params_dict)
async for output in engine.generate(inputs=inputs, sampling_params=sampling_params, request_id=f"{time.time()}"): async for output in engine.generate(prompt=inputs, sampling_params=sampling_params, request_id=f"{time.time()}"):
yield output.outputs[0].text yield output.outputs[0].text