多线程的使用bug #15
Description
尝试使用多线程 sample识别任务--workers 2
结果异常报错如下：
Traceback (most recent call last):
File "/data/vjuicefs_ai_gpt_audio_train/public_data/blaudio_train/home/kangdong/UltraEval-Audio/audio_eval/audio_evals/eval_task.py", line 75, in _run
score, ans = self._eval(
File "/data/vjuicefs_ai_gpt_audio_train/public_data/blaudio_train/home/kangdong/UltraEval-Audio/audio_eval/audio_evals/eval_task.py", line 52, in _eval
output = self.predictor.inference(prompt)
File "/data/vjuicefs_ai_gpt_audio_train/public_data/blaudio_train/home/kangdong/UltraEval-Audio/audio_eval/audio_evals/models/model.py", line 33, in inference
return self._inference(prompt, **sample_params)
File "/data/vjuicefs_ai_gpt_audio_train/public_data/blaudio_train/home/kangdong/UltraEval-Audio/audio_eval/audio_evals/models/vivo_model.py", line 70, in _inference
res = self._model.chat(
File "/root/.cache/huggingface/modules/transformers_modules/MiniCPM-o-2_6/modeling_minicpmo.py", line 1132, in chat
res, outputs = self.generate(
File "/root/.cache/huggingface/modules/transformers_modules/MiniCPM-o-2_6/modeling_minicpmo.py", line 943, in generate
outputs = self._decode(model_inputs["inputs_embeds"], tokenizer,
File "/root/.cache/huggingface/modules/transformers_modules/MiniCPM-o-2_6/modeling_minicpmo.py", line 713, in _decode
outputs = self.llm.generate(
File "/usr/local/lib/python3.10/dist-packages/torch/utils/_contextlib.py", line 115, in decorate_context
return func(*args, **kwargs)
File "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py", line 2255, in generate
result = self._sample(
File "/usr/local/lib/python3.10/dist-packages/transformers/generation/utils.py", line 3300, in _sample
next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1)
RuntimeError: probability tensor contains either inf, nan or element < 0