efort
/
glm4
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

correct compute_metrics function

This commit is contained in:
zhipuch 2024-11-19 09:35:07 +08:00
parent a7dbfa5ada
commit 7a14e5d352
2 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
finetune_demo/finetune.py
View File

@ -374,8 +374,8 @@ def compute_metrics(eval_preds: EvalPrediction, tokenizer):
batched_label_ids[batched_label_ids==-100] = tokenizer.pad_token_id
metrics_dct = {'rouge-1': [], 'rouge-2': [], 'rouge-l': [], 'bleu-4': []}
for pred_ids, label_ids in zip(batched_pred_ids, batched_label_ids):
pred_txt = tokenizer.decode(pred_ids).strip()
label_txt = tokenizer.decode(label_ids).strip()
pred_txt = tokenizer.decode(pred_ids, skip_special_tokens=True).strip()
label_txt = tokenizer.decode(label_ids, skip_special_tokens=True).strip()
pred_tokens = list(jieba.cut(pred_txt))
label_tokens = list(jieba.cut(label_txt))
rouge = Rouge()

4
finetune_demo/finetune_vision.py
View File

@ -420,8 +420,8 @@ def compute_metrics(eval_preds: EvalPrediction, tokenizer):
batched_label_ids[batched_label_ids==-100] = tokenizer.pad_token_id
metrics_dct = {'rouge-1': [], 'rouge-2': [], 'rouge-l': [], 'bleu-4': []}
for pred_ids, label_ids in zip(batched_pred_ids, batched_label_ids):
pred_txt = tokenizer.decode(pred_ids).strip()
label_txt = tokenizer.decode(label_ids).strip()
pred_txt = tokenizer.decode(pred_ids, skip_special_tokens=True).strip()
label_txt = tokenizer.decode(label_ids, skip_special_tokens=True).strip()
pred_tokens = list(jieba.cut(pred_txt))
label_tokens = list(jieba.cut(label_txt))
rouge = Rouge()