efort
/
glm4
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #655 from zhipuch/main 

correct compute_metrics function
This commit is contained in:
Yuxuan.Zhang 2024-11-25 22:38:22 +08:00 committed by GitHub
parent b052da77b1 7a14e5d352
commit c23abb0c59
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
finetune_demo/finetune.py
View File

@ -374,8 +374,8 @@ def compute_metrics(eval_preds: EvalPrediction, tokenizer):
batched_label_ids[batched_label_ids==-100] = tokenizer.pad_token_id batched_label_ids[batched_label_ids==-100] = tokenizer.pad_token_id
metrics_dct = {'rouge-1': [], 'rouge-2': [], 'rouge-l': [], 'bleu-4': []} metrics_dct = {'rouge-1': [], 'rouge-2': [], 'rouge-l': [], 'bleu-4': []}
for pred_ids, label_ids in zip(batched_pred_ids, batched_label_ids): for pred_ids, label_ids in zip(batched_pred_ids, batched_label_ids):
pred_txt = tokenizer.decode(pred_ids).strip() pred_txt = tokenizer.decode(pred_ids, skip_special_tokens=True).strip()
label_txt = tokenizer.decode(label_ids).strip() label_txt = tokenizer.decode(label_ids, skip_special_tokens=True).strip()
pred_tokens = list(jieba.cut(pred_txt)) pred_tokens = list(jieba.cut(pred_txt))
label_tokens = list(jieba.cut(label_txt)) label_tokens = list(jieba.cut(label_txt))
rouge = Rouge() rouge = Rouge()

4
finetune_demo/finetune_vision.py
View File

@ -420,8 +420,8 @@ def compute_metrics(eval_preds: EvalPrediction, tokenizer):
batched_label_ids[batched_label_ids==-100] = tokenizer.pad_token_id batched_label_ids[batched_label_ids==-100] = tokenizer.pad_token_id
metrics_dct = {'rouge-1': [], 'rouge-2': [], 'rouge-l': [], 'bleu-4': []} metrics_dct = {'rouge-1': [], 'rouge-2': [], 'rouge-l': [], 'bleu-4': []}
for pred_ids, label_ids in zip(batched_pred_ids, batched_label_ids): for pred_ids, label_ids in zip(batched_pred_ids, batched_label_ids):
pred_txt = tokenizer.decode(pred_ids).strip() pred_txt = tokenizer.decode(pred_ids, skip_special_tokens=True).strip()
label_txt = tokenizer.decode(label_ids).strip() label_txt = tokenizer.decode(label_ids, skip_special_tokens=True).strip()
pred_tokens = list(jieba.cut(pred_txt)) pred_tokens = list(jieba.cut(pred_txt))
label_tokens = list(jieba.cut(label_txt)) label_tokens = list(jieba.cut(label_txt))
rouge = Rouge() rouge = Rouge()