修复了 batch map 时默认批大小(1000)导致的 offset overflow 问题

2024-07-03 16:18:52 +08:00 · 2024-07-03 16:18:52 +08:00 · b45326e9ea
parent 3dec01c6d5
commit b45326e9ea
1 changed files with 2 additions and 0 deletions
--- a/finetune_demo/finetune_vision.py
+++ b/finetune_demo/finetune_vision.py
@ -236,6 +236,8 @@ class DataManager(object):
            batched=batched,
            remove_columns=remove_columns,
            num_proc=self._num_proc,
+            batch_size=500,    # Dynamically adjust according to mem size
+            writer_batch_size=500
        )