修复了 batch map 时默认批大小(1000)导致的 offset overflow 问题
This commit is contained in:
parent
3dec01c6d5
commit
b45326e9ea
|
@ -236,6 +236,8 @@ class DataManager(object):
|
||||||
batched=batched,
|
batched=batched,
|
||||||
remove_columns=remove_columns,
|
remove_columns=remove_columns,
|
||||||
num_proc=self._num_proc,
|
num_proc=self._num_proc,
|
||||||
|
batch_size=500, # Dynamically adjust according to mem size
|
||||||
|
writer_batch_size=500
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue