dataframe = dataframe.map(self.map_fn2, num_proc=16) 加载失败

[tcxia]

在dataframe = dataframe.map(self.map_fn2, num_proc=1)加载失败，报错信息如下File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/datasets/arrow_dataset.py", line 562, in wrapper out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/datasets/arrow_dataset.py", line 3332, in map for rank, done, content in iflatmap_unordered( File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/datasets/utils/py_utils.py", line 626, in iflatmap_unordered [async_result.get(timeout=0.05) for async_result in async_results] File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/datasets/utils/py_utils.py", line 626, in <listcomp> [async_result.get(timeout=0.05) for async_result in async_results] File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/multiprocess/pool.py", line 774, in get raise self._value File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/multiprocess/pool.py", line 540, in _handle_tasks put(task) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/multiprocess/connection.py", line 209, in send self._send_bytes(_ForkingPickler.dumps(obj)) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/multiprocess/reduction.py", line 54, in dumps cls(buf, protocol, *args, **kwds).dump(obj) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 420, in dump StockPickler.dump(self, obj) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 487, in dump self.save(obj) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 414, in save StockPickler.save(self, obj, save_persistent_id) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 560, in save f(self, obj) # Call unbound method with explicit self File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 902, in save_tuple save(element) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 414, in save StockPickler.save(self, obj, save_persistent_id) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 560, in save f(self, obj) # Call unbound method with explicit self File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 887, in save_tuple save(element) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 414, in save StockPickler.save(self, obj, save_persistent_id) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 560, in save f(self, obj) # Call unbound method with explicit self File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 1217, in save_module_dict StockPickler.save_dict(pickler, obj) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 972, in save_dict self._batch_setitems(obj.items()) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 998, in _batch_setitems save(v) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 414, in save StockPickler.save(self, obj, save_persistent_id) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 603, in save self.save_reduce(obj=obj, *rv) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 692, in save_reduce save(args) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 414, in save StockPickler.save(self, obj, save_persistent_id) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 560, in save f(self, obj) # Call unbound method with explicit self File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 887, in save_tuple save(element) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 414, in save StockPickler.save(self, obj, save_persistent_id) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 560, in save f(self, obj) # Call unbound method with explicit self File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 1985, in save_function _save_with_postproc(pickler, (_create_function, ( File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 1112, in _save_with_postproc pickler._batch_setitems(iter(source.items())) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 998, in _batch_setitems save(v) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 414, in save StockPickler.save(self, obj, save_persistent_id) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 560, in save f(self, obj) # Call unbound method with explicit self File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 1832, in save_type _save_with_postproc(pickler, (_create_type, ( File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 1098, in _save_with_postproc pickler.save_reduce(*reduction, obj=obj) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 692, in save_reduce save(args) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 414, in save StockPickler.save(self, obj, save_persistent_id) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 560, in save f(self, obj) # Call unbound method with explicit self File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 902, in save_tuple save(element) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 414, in save StockPickler.save(self, obj, save_persistent_id) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 560, in save f(self, obj) # Call unbound method with explicit self File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 1217, in save_module_dict StockPickler.save_dict(pickler, obj) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 972, in save_dict self._batch_setitems(obj.items()) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 998, in _batch_setitems save(v) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 414, in save StockPickler.save(self, obj, save_persistent_id) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 560, in save f(self, obj) # Call unbound method with explicit self File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 1985, in save_function _save_with_postproc(pickler, (_create_function, ( File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 1112, in _save_with_postproc pickler._batch_setitems(iter(source.items())) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 998, in _batch_setitems save(v) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/site-packages/dill/_dill.py", line 414, in save StockPickler.save(self, obj, save_persistent_id) File "/opt/nas/p/conda/envs/tcxia_verl_agent/lib/python3.10/pickle.py", line 578, in save rv = reduce(self.proto) TypeError: cannot pickle '_contextvars.ContextVar' object

[LeoYao123]

遇到了相同的问题，实测去掉num_proc能跑