sbjkx
/
mindspore
forked from OSSInnovation/mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

!6479 fix bucket_batch_by_length show pyfunc timeout 

Merge pull request !6479 from yanghaitao/yht_bucket_batch_ci
This commit is contained in:
mindspore-ci-bot 2020-09-19 09:30:43 +08:00 committed by Gitee
parent a79943ca98 da15495d7b
commit 1cfad93704
3 changed files with 6 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
mindspore/ccsrc/minddata/dataset/kernels/py_func_op.cc
View File

@ -50,14 +50,8 @@ Status PyFuncOp::Compute(const TensorRow &input, TensorRow *output) {
} else { } else {
ret_py_obj = this->py_func_ptr_(); ret_py_obj = this->py_func_ptr_();
} }
// Object is none if pyfunc timeout
if (ret_py_obj.is_none()) {
MS_LOG(INFO) << "Pyfunc execute time out";
goto TimeoutError;
}
if (output_type_ != DataType::DE_UNKNOWN) { if (output_type_ != DataType::DE_UNKNOWN) {
RETURN_IF_NOT_OK(CastOutput(ret_py_obj, output)); RETURN_IF_NOT_OK(CastOutput(ret_py_obj, output));
} else { } else {
if (py::isinstance<py::tuple>(ret_py_obj)) { if (py::isinstance<py::tuple>(ret_py_obj)) {
// In case of a n-m mapping, the return value will be a tuple of numpy arrays // In case of a n-m mapping, the return value will be a tuple of numpy arrays
@ -65,6 +59,10 @@ Status PyFuncOp::Compute(const TensorRow &input, TensorRow *output) {
// Iterate over two containers simultaneously for memory copy // Iterate over two containers simultaneously for memory copy
for (size_t i = 0; i < ret_py_tuple.size(); i++) { for (size_t i = 0; i < ret_py_tuple.size(); i++) {
py::object ret_py_ele = ret_py_tuple[i]; py::object ret_py_ele = ret_py_tuple[i];
// Object is none if pyfunc timeout
if (ret_py_ele.is_none()) {
goto TimeoutError;
}
if (!py::isinstance<py::array>(ret_py_ele)) { if (!py::isinstance<py::array>(ret_py_ele)) {
goto ShapeMisMatch; goto ShapeMisMatch;
} }

2
mindspore/dataset/engine/datasets.py
View File

@ -1999,7 +1999,7 @@ class _PythonCallable:
return result.get(60) return result.get(60)
except multiprocessing.TimeoutError: except multiprocessing.TimeoutError:
# Ensure c++ pyfunc threads exit normally if python sub-process is killed unnormally. # Ensure c++ pyfunc threads exit normally if python sub-process is killed unnormally.
return None return (None,)
except KeyboardInterrupt: except KeyboardInterrupt:
self.pool.terminate() self.pool.terminate()
self.pool.join() self.pool.join()

2
tests/ut/python/dataset/test_paddeddataset.py
View File

@ -504,7 +504,7 @@ def test_celeba_padded():
count = 0 count = 0
for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True): for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True):
count = count + 1 count = count + 1
assert count == 2 assert count == 4
if __name__ == '__main__': if __name__ == '__main__':