From 860810b9c40bb42bd38c27f9342798c96fefeaf2 Mon Sep 17 00:00:00 2001 From: zetongzhao Date: Tue, 14 Sep 2021 16:13:51 -0400 Subject: [PATCH] check python interpreter --- mindspore/ccsrc/minddata/dataset/engine/serdes.cc | 5 ++++- tests/ut/cpp/dataset/deserialize_test.cc | 7 +++++++ .../dataset/tf_file_dataset/pyvision_dataset_pipeline.json | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 tests/ut/data/dataset/tf_file_dataset/pyvision_dataset_pipeline.json diff --git a/mindspore/ccsrc/minddata/dataset/engine/serdes.cc b/mindspore/ccsrc/minddata/dataset/engine/serdes.cc index 99db8bbc4e6..2ffc2218fe7 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/serdes.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/serdes.cc @@ -17,6 +17,7 @@ #include #include "minddata/dataset/engine/serdes.h" +#include "minddata/dataset/core/pybind_support.h" #include "utils/file_utils.h" #include "utils/utils.h" @@ -232,7 +233,9 @@ Status Serdes::ConstructTensorOps(nlohmann::json json_obj, std::vector> output; for (nlohmann::json item : json_obj) { if (item.find("python_module") != item.end()) { - RETURN_IF_NOT_OK(PyFuncOp::from_json(item, result)); + if (Py_IsInitialized()) { + RETURN_IF_NOT_OK(PyFuncOp::from_json(item, result)); + } } else { CHECK_FAIL_RETURN_UNEXPECTED(item.find("tensor_op_name") != item.end(), "Failed to find tensor_op_name"); CHECK_FAIL_RETURN_UNEXPECTED(item.find("tensor_op_params") != item.end(), "Failed to find tensor_op_params"); diff --git a/tests/ut/cpp/dataset/deserialize_test.cc b/tests/ut/cpp/dataset/deserialize_test.cc index e876b7c3c09..b76cfb44342 100644 --- a/tests/ut/cpp/dataset/deserialize_test.cc +++ b/tests/ut/cpp/dataset/deserialize_test.cc @@ -527,3 +527,10 @@ TEST_F(MindDataTestDeserialize, TestDeserializeConcatAlbumFlickr) { ds = std::make_shared(datasets, sampler, children_flag_and_nums, children_start_end_index); compare_dataset(ds); } + +TEST_F(MindDataTestDeserialize, TestDeserializePyFunc) { + MS_LOG(INFO) << "Doing MindDataTestDeserialize-PyFunc."; + std::shared_ptr ds1; + ASSERT_OK(Serdes::Deserialize("./data/dataset/tf_file_dataset/pyvision_dataset_pipeline.json", &ds1)); + EXPECT_NE(ds1, nullptr); +} \ No newline at end of file diff --git a/tests/ut/data/dataset/tf_file_dataset/pyvision_dataset_pipeline.json b/tests/ut/data/dataset/tf_file_dataset/pyvision_dataset_pipeline.json new file mode 100644 index 00000000000..7f13754314e --- /dev/null +++ b/tests/ut/data/dataset/tf_file_dataset/pyvision_dataset_pipeline.json @@ -0,0 +1 @@ +{"callback":[],"children":[{"callback":[],"children":[{"children":[],"columns_list":["image","label"],"dataset_files":["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"],"num_parallel_workers":8,"num_samples":0,"num_shards":1,"op_type":"TFRecordDataset","schema_file_path":"../data/dataset/test_tf_file_3_images/datasetSchema.json","shard_equal_rows":false,"shard_id":0,"shuffle":0,"shuffle_files":false,"shuffle_global":false}],"input_columns":["image"],"num_parallel_workers":8,"op_type":"Map","operations":[{"python_module":"mindspore.dataset.transforms.py_transforms","tensor_op_name":"Compose","tensor_op_params":{"random":false,"transforms":[{"python_module":"mindspore.dataset.vision.py_transforms","tensor_op_name":"Decode","tensor_op_params":{"random":false}},{"python_module":"mindspore.dataset.vision.py_transforms","tensor_op_name":"CenterCrop","tensor_op_params":{"random":false,"size":[32,32]}},{"python_module":"mindspore.dataset.vision.py_transforms","tensor_op_name":"ToTensor","tensor_op_params":{"output_type":"float32","random":false}}]}}],"output_columns":["image"],"project_columns":[]}],"input_columns":["image"],"num_parallel_workers":8,"op_type":"Map","operations":[{"python_module":"mindspore.dataset.transforms.py_transforms","tensor_op_name":"RandomApply","tensor_op_params":{"prob":0.5,"transforms":[{"python_module":"mindspore.dataset.vision.py_transforms","tensor_op_name":"RandomColorAdjust","tensor_op_params":{"brightness":[1,1],"contrast":[1,1],"hue":[0,0],"saturation":[1,1]}},{"python_module":"mindspore.dataset.vision.py_transforms","tensor_op_name":"FiveCrop","tensor_op_params":{"random":false,"size":1}},{"python_module":"mindspore.dataset.vision.py_transforms","tensor_op_name":"Grayscale","tensor_op_params":{"num_output_channels":1,"random":false}},{"python_module":"mindspore.dataset.transforms.py_transforms","tensor_op_name":"OneHotOp","tensor_op_params":{"num_classes":1,"random":false,"smoothing_rate":0.0}}]}}],"output_columns":["image"],"project_columns":[]} \ No newline at end of file