p83571920
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

check python interpreter

This commit is contained in:
zetongzhao 2021-09-14 16:13:51 -04:00
parent ae9f09cc1a
commit 860810b9c4
3 changed files with 12 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
mindspore/ccsrc/minddata/dataset/engine/serdes.cc
View File

@ -17,6 +17,7 @@
#include <stack>
#include "minddata/dataset/engine/serdes.h"
#include "minddata/dataset/core/pybind_support.h"
#include "utils/file_utils.h"
#include "utils/utils.h"
@ -232,7 +233,9 @@ Status Serdes::ConstructTensorOps(nlohmann::json json_obj, std::vector<std::shar
std::vector<std::shared_ptr<TensorOperation>> output;
for (nlohmann::json item : json_obj) {
if (item.find("python_module") != item.end()) {
RETURN_IF_NOT_OK(PyFuncOp::from_json(item, result));
if (Py_IsInitialized()) {
RETURN_IF_NOT_OK(PyFuncOp::from_json(item, result));
}
} else {
CHECK_FAIL_RETURN_UNEXPECTED(item.find("tensor_op_name") != item.end(), "Failed to find tensor_op_name");
CHECK_FAIL_RETURN_UNEXPECTED(item.find("tensor_op_params") != item.end(), "Failed to find tensor_op_params");

7
tests/ut/cpp/dataset/deserialize_test.cc
View File

@ -527,3 +527,10 @@ TEST_F(MindDataTestDeserialize, TestDeserializeConcatAlbumFlickr) {
ds = std::make_shared<ConcatNode>(datasets, sampler, children_flag_and_nums, children_start_end_index);
compare_dataset(ds);
}
TEST_F(MindDataTestDeserialize, TestDeserializePyFunc) {
MS_LOG(INFO) << "Doing MindDataTestDeserialize-PyFunc.";
std::shared_ptr<DatasetNode> ds1;
ASSERT_OK(Serdes::Deserialize("./data/dataset/tf_file_dataset/pyvision_dataset_pipeline.json", &ds1));
EXPECT_NE(ds1, nullptr);
}

1
tests/ut/data/dataset/tf_file_dataset/pyvision_dataset_pipeline.json Normal file
View File

@ -0,0 +1 @@
{"callback":[],"children":[{"callback":[],"children":[{"children":[],"columns_list":["image","label"],"dataset_files":["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"],"num_parallel_workers":8,"num_samples":0,"num_shards":1,"op_type":"TFRecordDataset","schema_file_path":"../data/dataset/test_tf_file_3_images/datasetSchema.json","shard_equal_rows":false,"shard_id":0,"shuffle":0,"shuffle_files":false,"shuffle_global":false}],"input_columns":["image"],"num_parallel_workers":8,"op_type":"Map","operations":[{"python_module":"mindspore.dataset.transforms.py_transforms","tensor_op_name":"Compose","tensor_op_params":{"random":false,"transforms":[{"python_module":"mindspore.dataset.vision.py_transforms","tensor_op_name":"Decode","tensor_op_params":{"random":false}},{"python_module":"mindspore.dataset.vision.py_transforms","tensor_op_name":"CenterCrop","tensor_op_params":{"random":false,"size":[32,32]}},{"python_module":"mindspore.dataset.vision.py_transforms","tensor_op_name":"ToTensor","tensor_op_params":{"output_type":"float32","random":false}}]}}],"output_columns":["image"],"project_columns":[]}],"input_columns":["image"],"num_parallel_workers":8,"op_type":"Map","operations":[{"python_module":"mindspore.dataset.transforms.py_transforms","tensor_op_name":"RandomApply","tensor_op_params":{"prob":0.5,"transforms":[{"python_module":"mindspore.dataset.vision.py_transforms","tensor_op_name":"RandomColorAdjust","tensor_op_params":{"brightness":[1,1],"contrast":[1,1],"hue":[0,0],"saturation":[1,1]}},{"python_module":"mindspore.dataset.vision.py_transforms","tensor_op_name":"FiveCrop","tensor_op_params":{"random":false,"size":1}},{"python_module":"mindspore.dataset.vision.py_transforms","tensor_op_name":"Grayscale","tensor_op_params":{"num_output_channels":1,"random":false}},{"python_module":"mindspore.dataset.transforms.py_transforms","tensor_op_name":"OneHotOp","tensor_op_params":{"num_classes":1,"random":false,"smoothing_rate":0.0}}]}}],"output_columns":["image"],"project_columns":[]}