From 1136ee3ffd6f59f914ceb820b0993d63901b08c7 Mon Sep 17 00:00:00 2001 From: Cathy Wong Date: Wed, 25 May 2022 11:57:26 -0400 Subject: [PATCH] [MD] Transform Unification Feature - ToTensor op: Remove Python implementation, regenerate golden files --- .../dataset/kernels/ir/image/bindings.cc | 4 +- .../ccsrc/minddata/dataset/engine/serdes.cc | 7 +- .../ccsrc/minddata/dataset/engine/serdes.h | 1 + .../dataset/kernels/image/to_tensor_op.cc | 2 +- .../dataset/kernels/image/to_tensor_op.h | 6 +- .../dataset/kernels/ir/vision/to_tensor_ir.cc | 20 +- .../dataset/kernels/ir/vision/to_tensor_ir.h | 4 +- .../mindspore/dataset/vision/transforms.py | 13 +- .../golden/autocontrast_01_result_py.npz | Bin 607 -> 603 bytes .../dataset/golden/cut_out_01_c_result.npz | Bin 644 -> 640 bytes .../dataset/golden/cut_out_02_c_result.npz | Bin 640 -> 640 bytes .../golden/equalize_01_result_py_unified.npz | Bin 709 -> 709 bytes .../dataset/golden/five_crop_01_result.npz | Bin 644 -> 640 bytes .../golden/invert_01_result_py_unified.npz | Bin 709 -> 709 bytes .../linear_transformation_01_result.npz | Bin 644 -> 640 bytes .../dataset/golden/normalize_01_c_result.npz | Bin 644 -> 640 bytes .../golden/normalize_01_to_pil_result.npz | Bin 644 -> 640 bytes .../golden/normalize_02_to_pil_result.npz | Bin 644 -> 640 bytes .../golden/normalize_03_to_pil_result.npz | Bin 644 -> 640 bytes .../golden/normalize_04_to_pil_result.npz | Bin 644 -> 640 bytes .../data/dataset/golden/pad_01_c_result.npz | Bin 644 -> 640 bytes .../data/dataset/golden/pad_01_py_result.npz | Bin 644 -> 640 bytes .../golden/random_affine_01_result.npz | Bin 644 -> 640 bytes .../dataset/golden/random_apply_01_result.npz | Bin 644 -> 640 bytes .../dataset/golden/random_color_01_result.npz | Bin 713 -> 709 bytes .../random_color_adjust_01_c_result.npz | Bin 644 -> 640 bytes .../random_color_adjust_01_py_result.npz | Bin 644 -> 640 bytes .../golden/random_crop_01_py_result.npz | Bin 644 -> 640 bytes .../golden/random_crop_02_py_result.npz | Bin 644 -> 640 bytes .../golden/random_crop_03_py_result.npz | Bin 644 -> 640 bytes .../golden/random_crop_05_py_result.npz | Bin 644 -> 640 bytes .../golden/random_crop_07_py_result.npz | Bin 644 -> 640 bytes .../golden/random_crop_08_py_result.npz | Bin 644 -> 640 bytes .../random_crop_and_resize_01_c_result.npz | Bin 644 -> 640 bytes .../random_crop_and_resize_01_py_result.npz | Bin 644 -> 640 bytes .../random_crop_and_resize_02_c_result.npz | Bin 644 -> 640 bytes .../random_crop_and_resize_02_py_result.npz | Bin 644 -> 640 bytes .../random_crop_and_resize_03_c_result.npz | Bin 644 -> 640 bytes .../random_crop_and_resize_03_py_result.npz | Bin 644 -> 640 bytes .../golden/random_erasing_01_result.npz | Bin 644 -> 640 bytes .../golden/random_grayscale_01_result.npz | Bin 644 -> 640 bytes .../golden/random_grayscale_02_result.npz | Bin 644 -> 640 bytes .../golden/random_horizontal_01_py_result.npz | Bin 644 -> 640 bytes .../golden/random_lighting_py_01_result.npz | Bin 709 -> 709 bytes .../dataset/golden/random_order_01_result.npz | Bin 644 -> 640 bytes .../golden/random_rotation_01_c_result.npz | Bin 644 -> 640 bytes .../golden/random_rotation_01_py_result.npz | Bin 817 -> 813 bytes .../golden/random_sharpness_py_01_result.npz | Bin 713 -> 709 bytes .../golden/random_vertical_01_py_result.npz | Bin 644 -> 640 bytes .../dataset/golden/ten_crop_01_result.npz | Bin 644 -> 640 bytes .../data/dataset/golden/to_pil_01_result.npz | Bin 644 -> 640 bytes .../data/dataset/golden/to_pil_02_result.npz | Bin 644 -> 640 bytes tests/ut/python/dataset/test_five_crop.py | 3 +- tests/ut/python/dataset/test_random_crop.py | 4 +- .../ut/python/dataset/test_serdes_dataset.py | 365 ++++++++++++++++-- tests/ut/python/dataset/test_ten_crop.py | 3 +- .../dataset_deprecated/test_serdes_dataset.py | 254 ++++++++++++ 57 files changed, 615 insertions(+), 71 deletions(-) create mode 100644 tests/ut/python/dataset_deprecated/test_serdes_dataset.py diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/ir/image/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/ir/image/bindings.cc index baa05ca53c4..a1b2cb63002 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/ir/image/bindings.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/kernels/ir/image/bindings.cc @@ -719,8 +719,8 @@ PYBIND_REGISTER(ToTensorOperation, 1, ([](const py::module *m) { (void) py::class_>( *m, "ToTensorOperation") - .def(py::init([](const std::string &data_type) { - auto totensor = std::make_shared(data_type); + .def(py::init([](const std::string &output_type) { + auto totensor = std::make_shared(output_type); THROW_IF_ERROR(totensor->ValidateParams()); return totensor; })); diff --git a/mindspore/ccsrc/minddata/dataset/engine/serdes.cc b/mindspore/ccsrc/minddata/dataset/engine/serdes.cc index 20acaf8845f..316f084b082 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/serdes.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/serdes.cc @@ -260,7 +260,9 @@ Status Serdes::ConstructTensorOps(nlohmann::json json_obj, std::vector> tmp_res; + RETURN_IF_NOT_OK(PyFuncOp::from_json(item, &tmp_res)); + output.insert(output.end(), tmp_res.begin(), tmp_res.end()); } else { LOG_AND_RETURN_STATUS_SYNTAX_ERROR( "Python module is not initialized or Pyfunction is not supported on this platform."); @@ -275,9 +277,9 @@ Status Serdes::ConstructTensorOps(nlohmann::json json_obj, std::vector &input, std::shared_ptr *output) { IO_CHECK(input, output); // Rescale and convert HWC to CHW format - return ToTensor(input, output, data_type_); + return ToTensor(input, output, output_type_); } } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/to_tensor_op.h b/mindspore/ccsrc/minddata/dataset/kernels/image/to_tensor_op.h index d288e30d6a2..a8dd75f1f61 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/to_tensor_op.h +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/to_tensor_op.h @@ -28,9 +28,9 @@ namespace mindspore { namespace dataset { class ToTensorOp : public TensorOp { public: - explicit ToTensorOp(const DataType &data_type) : data_type_(data_type) {} + explicit ToTensorOp(const DataType &output_type) : output_type_(output_type) {} - explicit ToTensorOp(const std::string &data_type) { data_type_ = DataType(data_type); } + explicit ToTensorOp(const std::string &output_type) { output_type_ = DataType(output_type); } ~ToTensorOp() override = default; @@ -39,7 +39,7 @@ class ToTensorOp : public TensorOp { std::string Name() const override { return kToTensorOp; } private: - DataType data_type_; + DataType output_type_; }; } // namespace dataset } // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/to_tensor_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/to_tensor_ir.cc index d3966906ab8..b9234939b91 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/to_tensor_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/to_tensor_ir.cc @@ -23,9 +23,9 @@ namespace mindspore { namespace dataset { namespace vision { -ToTensorOperation::ToTensorOperation(const std::string &data_type) { - DataType temp_data_type(data_type); - data_type_ = temp_data_type; +ToTensorOperation::ToTensorOperation(const std::string &output_type) { + DataType temp_output_type(output_type); + output_type_ = temp_output_type; } ToTensorOperation::~ToTensorOperation() = default; @@ -33,26 +33,26 @@ ToTensorOperation::~ToTensorOperation() = default; std::string ToTensorOperation::Name() const { return kToTensorOperation; } Status ToTensorOperation::ValidateParams() { - if (data_type_ == DataType::DE_UNKNOWN) { - std::string err_msg = "ToTensor: Invalid data type"; + if (output_type_ == DataType::DE_UNKNOWN) { + std::string err_msg = "ToTensor: Invalid data type for output_type parameter."; LOG_AND_RETURN_STATUS_SYNTAX_ERROR(err_msg); } return Status::OK(); } -std::shared_ptr ToTensorOperation::Build() { return std::make_shared(data_type_); } +std::shared_ptr ToTensorOperation::Build() { return std::make_shared(output_type_); } Status ToTensorOperation::to_json(nlohmann::json *out_json) { nlohmann::json args; - args["data_type"] = data_type_.ToString(); + args["output_type"] = output_type_.ToString(); *out_json = args; return Status::OK(); } Status ToTensorOperation::from_json(nlohmann::json op_params, std::shared_ptr *operation) { - RETURN_IF_NOT_OK(ValidateParamInJson(op_params, "data_type", kToTensorOperation)); - std::string data_type = op_params["data_type"]; - *operation = std::make_shared(data_type); + RETURN_IF_NOT_OK(ValidateParamInJson(op_params, "output_type", kToTensorOperation)); + std::string output_type = op_params["output_type"]; + *operation = std::make_shared(output_type); return Status::OK(); } } // namespace vision diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/to_tensor_ir.h b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/to_tensor_ir.h index af557ba6d14..9078cf8cba3 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/to_tensor_ir.h +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/vision/to_tensor_ir.h @@ -35,7 +35,7 @@ constexpr char kToTensorOperation[] = "ToTensor"; class ToTensorOperation : public TensorOperation { public: - explicit ToTensorOperation(const std::string &data_type); + explicit ToTensorOperation(const std::string &output_type); ~ToTensorOperation(); @@ -50,7 +50,7 @@ class ToTensorOperation : public TensorOperation { static Status from_json(nlohmann::json op_params, std::shared_ptr *operation); private: - DataType data_type_; + DataType output_type_; }; } // namespace vision } // namespace dataset diff --git a/mindspore/python/mindspore/dataset/vision/transforms.py b/mindspore/python/mindspore/dataset/vision/transforms.py index b3681353e64..c6143c06e2c 100644 --- a/mindspore/python/mindspore/dataset/vision/transforms.py +++ b/mindspore/python/mindspore/dataset/vision/transforms.py @@ -3456,18 +3456,7 @@ class ToTensor(TensorOperation, PyTensorOperation): output_type = nptype_to_detype(output_type) self.output_type = str(output_type) self.random = False - - def execute_py(self, img): - """ - Execute method. - - Args: - img (Union[PIL Image, numpy.ndarray]): PIL Image or numpy.ndarray to be type converted. - - Returns: - numpy.ndarray, converted numpy.ndarray with desired type. - """ - return util.to_tensor(img, self.output_type) + self.implementation = Implementation.C def parse(self): return cde.ToTensorOperation(self.output_type) diff --git a/tests/ut/data/dataset/golden/autocontrast_01_result_py.npz b/tests/ut/data/dataset/golden/autocontrast_01_result_py.npz index 252aee514b244ffa5fe6014d702857b89db0996d..239c9fed9001fe2f149a96ef4c84835bba785ec7 100644 GIT binary patch delta 188 zcmcc5a+}2@z?+#xgaHB+8N8~em0V_IU;trG1`&qDqM~>My}W`-Mg{?}AV}fmi6)Y5T!ZPY%g{8QazMkZNKxC17uGwHDjF@fv@0ISX~*Z=?k diff --git a/tests/ut/data/dataset/golden/cut_out_01_c_result.npz b/tests/ut/data/dataset/golden/cut_out_01_c_result.npz index ce8c3237989efb6fe253511775f26d1d079193d5..87b0cc9cced7576e192c8674badafbe0f5b6abe1 100644 GIT binary patch delta 95 zcmZo+ZD2JC@MdNaVSoTd2G^fkZvJ3oU;trG1`&qDqM~>My}W`-Mg{?}AV}eli6)>?hDC7VD delta 123 zcmZo*ZD17&@MdNaVSoTd2FZy*lQs&?VH9}Pc)H6rO2R?W<0yNG^48eN`xsRO7Pu;k zu1I%&aN_f4DY*t`waK3tRSeThc6-OQe^9sb%)VOF^UJePwWLrjDK)jEP(4X6z?+dt Mgc;TR$?i;+02o0fy#N3J diff --git a/tests/ut/data/dataset/golden/equalize_01_result_py_unified.npz b/tests/ut/data/dataset/golden/equalize_01_result_py_unified.npz index b40d4db7da2ca6c119ff71a8a1a0f159f901343a..50608da7e4f30f29a4cacfb6c09610b431d64925 100644 GIT binary patch delta 78 zcmX@gdX!Ztz?+#xgaHB+8G7`r*f$EzVH99B*>zdiT;Q-);KYVQk?a#E?_*TK6#T?E MjRmA-au1Uw02$sFC;$Ke delta 78 zcmX@gdX!Ztz?+#xgaHB+8CZ3*UTzec!zj>Gs}?2jdls8leagZMr(JGN-p8ndDfo$T M8Vg9vMy}W`-Mg{?}AV}eli6)-o*6D!qryH>(;bP p{st5@NLtZ&ecR__x@z03Lhe6znyken%L%u3asZPan=uo}765H+H;e!P delta 186 zcmZo*ZDBPD@MdNaVSoTdhFLK?cm82yU;trG1`&qDqM~>My}W`-Mg{?}AV}e#i6);| zycxV1Cu=aqLKy9g>H_)PBId#T7E_MgIQ&}HCA)m`K1LORc5zqcyRY7~stGhn#_ZBt zG5HgtirLfl|L-R{A38kAwW&B=uFShowWLrjDK)jEP(4X6z?+dtgc*yICi^hyv6(Z0 GTmS&I05#kI diff --git a/tests/ut/data/dataset/golden/invert_01_result_py_unified.npz b/tests/ut/data/dataset/golden/invert_01_result_py_unified.npz index a0c74bbb0262bc5ae2b0f157a052ee2ce8146250..17fd10ac2e2000d8a6ac6b7eebc9700618043f96 100644 GIT binary patch delta 78 zcmX@gdX!Ztz?+#xgaHB+8NNjCxwcVg4x>Q6UHP%)u__@7UM#AtY>D!d_c5wq3Vvdo M#sbnZxrfOT07Lv2xBvhE delta 78 zcmX@gdX!Ztz?+#xgaHB+8SZo6wA&~&hf$#8^_f?5`98m8vGe=(V#0pD$@>^pFa?T`AMOAE diff --git a/tests/ut/data/dataset/golden/linear_transformation_01_result.npz b/tests/ut/data/dataset/golden/linear_transformation_01_result.npz index ef54a007b599b36b0f3808f74cc1aaf1ae8ceb47..dc25c6ac3dff9c583536b227bef2e24ea9d6140b 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTd1_7(nRzDaS7(kelL4+Z(s3_h*FR!4IkwE|~2vYcCqRA(w z&W_1ijIki9n^9fByW-FGKgpS2otwUgupie^oqUKzFn};8g9t-nQBk~sUS2^ZBZB}~5Tx+WM3YY} z-VEN1lQkG)A&hoLb%Ab~m{UvUS+7~kK6_#HF2=o+_c5vn__@_S&MDft%>UD8n?G+^ zy(fQSR55$H?!Lj7TOViHoL_d}#>4YEg{mcmYDuZ7C57rqdI8>yOd`x!oHW^oNsrB( H3FHC*RXIC> diff --git a/tests/ut/data/dataset/golden/normalize_01_c_result.npz b/tests/ut/data/dataset/golden/normalize_01_c_result.npz index 9222f48571246b0cdc267ae357719196429e5094..28be8c06ac6c3c09ce228b17a1413c344fd57c1c 100644 GIT binary patch delta 95 zcmZo+ZD2JC@MdNaVSoTdhUrh23;bYYU;trG1`&qDqM~>My}W`-Mg{?}AV}eli6)My}W`-Mg{?}AV}e#i6);| eycxV1Cu=aqLKy9gdpO~GC;Kqzv6(Z0lmY;faTvb< diff --git a/tests/ut/data/dataset/golden/normalize_01_to_pil_result.npz b/tests/ut/data/dataset/golden/normalize_01_to_pil_result.npz index 9d794aa5f2331aca3728595469f9b43619c0922d..0a5a3ac633dae0c0509b6756dde766465383fbab 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhQA%_7X4skU;trG1`&qDqM~>My}W`-Mg{?}AV}eli6)KtZR#n0wzt29-A;>7m=q;ns9I8}mXw-WQmCG!7vRmvB*KivNt1n;^w`Xq HKrR3Pvcos2 diff --git a/tests/ut/data/dataset/golden/normalize_02_to_pil_result.npz b/tests/ut/data/dataset/golden/normalize_02_to_pil_result.npz index 82ab777c10b0f8d4276f955cbe17d58934ec7c0d..34fa77caeec99387db79d2b965475ed94efbc2b6 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhK_g1F+Ug?7(kelL4+Z(s3_h*FR!4IkwE|~2vYcCqRA(w z&W_1ijIki9n^9e0`vw8=dDsF>O+4{%Wc)Iq6ZS(jh?_*RE*uPM3@vaV=x1ohXcAg^V zGA4gwR543Pzj3_hkl$jl-BWuqc$sSpRZ9xhl2TJk3e}VJ0=yZSM3}KSX|fNK9-BE6 G$OQmoQ8x^v4+Xt ofPxXs>AA_PU-hn)T&$WNw&(g}Ehbq`xV4i5nDp3;nLxGx01PxPZ~y=R delta 186 zcmZo*ZDBPD@MdNaVSoTdhRqwEEdImDzyQLW3?dAPMMd!jdU*wvj0^%`L6E{f6HPv` zcr$o2PS#+Ig)rI~)dgl7XeUT}H=gdkT5WzW_}TNx`xsROrXPBmFg0uY>gQ#`xwhH4 zW|KcLs+ipflbda-I{(eBEk`HKp0ag9p=wE?T2g9iNuhd@UVt|vlL#{wCr$QY(ql7c H0=WPHA^kce diff --git a/tests/ut/data/dataset/golden/normalize_04_to_pil_result.npz b/tests/ut/data/dataset/golden/normalize_04_to_pil_result.npz index a51404c5c3829a1d0466031c5038ba6216786feb..cc6d2b0657d4a21cdc4f278f5d1bfbc2fe683575 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhD+am|NFtnzyQLW3?dAPMMd!jdU*wvj0^%`L6E{96HPub zb#_eFVvGe*-Hhr2Q7$32Ri!>$-?jKIcj#37oP3B;MPSFLkGpd$R%ic7*=~5`+*+l{ p-++ReW@Y)!;c0)bl$l;ywQ*(pWGyCHPPny`1DN#KjF~{T0034PH*o*} delta 186 zcmZo*ZDBPD@MdNaVSoTd2H!>{;XjNF3?R(OAi|JXR1|NZmse29$RGd~1S$M8(c}}0 zH-k6hWDUkx2&0`*UEtWX>34nJf0(AEyP;<3w5K;F?_*REVC4EW+xg{7@0L5a@^jAFzK-wGl7%=0EZD8D*ylh delta 99 zcmZo*ZDBPD@MdNaVSoTdhR!23=6@I&7(kelL4+Z(s3_h*FR!4IkwE|~2vYcGqRA%~ dZw7D1$r_BY5Jo%W9!|L4$v#YaZ01ZLr2sQW7Qp}j diff --git a/tests/ut/data/dataset/golden/pad_01_py_result.npz b/tests/ut/data/dataset/golden/pad_01_py_result.npz index 095257f847e31328065fa03f8f2021872f1a565e..1e3fdb8b98204b88afaef89aaad2b5f3f15e1054 100644 GIT binary patch delta 157 zcmZo+ZD2JC@MdNaVSoTdh6^v3YW-kjU;trG1`&qDqM~>My}W`-Mg{?}AV}eli6)*Pa>DguA@pRV0vba?Ob(5{;wQWMWk p{st7RWD=cwP-71d&s!OG?fqt3CTlUtalmYy9KfWMy}W`-Mg{?}AV}e#i6);| zycxV1Cu=aqLKy9g>H_6C8_lC8q!=&LJtG>pNW^9GK1LORZw{OOus;qxv*6a8R%PFg r{>h(!f-mP!I&x8Xx-RoGbH-B{Tf!!*G0AbjY@Y1Hq{n8?1hNPKDE~4Y diff --git a/tests/ut/data/dataset/golden/random_affine_01_result.npz b/tests/ut/data/dataset/golden/random_affine_01_result.npz index 85a887961d88bf0f1c60d8fba3c13fa6d80f3979..656b79ffcf01c3a5640cc8942bd41ceece562541 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhSPCdlz%WXFn};8g9t-nQBk~sUS2^ZBZB}~5Tx+OM3YZU zogI_47-K0}1TZTGS_7|4+OH!Px8+IMy}W`-Mg{?}AV}e#i6);| zycxV1Cu=aqLKy9g>H;6K5;S;RoXY>+x@?p5R54?H>y|NBM&Kgf^$?~!Bge^wswIVLNvWwNh3ZLq0p5&EBFtEvG}(tqkIkG3 G0}1s=Ip%OR{1Fw)S;0n-(!Yo_vT=Mc|Ou&*grfUE(6HM2R_fKifL_ o8&I$}`jgjVZ~Sti>eD3Ac7~0Fxe@F%!rZ04kX>iU0rr delta 186 zcmZo*ZDBPD@MdNaVSoTdhBFIgT>daJFn};8g9t-nQBk~sUS2^ZBZB}~5Tx+WM3YY} z-VEN1lQkG)A&hoLb%9rnr@L&UBpehyjlHC5389si`G}>PdP5-i%Bl%vhW>*@sDw&72A3 F0sy)rGGhP$ diff --git a/tests/ut/data/dataset/golden/random_color_01_result.npz b/tests/ut/data/dataset/golden/random_color_01_result.npz index be34bcdf34976f9c810794556a491fafc47ea244..4a9987076115c0d77932e9400da387f49f4be28b 100644 GIT binary patch delta 180 zcmX@fdX&{9z?+#xgaHB+87@?Wow8+OU;trG1`&qDqM~>My}W`-Mg{?}AV{I@M3YZU zogI_481+~)a}(23C&w~sfdsl4)dgfe zyJ+y;GhSY{w@|*MP$4Nbf)!|5PGVAO&SZ8bmC4T-1t;4uDTC<(CK*n+V6^03FQ0000xbKN@gIx~3?R(OAi|JXR1|NZmse29$RGd~1S$M6(c}|T bXUAkM##j*5&A5jXu5WSxlOCHf6G#~Ve9jbc delta 99 zcmZo*ZDBPD@MdNaVSoTd23N7`fB!HtFn};8g9t-nQBk~sUS2^ZBZB}~5Tx+WM3YY} d-VEN1lQkG)A&hp$J)Cg8lYN-<*vy$gN&!Ic7asrs diff --git a/tests/ut/data/dataset/golden/random_color_adjust_01_py_result.npz b/tests/ut/data/dataset/golden/random_color_adjust_01_py_result.npz index 7855123b6712d346ad6e77ba7a28b9f7ec346527..87b85205a134984e5b45cd3294341a5e89313765 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhO@DIuK!?UU;trG1`&qDqM~>My}W`-Mg{?}AV}eli6)p~rHu(^viU5!F&F4?P>eD3Ac7~0Fxe@F%!rZ03j1HA^-pY delta 186 zcmZo*ZDBPD@MdNaVSoTdhPi8IUi!nxzyQLW3?dAPMMd!jdU*wvj0^%`L6E{f6HPv` zcr$o2PS#+Ig)rI~)di0H`@7+8>w4MOQqn2B%fD}!ypK^upn+X*>NE3w+4rjR^>+82 zKQ;Liql#Hcsquy27WSF58Yb`MKb@^us9I8}mXw-WQmCG!7vRmvB*KivNt1n;^w`Xq HKrR3PC*e6x diff --git a/tests/ut/data/dataset/golden/random_crop_01_py_result.npz b/tests/ut/data/dataset/golden/random_crop_01_py_result.npz index 15041f8e52f81daccb8ee1d9228697a912085563..86f99ef30262eda76c1ec6a7d4bafdc351dc7de8 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhF=cLzx-fiU;trG1`&qDqM~>My}W`-Mg{?}AV}eli6)-W?Dm4?+XY18wQ1{C5389si`G}>PdP5-i%Bl%vhW>*@sDw&72A3 F0swTnG_e2x diff --git a/tests/ut/data/dataset/golden/random_crop_02_py_result.npz b/tests/ut/data/dataset/golden/random_crop_02_py_result.npz index 14cf210d3ae2866a4fbab4d89d1afcb91b3b8221..751d65f34c682edbf351132fefa0d3ead3a487d7 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhI2-?l0O(37(kelL4+Z(s3_h*FR!4IkwE|~2vYcCqRA(w z&W_1ijIki9n^9e$#$$@zrZ11gEZFML_Mwt(Z-%))^e@jz%bN>0B}S o8&L44eD3Ac7~0Fxe@F%!rZ0GnqpBLDyZ delta 186 zcmZo*ZDBPD@MdNaVSoTdh8V+&r+*k37(kelL4+Z(s3_h*FR!4IkwE|~2vYcGqRA%~ zZw7D1$r_BY5Jo$ry1+!4s#SBY+?|~CDrAn!T!rtG_c5vnFtV45J4n1r2wrut?(xTu z9g{yXs+cX1(f?{OOQhh;vc5|U`!>BRR4pk~OG-^GDO69=3-D%S5@E*Tq{%)^dTi!Q HAQu1tSqwMI diff --git a/tests/ut/data/dataset/golden/random_crop_03_py_result.npz b/tests/ut/data/dataset/golden/random_crop_03_py_result.npz index 3333dd17841153a4453cb2a8e75dce6e8f10ce56..8a313194c626158feed618576a3acde86b40795b 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhPt3#5kD9i7(kelL4+Z(s3_h*FR!4IkwE|~2vYcCqRA(w z&W_1ijIki9n^9e0)!jXXC0Q{7Tl+ehO^cWxPd>z`B2Z>*SrS{ExpZ#Kg5^O6rhS?G o4Ji0aXzHR2*EdTuct51_ZMm~xvKEspC*0b}0Ze*q#!Mhv02~c6^#A|> delta 186 zcmZo*ZDBPD@MdNaVSoTd2A6#;6@M5R7(kelL4+Z(s3_h*FR!4IkwE|~2vYcGqRA%~ zZw7D1$r_BY5Jo$ry1=W((_OYv5)O(UN7+M^x5iH1$EYH(z*SLnMY{8Y6Q4gz$u&5u zP5#8FVwPUA+dHoPgSwSx_SKr6U!H}kC5389si`G}>PdP5-i%Bl%vhW>*@sDw&72A3 F0suvqG+Y1x diff --git a/tests/ut/data/dataset/golden/random_crop_05_py_result.npz b/tests/ut/data/dataset/golden/random_crop_05_py_result.npz index 35d187ac8a4c59aaef68ee0df1e349b1097f52e0..bf082e70ded60d356e5af570c3ff8241dd02c8c6 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhA+x98h0{h@g}E37b=fM^~YBToV;|BbMhfZ6#<76v#&3eT&_}kDfHxzP2s0KZP4;2ZV>4$0 Gxc~qK1vY&E diff --git a/tests/ut/data/dataset/golden/random_crop_07_py_result.npz b/tests/ut/data/dataset/golden/random_crop_07_py_result.npz index 68339c0aa46c2b362e1bd180bb1a17e6ca088ffa..172f03e8a18c45bafc5c93cbde764353efa03816 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTd25Zannm-sB7(kelL4+Z(s3_h*FR!4IkwE|~2vYcCqRA(w z&W_1ijIki9n^9fB@8^-l7v4JOul|?j?6P{#gvo~(RRkjZ3YL3ougu@ho3v@k6sP9N p-++QnoAn*6e|Z(S4#%DZPwRZQ(|ph5eT*ssZ#>WLx;Ja*zHqadm-1#O z`%nJFsA9HawR-$FHmS+|r61dMd$zwWR4pk~OG-^GDO69=3-D%S5@E*Tq{%)^dTi!Q HAQu1tXu>!H diff --git a/tests/ut/data/dataset/golden/random_crop_08_py_result.npz b/tests/ut/data/dataset/golden/random_crop_08_py_result.npz index cc26b0322f806944446ead425aee540c2110844d..ec6fbe213f9a1a4a526c2b4142336281816e5bac 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhP9%Xq<=6nFn};8g9t-nQBk~sUS2^ZBZB}~5Tx+OM3YZU zogI_47-K0|M-_Ik9FT8cmU;Qu5*=6;f36l>ost9Zg5!v}Q>QAq8%Apqr`xa+R p{st6u+N|$j{p<627QTovg(q%L%u3asZPan=uo}769jjHPZk9 delta 186 zcmZo*ZDBPD@MdNaVSoTdhLYJWp???|7(kelL4+Z(s3_h*FR!4IkwE|~2vYcGqRA%~ zZw7D1$r_BY5Jo$rx&Uv4z>Z(S4#%DZPwRZQ(|ph5eT*ss7LMjNLK8aqZgzh6opbQ6 z=HyR|DrPHItH*z1lbYOL`mtTNXZ!0y)sjNBq}0@sLiHrQ0B=Sn5oRn-n(V`*$7aq1 GasdG63pR5A diff --git a/tests/ut/data/dataset/golden/random_crop_and_resize_01_c_result.npz b/tests/ut/data/dataset/golden/random_crop_and_resize_01_c_result.npz index 7952076673371563bb180a9d5f4511c104ff3329..e475532a162da3fe3fbd73d9683669a4f222c611 100644 GIT binary patch delta 95 zcmZo+ZD2JC@MdNaVSoTdhRwBeM1C+bFn};8g9t-nQBk~sUS2^ZBZB}~5Tx+OM3YZU bogI_47-KMy}W`-Mg{?}AV}eli6)Ov!F3t5JHw zGWipuikZf$H_s|#0+(n_cDP*QzCo=}wWLrjDK)jEP(4X6z?+dtgc*yICi^hyv6(Z0 GTmS&_(>7B8 diff --git a/tests/ut/data/dataset/golden/random_crop_and_resize_02_c_result.npz b/tests/ut/data/dataset/golden/random_crop_and_resize_02_c_result.npz index a8c5bf8e9844a0af9aa8a9a84b5fa11bcd6ab446..5c36258f01862ade9ffe54e86e74d901910dec40 100644 GIT binary patch delta 95 zcmZo+ZD2JC@MdNaVSoTd2BrFK96uNt7(kelL4+Z(s3_h*FR!4IkwE|~2vYcCqRA(w b&W_1ijIki9n{f{(T;JpXCOtM|CXg}!)(93| delta 99 zcmZo*ZDBPD@MdNaVSoTdhWdbnM}HU@7(kelL4+Z(s3_h*FR!4IkwE|~2vYcGqRA%~ dZw7D1$r_BY5Jo%W9!|L4$v#YaZ01ZLr2scU7UKW_ diff --git a/tests/ut/data/dataset/golden/random_crop_and_resize_02_py_result.npz b/tests/ut/data/dataset/golden/random_crop_and_resize_02_py_result.npz index dd30307bc5f4d7f2cededcf0d5ccf0402af10223..df30575b38b21a57867184721098dda5c9983620 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhT>Jvm;7L4U;trG1`&qDqM~>My}W`-Mg{?}AV}eli6)YdyR&6#t+r}D{G5HXqioh!G?z^uaNj%kw(e`(&5SN_% o4Jhd6lV8gJb;sw`N9tMk@}0jmS&K=Q6K?I~046;)VDfHxzP2s0KZP4;2ZV>4$0 Gxc~q~rZgP@ diff --git a/tests/ut/data/dataset/golden/random_crop_and_resize_03_c_result.npz b/tests/ut/data/dataset/golden/random_crop_and_resize_03_c_result.npz index 229c9c9f2b61e905d2ace7249641e4414f5dc83c..83003ca27150c59b63edcc2b703554ab83f308c3 100644 GIT binary patch delta 95 zcmZo+ZD2JC@MdNaVSoTd2By9>0zViT7(kelL4+Z(s3_h*FR!4IkwE|~2vYcCqRA(w b&W_1ijIki9n{f{(T;JpXCOtM|CXg}!&cYTq delta 99 zcmZo*ZDBPD@MdNaVSoTdhWLe5zy2^XFn};8g9t-nQBk~sUS2^ZBZB}~5Tx+WM3YY} d-VEN1lQkG)A&hp$J)Cg8lYN-<*vy$gN&#F`7pVXM diff --git a/tests/ut/data/dataset/golden/random_crop_and_resize_03_py_result.npz b/tests/ut/data/dataset/golden/random_crop_and_resize_03_py_result.npz index 621460e34ca7ba50d60de400e2d18d8e7ee93ef4..9b2eb1935c9ff8928e2f35190a98afa39fa25c15 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTd26^M7DnA$*7(kelL4+Z(s3_h*FR!4IkwE|~2vYcCqRA(w z&W_1ijIki9n^9dLs%2`yq&=*bJWHI}`%6|HntX^+MZlqAQd64go;^D5vK#9YW^SJR o4Jf!r;^-&d`VAUtr%z>h8JfCH)?$+7gj+i~fJu+dm%D^jfL(3%2(| zgeQMuR524{(Ecc;8rQ4V*Y>QDfHxzP2s0KZP4;2ZV>4$0 Gxc~sPrZoTn diff --git a/tests/ut/data/dataset/golden/random_erasing_01_result.npz b/tests/ut/data/dataset/golden/random_erasing_01_result.npz index 3333dd17841153a4453cb2a8e75dce6e8f10ce56..8a313194c626158feed618576a3acde86b40795b 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhPt3#5kD9i7(kelL4+Z(s3_h*FR!4IkwE|~2vYcCqRA(w z&W_1ijIki9n^9e0)!jXXC0Q{7Tl+ehO^cWxPd>z`B2Z>*SrS{ExpZ#Kg5^O6rhS?G o4Ji0aXzHR2*EdTuct51_ZMm~xvKEspC*0b}0Ze*q#!Mhv02~c6^#A|> delta 186 zcmZo*ZDBPD@MdNaVSoTd2A6#;6@M5R7(kelL4+Z(s3_h*FR!4IkwE|~2vYcGqRA%~ zZw7D1$r_BY5Jo$ry1=W((_OYv5)O(UN7+M^x5iH1$EYH(z*SLnMY{8Y6Q4gz$u&5u zP5#8FVwPUA+dHoPgSwSx_SKr6U!H}kC5389si`G}>PdP5-i%Bl%vhW>*@sDw&72A3 F0suvqG+Y1x diff --git a/tests/ut/data/dataset/golden/random_grayscale_01_result.npz b/tests/ut/data/dataset/golden/random_grayscale_01_result.npz index d606ab8619e955f7d35ecb0939175938bd2604ec..6028f5bbf0b83621fb7fcf88193955d30e583067 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTd28(@v4Sz5)Fn};8g9t-nQBk~sUS2^ZBZB}~5Tx+OM3YZU zogI_47-K0}1s=Ip%OR{1Fw)S;0n-(!Yo_vT=MIa%)Rl`{8<-3(9Yca`k!mXVgz@*1!%mlIp0OY+fNdN!< delta 186 zcmZo*ZDBPD@MdNaVSoTd23Ki?@;{6W3?R(OAi|JXR1|NZmse29$RGd~1S$M8(c}}0 zH-k6hWDUkx2&0`*UEo#Y=`PzS2?s@wqwFEdTVp5hV^k5C=CN@l=h|*#&%#}EV}kb0 zo&1SW#jG!8odbtJ;!2}u8$u@VbNUymmK3TbrKXk?swe3Mcr!AIFk^AjWFICyHghJB F3jkBdF&Y2> diff --git a/tests/ut/data/dataset/golden/random_grayscale_02_result.npz b/tests/ut/data/dataset/golden/random_grayscale_02_result.npz index 3333dd17841153a4453cb2a8e75dce6e8f10ce56..8a313194c626158feed618576a3acde86b40795b 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhPt3#5kD9i7(kelL4+Z(s3_h*FR!4IkwE|~2vYcCqRA(w z&W_1ijIki9n^9e0)!jXXC0Q{7Tl+ehO^cWxPd>z`B2Z>*SrS{ExpZ#Kg5^O6rhS?G o4Ji0aXzHR2*EdTuct51_ZMm~xvKEspC*0b}0Ze*q#!Mhv02~c6^#A|> delta 186 zcmZo*ZDBPD@MdNaVSoTd2A6#;6@M5R7(kelL4+Z(s3_h*FR!4IkwE|~2vYcGqRA%~ zZw7D1$r_BY5Jo$ry1=W((_OYv5)O(UN7+M^x5iH1$EYH(z*SLnMY{8Y6Q4gz$u&5u zP5#8FVwPUA+dHoPgSwSx_SKr6U!H}kC5389si`G}>PdP5-i%Bl%vhW>*@sDw&72A3 F0suvqG+Y1x diff --git a/tests/ut/data/dataset/golden/random_horizontal_01_py_result.npz b/tests/ut/data/dataset/golden/random_horizontal_01_py_result.npz index 262c01c99f73391b088e616852d652346dc70576..95bb484c8e6513cdf23e484f6efcebbc3ce1f3d6 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhW(o^e)+-3zyQLW3?dAPMMd!jdU*wvj0^%`L6E{96HPub zb#_eFVvGe*-Hhr2tM2Y8EXj%y*xJ|0Y+A(pc=91e6#=Dqg&1e*i*7qA{VZ_1_+Fw=wWLrjDK)jEP(4X6z?+dtgc*yICi^hyv6(Z0 GTmS$f4m7v` diff --git a/tests/ut/data/dataset/golden/random_lighting_py_01_result.npz b/tests/ut/data/dataset/golden/random_lighting_py_01_result.npz index 6b2eea95e5e6954ec412ed4e4b0862c03d4f2019..0d84a0750ed2e91d66ba7d104e81846b72b84b4c 100644 GIT binary patch delta 78 zcmX@gdX!Ztz?+#xgaHB+8CsTj{@y4whf%A{fWJnJWaVw}bT(lWV+$r1qbQyz@~ delta 78 zcmX@gdX!Ztz?+#xgaHB+84kT(bbX`H97cid@tXBv+WUOCTrAGSxz>E$9FhP4 diff --git a/tests/ut/data/dataset/golden/random_order_01_result.npz b/tests/ut/data/dataset/golden/random_order_01_result.npz index e2d04834adaba35eac62de6fca3df38c8fa8b8bb..8c6a19f5d661e89c052b21b3bb5119cd73546062 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhJq^}3w|&%Fn};8g9t-nQBk~sUS2^ZBZB}~5Tx+OM3YZU zogI_47-K0}1f~2nFQ=&L#wMsC#uw|ILO+Lh^BCv4R%m2zMC3-D=NzUI>rzuYU o1{Cc3YM~VQcF%;1x9+8XFL<|SvKEspC*0b}0Ze*q#!Mhv09>Lpp8x;= delta 186 zcmZo*ZDBPD@MdNaVSoTdh8yKeH~e8_U;trG1`&qDqM~>My}W`-Mg{?}AV}e#i6);| zycxV1Cu=aqLKy9g>H?2>AG~R;n#??1xO29Lf9b8s`xsRO{9^r5t$d$7v<$R0zRNz_ zY4RsV6|?EnL*|~do^?5MvN*>%x0td*)sjNBq}0@sLiHrQ0B=Sn5oRn-n(V`*$7aq1 GasdDp3N|(X diff --git a/tests/ut/data/dataset/golden/random_rotation_01_c_result.npz b/tests/ut/data/dataset/golden/random_rotation_01_c_result.npz index 794d6b714e8ba6d11ae9535275a674cd8ecbc738..96f81392f46e9142c6fdab715ba7b412b047fb33 100644 GIT binary patch delta 95 zcmZo+ZD2JC@MdNaVSoTdhLD#(y?!tMy}W`-Mg{?}AV}fTi6)76;L?wkW?|iR3 z@u%`(-R_HD>nDF>R8gF1#}NK_Ki7s0`_^eqeZ#|4s9I8}mXsR73bZ08F)1}?vNe;6 zfbrQkXU^?wFq$ua`(1rR$?3`YOe&L~F$zvz!lW!vs~dl+!OW@nH~ZBluCjk$CSPZg T<%IiT@?R!Bwg*6;FfafBPNz`5 delta 263 zcmZ3>wvo*wz?+#xgaHB+89rQDzj7rL0|N+iGKerF78S)C=;akuGBOB&1wjf|PBi(% z;?3aAI9Y>Hk2NPTDK%$uG@}+osGU(=!1(N&Gw1d-7|oZz{jNTuw`Pg?9(4T8>RlXKp_xbz($@7_1%v931NeKn86+XMZmiN!Itj0pi il0vJb)YOtf>m5&W-R`j{F6zK?FrDA3=9CH5n9Ut diff --git a/tests/ut/data/dataset/golden/random_sharpness_py_01_result.npz b/tests/ut/data/dataset/golden/random_sharpness_py_01_result.npz index ab95db32c9bd3f9fde04e540316dd7649433507f..3732e00521eb1cf5dc311b5f87f8b3f918355446 100644 GIT binary patch delta 180 zcmX@fdX&{9z?+#xgaHB+8ER_(o7plkFn};8g9t-nQBk~sUS2^ZBZB}~5Twv{qRA(w z&W_1ijC!n@xrynilVcgRKmy&2>H=?KzAb;VIEYnn2G=+7^i{7WA7WHdPz$C*7cg*CeOnPh; JKqoOU006Z{GJya9 delta 210 zcmX@gdXm*7z?+#xgaHB+8Gd~@+waK4zyQLW3?dAPMMd!jdU*wvj0^%`L6Abni6);| zycxV1Cu=b3vF0QurRGeIX4HZRwKJ*#aXl5s7?2i?2)HMNt5?6swkoh7Rr|t zDkP;wumVlY%uP&Boy^RnBA_rWVrjXaA*0y>_58nn$M Z@z`BEb7wWcT%#d%1TMz2bi`FSC8} pH=y9<=w*L4Nl0wAkYCf9_Ir8kWGyCHPPny`1DN#KjF~{T003}DGzb6y delta 186 zcmZo*ZDBPD@MdNaVSoTdhUEsF7yd9ZFn};8g9t-nQBk~sUS2^ZBZB}~5Tx+WM3YY} z-VEN1lQkG)A&hoLb%9rnr@L&UBpehyjJZ(+L?Ct8_(Nht5fT1xm5~POA6JJQd3I`)syrBycwB9n6Wr%vJaCUn>iE6 F1pr%2F`57X diff --git a/tests/ut/data/dataset/golden/ten_crop_01_result.npz b/tests/ut/data/dataset/golden/ten_crop_01_result.npz index 9e51610558cae40a29cb66453ce55790fb4df797..7f11fcac560f41d0df21744239f36f2984ef1874 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhQ2lTzy4rkU;trG1`&qDqM~>My}W`-Mg{?}AV}eli6)x+ZSIyq_T&(-|V?`!^` pe*+4(%vkz%d5x_cgXHP6_OG72nXJVm%L%u3asZPan=uo}764VCH$4CV delta 186 zcmZo*ZDBPD@MdNaVSoTdhKm~)y!^w+zyQLW3?dAPMMd!jdU*wvj0^%`L6E{f6HPv` zcr$o2PS#+Ig)rI~)dk$-ugREgcsTRK(ncPE-RYAj?_*REsC7&KnJ;B|bD>yOd`x!oHW^oNsrB( H3FHC*EGRmu diff --git a/tests/ut/data/dataset/golden/to_pil_01_result.npz b/tests/ut/data/dataset/golden/to_pil_01_result.npz index 0f9a34dd412681313ef01068d5f1b3785d761380..2e22eb355e083a0da92f2810fef4351232cfc9c4 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhTo-z3_lndz?hRkgdwr0DBeIXub`5VK>#cWQut${$tR}H zj>%e#u^_6OQC)!Fxc~3lY10)~&VO)Nef_;B2$F<4d nfPxvmw|epmU+;JQ+g7t9u3u@g7LzO|+}g31@d#kNPRSQ$Q3tkthmK3TbrKXk?swe3Mcr!AIFk^AjWFICyHghJB F3jj!3G|T`1 diff --git a/tests/ut/data/dataset/golden/to_pil_02_result.npz b/tests/ut/data/dataset/golden/to_pil_02_result.npz index 67faba626a429ece5357b9ad3188321035eb4b6c..ad14ad672cdc394bab57e36f809070d4ac455f8b 100644 GIT binary patch delta 158 zcmZo+ZD2JC@MdNaVSoTdhCQKY-F`4KFn};8g9t-nQBk~sUS2^ZBZB}~5Tx+OM3YZU zogI_47-K0}1G=cq=Y7vVZ_Wrok8T#I>Zt@{U6@lvpzZkFkzVv4LGj)^aiE6 F1pr{uG)Mpd diff --git a/tests/ut/python/dataset/test_five_crop.py b/tests/ut/python/dataset/test_five_crop.py index 8aa66885615..50bf11ba1b9 100644 --- a/tests/ut/python/dataset/test_five_crop.py +++ b/tests/ut/python/dataset/test_five_crop.py @@ -95,7 +95,8 @@ def test_five_crop_error_msg(): with pytest.raises(RuntimeError) as info: for _ in data: pass - error_msg = "TypeError: execute_py() takes 2 positional arguments but 6 were given" + error_msg = \ + "Unexpected error. map operation: [ToTensor] failed. The op is OneToOne, can only accept one tensor as input." assert error_msg in str(info.value) diff --git a/tests/ut/python/dataset/test_random_crop.py b/tests/ut/python/dataset/test_random_crop.py index 64e012bf0d7..c1e0e8f9036 100644 --- a/tests/ut/python/dataset/test_random_crop.py +++ b/tests/ut/python/dataset/test_random_crop.py @@ -546,7 +546,9 @@ def test_random_crop_09(): with pytest.raises(RuntimeError) as error_info: for _ in data.create_dict_iterator(num_epochs=1, output_numpy=True): pass - assert "img should be PIL image." in str(error_info.value) + error_msg = \ + "Unexpected error. map operation: [RandomCrop] failed. Pad: input shape is not or , got rank: 3" + assert error_msg in str(error_info.value) diff --git a/tests/ut/python/dataset/test_serdes_dataset.py b/tests/ut/python/dataset/test_serdes_dataset.py index c2c5ca59bda..61b4125038c 100644 --- a/tests/ut/python/dataset/test_serdes_dataset.py +++ b/tests/ut/python/dataset/test_serdes_dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,7 +22,6 @@ import os import pytest import numpy as np -from util import config_get_set_num_parallel_workers, config_get_set_seed import mindspore.common.dtype as mstype import mindspore.dataset as ds @@ -30,14 +29,19 @@ import mindspore.dataset.transforms.transforms as transforms import mindspore.dataset.vision.transforms as vision from mindspore import log as logger from mindspore.dataset.vision import Inter +from util import config_get_set_num_parallel_workers, config_get_set_seed def test_serdes_imagefolder_dataset(remove_json_files=True): """ - Test simulating resnet50 dataset pipeline. + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize with dataset pipeline that simulates ResNet50 + Expectation: Output verified for multiple deserialized pipelines """ data_dir = "../data/dataset/testPK/data" - ds.config.set_seed(1) + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) # define data augmentation parameters rescale = 1.0 / 255.0 @@ -98,6 +102,10 @@ def test_serdes_imagefolder_dataset(remove_json_files=True): logger.info("Number of data in data1: {}".format(num_samples)) assert num_samples == 11 + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + # Remove the generated json file if remove_json_files: delete_json_files("imagenet_dataset_pipeline") @@ -105,10 +113,14 @@ def test_serdes_imagefolder_dataset(remove_json_files=True): def test_serdes_mnist_dataset(remove_json_files=True): """ - Test serdes on mnist dataset pipeline. + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize with MnistDataset pipeline + Expectation: Output verified for multiple deserialized pipelines """ data_dir = "../data/dataset/testMnistData" - ds.config.set_seed(1) + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) data1 = ds.MnistDataset(data_dir, num_samples=100) one_hot_encode = transforms.OneHot(10) # num_classes is input argument @@ -140,15 +152,22 @@ def test_serdes_mnist_dataset(remove_json_files=True): logger.info("mnist total num samples is {}".format(str(num))) assert num == 10 + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + if remove_json_files: delete_json_files("mnist_dataset_pipeline") def test_serdes_cifar10_dataset(remove_json_files=True): """ - Test serdes on Cifar10 dataset pipeline + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize with Cifar10Dataset pipeline + Expectation: Output verified for multiple deserialized pipelines """ data_dir = "../data/dataset/testCifar10Data" + original_seed = config_get_set_seed(1) original_num_parallel_workers = config_get_set_num_parallel_workers(1) @@ -179,19 +198,22 @@ def test_serdes_cifar10_dataset(remove_json_files=True): assert num_samples == 2 - # Restore configuration num_parallel_workers + # Restore configuration ds.config.set_seed(original_seed) ds.config.set_num_parallel_workers(original_num_parallel_workers) + if remove_json_files: delete_json_files("cifar10_dataset_pipeline") def test_serdes_celeba_dataset(remove_json_files=True): """ - Test serdes on Celeba dataset pipeline. + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize with CelebADataset pipeline + Expectation: Output verified for multiple deserialized pipelines """ - DATA_DIR = "../data/dataset/testCelebAData/" - data1 = ds.CelebADataset(DATA_DIR, decode=True, num_shards=1, shard_id=0) + data_dir = "../data/dataset/testCelebAData/" + data1 = ds.CelebADataset(data_dir, decode=True, num_shards=1, shard_id=0) # define map operations data1 = data1.repeat(2) center_crop = vision.CenterCrop((80, 80)) @@ -214,11 +236,13 @@ def test_serdes_celeba_dataset(remove_json_files=True): def test_serdes_csv_dataset(remove_json_files=True): """ - Test serdes on Csvdataset pipeline. + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize with CSVDataset pipeline + Expectation: Output verified for multiple deserialized pipelines """ - DATA_DIR = "../data/dataset/testCSV/1.csv" + data_dir = "../data/dataset/testCSV/1.csv" data1 = ds.CSVDataset( - DATA_DIR, + data_dir, column_defaults=["1", "2", "3", "4"], column_names=['col1', 'col2', 'col3', 'col4'], shuffle=False) @@ -243,9 +267,12 @@ def test_serdes_csv_dataset(remove_json_files=True): def test_serdes_voc_dataset(remove_json_files=True): """ - Test serdes on VOC dataset pipeline. + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize with VOCDataset pipeline + Expectation: Output verified for multiple deserialized pipelines """ data_dir = "../data/dataset/testVOC2012" + original_seed = config_get_set_seed(1) original_num_parallel_workers = config_get_set_num_parallel_workers(1) @@ -270,20 +297,25 @@ def test_serdes_voc_dataset(remove_json_files=True): assert num_samples == 7 - # Restore configuration num_parallel_workers + # Restore configuration ds.config.set_seed(original_seed) ds.config.set_num_parallel_workers(original_num_parallel_workers) + if remove_json_files: delete_json_files("voc_dataset_pipeline") def test_serdes_zip_dataset(remove_json_files=True): """ - Test serdes on zip dataset pipeline. + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize with zipped pipeline + Expectation: Output verified for multiple deserialized pipelines """ files = ["../data/dataset/testTFTestAllTypes/test.data"] schema_file = "../data/dataset/testTFTestAllTypes/datasetSchema.json" - ds.config.set_seed(1) + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) ds0 = ds.TFRecordDataset(files, schema=schema_file, shuffle=ds.Shuffle.GLOBAL) data1 = ds.TFRecordDataset(files, schema=schema_file, shuffle=ds.Shuffle.GLOBAL) @@ -318,28 +350,35 @@ def test_serdes_zip_dataset(remove_json_files=True): rows += 1 assert rows == 12 + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + if remove_json_files: delete_json_files("zip_dataset_pipeline") def test_serdes_random_crop(): """ - Test serdes on RandomCrop pipeline. + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipeline with RandomCrop op + Expectation: Output verified for multiple deserialized pipelines """ logger.info("test_random_crop") - DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] - SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] + schema_dir = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + original_seed = config_get_set_seed(1) original_num_parallel_workers = config_get_set_num_parallel_workers(1) # First dataset - data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) + data1 = ds.TFRecordDataset(data_dir, schema_dir, columns_list=["image"]) decode_op = vision.Decode() random_crop_op = vision.RandomCrop([512, 512], [200, 200, 200, 200]) data1 = data1.map(operations=decode_op, input_columns="image") data1 = data1.map(operations=random_crop_op, input_columns="image") - # Serializing into python dictionary + # Serializing into Python dictionary ds1_dict = ds.serialize(data1) # Serializing into json object _ = json.dumps(ds1_dict, indent=2) @@ -348,7 +387,7 @@ def test_serdes_random_crop(): data1_1 = ds.deserialize(input_dict=ds1_dict) # Second dataset - data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"]) + data2 = ds.TFRecordDataset(data_dir, schema_dir, columns_list=["image"]) data2 = data2.map(operations=decode_op, input_columns="image") for item1, item1_1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True), @@ -357,14 +396,16 @@ def test_serdes_random_crop(): np.testing.assert_array_equal(item1['image'], item1_1['image']) _ = item2["image"] - # Restore configuration num_parallel_workers + # Restore configuration ds.config.set_seed(original_seed) ds.config.set_num_parallel_workers(original_num_parallel_workers) def test_serdes_to_device(remove_json_files=True): """ - Test serdes on transfer dataset pipeline. + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipeline with to_device op + Expectation: Serialized versus Deserialized+reserialized pipeline output verified """ data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" @@ -375,15 +416,20 @@ def test_serdes_to_device(remove_json_files=True): def test_serdes_pyvision(remove_json_files=True): """ - Test serdes on py_transform pipeline. + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipelines with Python implementation selected for vision ops + Expectation: Serialized versus Deserialized+reserialized pipeline output verified """ data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + data1 = ds.TFRecordDataset(data_dir, schema_file, columns_list=["image", "label"], shuffle=False) transforms1 = [ vision.Decode(True), - vision.CenterCrop([32, 32]), - vision.ToTensor() + vision.CenterCrop([32, 32]) ] transforms2 = [ vision.RandomColorAdjust(), @@ -393,26 +439,257 @@ def test_serdes_pyvision(remove_json_files=True): data1 = data1.map(operations=transforms.Compose(transforms1), input_columns=["image"]) data1 = data1.map(operations=transforms.RandomApply(transforms2), input_columns=["image"]) util_check_serialize_deserialize_file(data1, "pyvision_dataset_pipeline", remove_json_files) + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + if remove_json_files: + delete_json_files("pyvision_dataset_pipeline") + + +def test_serdes_pyfunc(remove_json_files=True): + """ + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipelines with Python functions + Expectation: Serialized versus Deserialized+reserialized pipeline output verified + """ + data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] + schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + data2 = ds.TFRecordDataset(data_dir, schema_file, columns_list=["image", "label"], shuffle=False) data2 = data2.map(operations=(lambda x, y, z: ( np.array(x).flatten().reshape(10, 39), np.array(y).flatten().reshape(10, 39), np.array(z).flatten().reshape(10, 1) ))) - ds.serialize(data2, "pyvision_dataset_pipeline.json") - assert validate_jsonfile("pyvision_dataset_pipeline.json") is True + ds.serialize(data2, "pyfunc_dataset_pipeline.json") + assert validate_jsonfile("pyfunc_dataset_pipeline.json") is True + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) if remove_json_files: - delete_json_files("pyvision_dataset_pipeline") + delete_json_files("pyfunc_dataset_pipeline") + + +def test_serdes_inter_mixed_map(remove_json_files=True): + """ + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipelines in which each map op has the same + implementation (Python or C++) of ops + Expectation: Serialized versus Deserialized+reserialized pipeline output verified + """ + data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] + schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + data1 = ds.TFRecordDataset(data_dir, schema_file, columns_list=["image", "label"], shuffle=False) + # The following map op uses Python implementation of ops + data1 = data1.map(operations=[vision.Decode(True), vision.CenterCrop([24, 24])], input_columns=["image"]) + # The following map op uses C++ implementation of ToTensor op + data1 = data1.map(operations=[vision.ToTensor()], input_columns=["image"]) + # The following map op uses C++ implementation of ops + data1 = data1.map(operations=[vision.HorizontalFlip(), vision.VerticalFlip()], input_columns=["image"]) + # The following map op uses Python implementation of ops + data1 = data1.map(operations=[vision.ToPIL(), vision.FiveCrop((18, 22))], input_columns=["image"]) + + util_check_serialize_deserialize_file(data1, "inter_mixed_map_pipeline", remove_json_files) + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + if remove_json_files: + delete_json_files("inter_mixed_map_pipeline") + + +def test_serdes_intra_mixed_py2c_map(remove_json_files=True): + """ + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipelines in which each map op has a mix of Python implementation + then C++ implementation of ops + Expectation: Serialized versus Deserialized+reserialized pipeline output verified + """ + data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] + schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + data1 = ds.TFRecordDataset(data_dir, schema_file, columns_list=["image", "label"], shuffle=False) + # The following map op uses mixed implementation of ops: + # - Decode - Python implementation + # - CenterCrop - Python Implementation + # - ToTensor - C++ implementation + # - RandonHorizontalFlip - C++ implementation + # - VerticalFlip - C++ implementation + transforms_list = [vision.Decode(True), + vision.CenterCrop([24, 24]), + vision.ToTensor(), + vision.RandomHorizontalFlip(), + vision.VerticalFlip()] + data1 = data1.map(operations=transforms_list, input_columns=["image"]) + data2 = util_check_serialize_deserialize_file(data1, "intra_mixed_py2c_map_pipeline", False) + + num_itr = 0 + # Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2) + for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True), + data2.create_dict_iterator(num_epochs=1, output_numpy=True)): + np.testing.assert_array_equal(item1['image'], item2['image']) + num_itr += 1 + assert num_itr == 3 + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + if remove_json_files: + delete_json_files("intra_mixed_py2c_map_pipeline") + + +def test_serdes_intra_mixed_c2py_map(remove_json_files=True): + """ + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipelines in which each map op has a mix of C++ implementation + then Python implementation of ops + Expectation: Serialized versus Deserialized+reserialized pipeline output verified + """ + data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] + schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + data1 = ds.TFRecordDataset(data_dir, schema_file, columns_list=["image", "label"], shuffle=False) + # The following map op uses mixed implementation of ops: + # - Decode - C++ implementation + # - RandomSolarize - C++ implementation + # - ToPIL - Python Implementation + # - CenterCrop - Python Implementation + transforms_list = [vision.Decode(), + vision.RandomSolarize((0, 127)), + vision.ToPIL(), + vision.CenterCrop([64, 64])] + data1 = data1.map(operations=transforms_list, input_columns=["image"]) + data2 = util_check_serialize_deserialize_file(data1, "intra_mixed_c2py_map_pipeline", False) + + num_itr = 0 + # Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2) + for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True), + data2.create_dict_iterator(num_epochs=1, output_numpy=True)): + np.testing.assert_array_equal(item1['image'], item2['image']) + num_itr += 1 + assert num_itr == 3 + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + if remove_json_files: + delete_json_files("intra_mixed_c2py_map_pipeline") + + +def test_serdes_totensor_normalize(remove_json_files=True): + """ + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipelines in which each map op has common scenario with + ToTensor and Normalize ops + Expectation: Serialized versus Deserialized+reserialized pipeline output verified + """ + data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] + schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + data1 = ds.TFRecordDataset(data_dir, schema_file, columns_list=["image", "label"], shuffle=False) + # The following map op uses mixed implementation of ops: + # - Decode - Python implementation + # - CenterCrop - Python Implementation + # - ToTensor - C++ implementation + # - Normalize - C++ implementation + transforms_list = [vision.Decode(True), + vision.CenterCrop([30, 50]), + vision.ToTensor(), + vision.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225], is_hwc=False)] + data1 = data1.map(operations=transforms_list, input_columns=["image"]) + data2 = util_check_serialize_deserialize_file(data1, "totensor_normalize_pipeline", False) + + num_itr = 0 + # Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2) + for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True), + data2.create_dict_iterator(num_epochs=1, output_numpy=True)): + np.testing.assert_array_equal(item1['image'], item2['image']) + num_itr += 1 + assert num_itr == 3 + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + if remove_json_files: + delete_json_files("totensor_normalize_pipeline") + + +def test_serdes_tonumpy(remove_json_files=True): + """ + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipelines with ToNumpy op + Expectation: Serialized versus Deserialized+reserialized pipeline output verified + """ + data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] + schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + data1 = ds.TFRecordDataset(data_dir, schema_file, columns_list=["image", "label"], shuffle=False) + # The following map op uses mixed implementation of ops: + # - Decode - Python implementation + # - CenterCrop - Python Implementation + # - ToNumpy - C++ implementation set + # - Crop - C++ implementation + transforms_list = [vision.Decode(to_pil=True), + vision.CenterCrop((200, 300)), + vision.ToNumpy(), + vision.Crop([5, 5], [40, 60])] + data1 = data1.map(operations=transforms_list, input_columns=["image"]) + data2 = util_check_serialize_deserialize_file(data1, "tonumpy_pipeline", False) + + num_itr = 0 + # Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2) + for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True), + data2.create_dict_iterator(num_epochs=1, output_numpy=True)): + np.testing.assert_array_equal(item1['image'], item2['image']) + num_itr += 1 + assert num_itr == 3 + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + if remove_json_files: + delete_json_files("tonumpy_pipeline") def test_serdes_uniform_augment(remove_json_files=True): """ - Test serdes on uniform augment. + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipeline with UniformAugment op + Expectation: Serialized versus Deserialized+reserialized pipeline output verified """ + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + data_dir = "../data/dataset/testPK/data" data = ds.ImageFolderDataset(dataset_dir=data_dir, shuffle=False) - ds.config.set_seed(1) transforms_ua = [vision.RandomHorizontalFlip(), vision.RandomVerticalFlip(), @@ -426,11 +703,18 @@ def test_serdes_uniform_augment(remove_json_files=True): data = data.map(operations=transforms_all, input_columns="image", num_parallel_workers=1) util_check_serialize_deserialize_file(data, "uniform_augment_pipeline", remove_json_files) + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + def skip_test_serdes_fill(remove_json_files=True): """ - Test serdes on Fill data transform. + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipelines with Fill op + Expectation: Serialized versus Deserialized+reserialized pipeline output verified """ + def gen(): yield (np.array([4, 5, 6, 7], dtype=np.int32),) @@ -447,7 +731,9 @@ def skip_test_serdes_fill(remove_json_files=True): def test_serdes_exception(): """ - Test exception case in serdes + Feature: Serialize and Deserialize Support + Description: Test exception cases + Expectation: Correct error is verified """ data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" @@ -505,6 +791,7 @@ def delete_json_files(filename): except IOError: logger.info("Error while deleting: {}".format(f)) + if __name__ == '__main__': test_serdes_imagefolder_dataset() test_serdes_mnist_dataset() @@ -516,6 +803,12 @@ if __name__ == '__main__': test_serdes_random_crop() test_serdes_to_device() test_serdes_pyvision() + test_serdes_pyfunc() + test_serdes_inter_mixed_map() + test_serdes_intra_mixed_py2c_map() + test_serdes_intra_mixed_c2py_map() + test_serdes_totensor_normalize() + test_serdes_tonumpy() test_serdes_uniform_augment() skip_test_serdes_fill() test_serdes_exception() diff --git a/tests/ut/python/dataset/test_ten_crop.py b/tests/ut/python/dataset/test_ten_crop.py index 1022e121806..15882c3af4c 100644 --- a/tests/ut/python/dataset/test_ten_crop.py +++ b/tests/ut/python/dataset/test_ten_crop.py @@ -179,7 +179,8 @@ def test_ten_crop_wrong_img_error_msg(): with pytest.raises(RuntimeError) as info: data.create_tuple_iterator(num_epochs=1).__next__() - error_msg = "TypeError: execute_py() takes 2 positional arguments but 11 were given" + error_msg = \ + "Unexpected error. map operation: [ToTensor] failed. The op is OneToOne, can only accept one tensor as input." assert error_msg in str(info.value) diff --git a/tests/ut/python/dataset_deprecated/test_serdes_dataset.py b/tests/ut/python/dataset_deprecated/test_serdes_dataset.py new file mode 100644 index 00000000000..e1c20b4892f --- /dev/null +++ b/tests/ut/python/dataset_deprecated/test_serdes_dataset.py @@ -0,0 +1,254 @@ +# Copyright 2020-2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +""" +Testing dataset serialize and deserialize in DE +""" +import filecmp +import glob +import json +import os + +import numpy as np + +import mindspore.dataset as ds +import mindspore.dataset.transforms.py_transforms as py_transforms +import mindspore.dataset.vision.c_transforms as c_vision +import mindspore.dataset.vision.py_transforms as py_vision +from mindspore import log as logger +from ..dataset.util import config_get_set_num_parallel_workers, config_get_set_seed + + +def test_serdes_pyvision(remove_json_files=True): + """ + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipelines with Python vision ops + Expectation: Serialized versus Deserialized+reserialized pipeline output verified + """ + data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] + schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + data1 = ds.TFRecordDataset(data_dir, schema_file, columns_list=["image", "label"], shuffle=False) + transforms1 = [ + py_vision.Decode(), + py_vision.CenterCrop([32, 32]) + ] + transforms2 = [ + py_vision.RandomColorAdjust(), + py_vision.FiveCrop(1), + py_vision.Grayscale() + ] + data1 = data1.map(operations=py_transforms.Compose(transforms1), input_columns=["image"]) + data1 = data1.map(operations=py_transforms.RandomApply(transforms2), input_columns=["image"]) + util_check_serialize_deserialize_file(data1, "depr_pyvision_dataset_pipeline", remove_json_files) + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + if remove_json_files: + delete_json_files("depr_pyvision_dataset_pipeline") + + +def test_serdes_pyfunc(remove_json_files=True): + """ + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipelines with Python functions + Expectation: Serialized versus Deserialized+reserialized pipeline output verified + """ + data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] + schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + data2 = ds.TFRecordDataset(data_dir, schema_file, columns_list=["image", "label"], shuffle=False) + data2 = data2.map(operations=(lambda x, y, z: ( + np.array(x).flatten().reshape(10, 39), + np.array(y).flatten().reshape(10, 39), + np.array(z).flatten().reshape(10, 1) + ))) + ds.serialize(data2, "pyfunc_dataset_pipeline.json") + assert validate_jsonfile("pyfunc_dataset_pipeline.json") is True + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + if remove_json_files: + delete_json_files("depr_pyfunc_dataset_pipeline") + + +def test_serdes_inter_mixed_map(remove_json_files=True): + """ + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipelines in which each map op has Python ops or C++ ops + Expectation: Serialized versus Deserialized+reserialized pipeline output verified + """ + data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] + schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + data1 = ds.TFRecordDataset(data_dir, schema_file, columns_list=["image", "label"], shuffle=False) + # The following map op uses Python ops + data1 = data1.map(operations=[py_vision.Decode(), py_vision.CenterCrop([24, 24])], input_columns=["image"]) + # The following map op uses Python ops + data1 = data1.map(operations=[py_vision.ToTensor(), py_vision.ToPIL()], input_columns=["image"]) + # The following map op uses C++ ops + data1 = data1.map(operations=[c_vision.HorizontalFlip(), c_vision.VerticalFlip()], input_columns=["image"]) + # The following map op uses Python ops + data1 = data1.map(operations=[py_vision.ToPIL(), py_vision.FiveCrop((18, 22))], input_columns=["image"]) + + util_check_serialize_deserialize_file(data1, "depr_inter_mixed_map_pipeline", remove_json_files) + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + if remove_json_files: + delete_json_files("depr_inter_mixed_map_pipeline") + + +def test_serdes_intra_mixed_py2c_map(remove_json_files=True): + """ + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipelines in which each map op has a mix of Python ops + then C++ ops + Expectation: Serialized versus Deserialized+reserialized pipeline output verified + """ + data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] + schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + data1 = ds.TFRecordDataset(data_dir, schema_file, columns_list=["image", "label"], shuffle=False) + transforms_list = [py_vision.Decode(), + py_vision.CenterCrop([24, 24]), + py_vision.ToTensor(), + py_vision.Normalize([0.48, 0.45, 0.40], [0.22, 0.22, 0.22]), + c_vision.RandomHorizontalFlip(), + c_vision.VerticalFlip()] + data1 = data1.map(operations=transforms_list, input_columns=["image"]) + data2 = util_check_serialize_deserialize_file(data1, "depr_intra_mixed_py2c_map_pipeline", False) + + num_itr = 0 + # Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2) + for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True), + data2.create_dict_iterator(num_epochs=1, output_numpy=True)): + np.testing.assert_array_equal(item1['image'], item2['image']) + num_itr += 1 + assert num_itr == 3 + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + if remove_json_files: + delete_json_files("depr_intra_mixed_py2c_map_pipeline") + + +def test_serdes_intra_mixed_c2py_map(remove_json_files=True): + """ + Feature: Serialize and Deserialize Support + Description: Test serialize and deserialize on pipelines in which each map op has a mix of C++ ops + then Python ops + Expectation: Serialized versus Deserialized+reserialized pipeline output verified + """ + data_dir = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"] + schema_file = "../data/dataset/test_tf_file_3_images/datasetSchema.json" + + original_seed = config_get_set_seed(1) + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + + data1 = ds.TFRecordDataset(data_dir, schema_file, columns_list=["image", "label"], shuffle=False) + transforms_list = [c_vision.Decode(), + c_vision.RandomSolarize((0, 127)), + py_vision.ToPIL(), + py_vision.CenterCrop([64, 64])] + data1 = data1.map(operations=transforms_list, input_columns=["image"]) + data2 = util_check_serialize_deserialize_file(data1, "depr_intra_mixed_c2py_map_pipeline", False) + + num_itr = 0 + # Iterate and compare the data in the original pipeline (data1) against the deserialized pipeline (data2) + for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1, output_numpy=True), + data2.create_dict_iterator(num_epochs=1, output_numpy=True)): + np.testing.assert_array_equal(item1['image'], item2['image']) + num_itr += 1 + assert num_itr == 3 + + # Restore configuration + ds.config.set_seed(original_seed) + ds.config.set_num_parallel_workers(original_num_parallel_workers) + + if remove_json_files: + delete_json_files("depr_intra_mixed_c2py_map_pipeline") + + +def util_check_serialize_deserialize_file(data_orig, filename, remove_json_files): + """ + Utility function for testing serdes files. It is to check if a json file is indeed created with correct name + after serializing and if it remains the same after repeatedly saving and loading. + :param data_orig: original data pipeline to be serialized + :param filename: filename to be saved as json format + :param remove_json_files: whether to remove the json file after testing + :return: The data pipeline after serializing and deserializing using the original pipeline + """ + file1 = filename + ".json" + file2 = filename + "_1.json" + ds.serialize(data_orig, file1) + assert validate_jsonfile(file1) is True + assert validate_jsonfile("wrong_name.json") is False + + data_changed = ds.deserialize(json_filepath=file1) + ds.serialize(data_changed, file2) + assert validate_jsonfile(file2) is True + assert filecmp.cmp(file1, file2, shallow=False) + + # Remove the generated json file + if remove_json_files: + delete_json_files(filename) + return data_changed + + +def validate_jsonfile(filepath): + try: + file_exist = os.path.exists(filepath) + with open(filepath, 'r') as jfile: + loaded_json = json.load(jfile) + except IOError: + return False + return file_exist and isinstance(loaded_json, dict) + + +def delete_json_files(filename): + file_list = glob.glob(filename + '.json') + glob.glob(filename + '_1.json') + for f in file_list: + try: + os.remove(f) + except IOError: + logger.info("Error while deleting: {}".format(f)) + + +if __name__ == '__main__': + test_serdes_pyvision() + test_serdes_pyfunc() + test_serdes_inter_mixed_map() + test_serdes_intra_mixed_py2c_map() + test_serdes_intra_mixed_c2py_map()