From f4fd0300c629e03d2ce36f176e25f3d9b5987557 Mon Sep 17 00:00:00 2001 From: Cathy Wong Date: Wed, 25 May 2022 20:49:09 -0400 Subject: [PATCH] [MD] Transform Unification - Phase 3: Update Smoke tests and remaining UTs --- .../dataset/kernels/data/data_utils.cc | 2 +- .../dataset/kernels/data/data_utils.h | 8 +--- .../dataset/kernels/ir/data/transforms_ir.cc | 2 +- .../dataset/transforms/transforms.py | 2 +- .../data_transfer/test_tdt_data_transfer.py | 6 +-- .../dataset/test_chinese_path_on_windows.py | 4 +- tests/st/dataset/test_gpu_autotune.py | 26 ++++++------- tests/st/fl/albert/src/dataset.py | 6 +-- .../image_classfication_dataset_process.md | 19 +++++----- .../image_classfication_dataset_process_en.md | 19 +++++----- .../fl/cross_silo_faster_rcnn/src/dataset.py | 5 ++- .../test_cross_silo_femnist.py | 21 +++++----- .../st/frontend_compile_cache/run_lenet_ps.py | 4 +- tests/st/mem_reuse/resnet_cifar_memreuse.py | 6 +-- tests/st/mem_reuse/resnet_cifar_normal.py | 6 +-- .../transformer/test_transformer.py | 4 +- .../st/model_zoo_tests/yolov3/src/dataset.py | 7 ++-- .../yolov3_darknet53/src/yolo_dataset.py | 6 +-- .../test_bert_tdt_lossscale.py | 4 +- .../bert/bert_performance/test_bert_thor.py | 4 +- .../bert_precision/test_bert_tdt_lossscale.py | 4 +- tests/st/networks/models/bert/src/dataset.py | 4 +- .../models/deeplabv3/src/md_dataset.py | 4 +- .../networks/models/resnet50/src/dataset.py | 6 +-- .../models/resnet50/src_thor/dataset.py | 30 +++++++-------- tests/st/networks/test_gpu_lenet.py | 6 +-- .../st/networks/test_gradient_accumulation.py | 4 +- tests/st/ops/ascend/test_tdt_data_ms.py | 4 +- tests/st/probability/bnn_layers/dataset.py | 6 +-- tests/st/probability/dpn/test_gpu_svi_cvae.py | 4 +- tests/st/probability/dpn/test_gpu_svi_vae.py | 4 +- tests/st/probability/dpn/test_gpu_vae_gan.py | 4 +- .../probability/toolbox/test_uncertainty.py | 6 +-- tests/st/probability/transforms/dataset.py | 6 +-- tests/st/probability/zhusuan/vae/utils.py | 6 +-- tests/st/profiler/test_profiler.py | 6 +-- tests/st/ps/full_ps/test_full_ps_lenet.py | 6 +-- ..._ps_embedding_heterogeneous_conv2d_adam.py | 6 +-- .../pynative/test_pynative_resnet50_ascend.py | 4 +- .../st/pynative/test_pynative_resnet50_gpu.py | 6 +-- tests/st/quantization/lenet_quant/dataset.py | 6 +-- .../quantization/mobilenetv2_quant/dataset.py | 6 +-- .../st/quantization/resnet50_quant/dataset.py | 6 +-- tests/st/summary/dataset.py | 6 +-- tests/st/tbe_networks/resnet_cifar.py | 6 +-- tests/st/tbe_networks/test_resnet_cifar_1p.py | 6 +-- tests/st/tbe_networks/test_resnet_cifar_8p.py | 6 +-- .../dataset/test_get_image_num_channels.py | 16 ++++---- .../ut/python/dataset/test_get_image_size.py | 38 ++++++++++--------- .../dataset/test_pyfunc_multiprocess.py | 1 - .../profiler/parser/test_minddata_analyzer.py | 18 +++++---- tests/ut/python/train/dataset.py | 7 ++-- tests/ut/python/utils/test_export.py | 19 +++++++++- 53 files changed, 220 insertions(+), 208 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.cc b/mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.cc index 52157804598..1296831c40f 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.cc @@ -106,7 +106,7 @@ Status OneHotEncoding(const std::shared_ptr &input, std::shared_ptrtype() == DataType::DE_UINT64) { RETURN_IF_NOT_OK(OneHotEncodingImpl(input, &out, num_classes, i, smoothing_rate)); } else { - RETURN_STATUS_UNEXPECTED("OneHot: OneHot only support input of int type, but got:" + input->type().ToString()); + RETURN_STATUS_UNEXPECTED("OneHot: OneHot only supports input of int type, but got:" + input->type().ToString()); } } out->Squeeze(); diff --git a/mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.h b/mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.h index eec1b0038ee..7a00cc4f1b1 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.h +++ b/mindspore/ccsrc/minddata/dataset/kernels/data/data_utils.h @@ -38,15 +38,11 @@ namespace dataset { // then it will fail. // @param output: Tensor. The shape of the output tensor is // and the type is same as input. -// @param num_classes: Number of classes to. +// @param num_classes: Number of classes in dataset. +// @param smoothing_rate: Adjustable hyperparameter for label smoothing level. Status OneHotEncoding(const std::shared_ptr &input, std::shared_ptr *output, dsize_t num_classes, double smoothing_rate = 0); -Status OneHotEncodingUnsigned(const std::shared_ptr &input, std::shared_ptr *output, - dsize_t num_classes, int64_t index); - -Status OneHotEncodingSigned(const std::shared_ptr &input, std::shared_ptr *output, dsize_t num_classes, - int64_t index); template Status OneHotEncodingImpl(const std::shared_ptr &input, std::shared_ptr *output, dsize_t num_classes, int64_t index, double smoothing_rate); diff --git a/mindspore/ccsrc/minddata/dataset/kernels/ir/data/transforms_ir.cc b/mindspore/ccsrc/minddata/dataset/kernels/ir/data/transforms_ir.cc index 09cc25ccf68..ceeff206dde 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/ir/data/transforms_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/ir/data/transforms_ir.cc @@ -168,7 +168,7 @@ Status OneHotOperation::ValidateParams() { LOG_AND_RETURN_STATUS_SYNTAX_ERROR(err_msg); } if (smoothing_rate_ < 0 || smoothing_rate_ > 1) { - std::string err_msg = "OneHot: Smoothing rate must be between 0 and 1, but got: " + std::to_string(num_classes_); + std::string err_msg = "OneHot: Smoothing rate must be between 0 and 1, but got: " + std::to_string(smoothing_rate_); LOG_AND_RETURN_STATUS_SYNTAX_ERROR(err_msg); } return Status::OK(); diff --git a/mindspore/python/mindspore/dataset/transforms/transforms.py b/mindspore/python/mindspore/dataset/transforms/transforms.py index 8f12b026e8e..907983aac29 100644 --- a/mindspore/python/mindspore/dataset/transforms/transforms.py +++ b/mindspore/python/mindspore/dataset/transforms/transforms.py @@ -554,7 +554,7 @@ class OneHot(TensorOperation): Raises: TypeError: `num_classes` is not of type int. - TypeError: `smoothing_rate` is not of type float. + TypeError: `smoothing_rate` is not of type float or int. ValueError: `smoothing_rate` is not in range [0.0, 1.0]. RuntimeError: Input tensor is not of type int. RuntimeError: Input tensor is not a 1-D tensor. diff --git a/tests/st/data_transfer/test_tdt_data_transfer.py b/tests/st/data_transfer/test_tdt_data_transfer.py index f845ea3f335..cd1ce2c6d22 100644 --- a/tests/st/data_transfer/test_tdt_data_transfer.py +++ b/tests/st/data_transfer/test_tdt_data_transfer.py @@ -1,4 +1,4 @@ -# Copyright 2021 Huawei Technologies Co., Ltd +# Copyright 2021-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,8 +21,8 @@ from mindspore.common.api import _cell_graph_executor from mindspore.common import dtype as mstype from mindspore.ops import operations as P import mindspore.dataset as de -from mindspore.dataset.vision import c_transforms as c_vision -from mindspore.dataset.transforms import c_transforms as c_trans +from mindspore.dataset.vision import transforms as c_vision +from mindspore.dataset.transforms import transforms as c_trans DATA_DIR = "/home/workspace/mindspore_dataset/cifar-10-verify-bin" diff --git a/tests/st/dataset/test_chinese_path_on_windows.py b/tests/st/dataset/test_chinese_path_on_windows.py index c5dcccdcb49..ff2992a82b4 100644 --- a/tests/st/dataset/test_chinese_path_on_windows.py +++ b/tests/st/dataset/test_chinese_path_on_windows.py @@ -1,4 +1,4 @@ -# Copyright 2019 Huawei Technologies Co., Ltd +# Copyright 2019-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ import pytest from PIL import Image import mindspore.dataset as ds -import mindspore.dataset.vision.c_transforms as vision +import mindspore.dataset.vision as vision from mindspore.mindrecord import FileWriter, SUCCESS def add_and_remove_cv_file(mindrecord): diff --git a/tests/st/dataset/test_gpu_autotune.py b/tests/st/dataset/test_gpu_autotune.py index 6065eb04e8b..45a7fc55f39 100644 --- a/tests/st/dataset/test_gpu_autotune.py +++ b/tests/st/dataset/test_gpu_autotune.py @@ -19,10 +19,8 @@ import pytest import numpy as np import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.transforms.py_transforms as py_transforms -import mindspore.dataset.vision.c_transforms as CV -import mindspore.dataset.vision.py_transforms as py_vision +import mindspore.dataset.transforms as transforms +import mindspore.dataset.vision as vision from mindspore import context, nn from mindspore.common import dtype as mstype, set_seed @@ -59,11 +57,11 @@ def create_dataset(data_path, batch_size=32, num_parallel_workers=1): shift_nml = -1 * 0.1307 / 0.3081 # Define map operations - resize_op = CV.Resize((resize_height, resize_width), interpolation=Inter.LINEAR) - rescale_nml_op = CV.Rescale(rescale_nml, shift_nml) - rescale_op = CV.Rescale(rescale, shift) - hwc2chw_op = CV.HWC2CHW() - type_cast_op = C.TypeCast(mstype.int32) + resize_op = vision.Resize((resize_height, resize_width), interpolation=Inter.LINEAR) + rescale_nml_op = vision.Rescale(rescale_nml, shift_nml) + rescale_op = vision.Rescale(rescale, shift) + hwc2chw_op = vision.HWC2CHW() + type_cast_op = transforms.TypeCast(mstype.int32) # Apply map operations on images mnist_ds = mnist_ds.map(operations=type_cast_op, input_columns="label", num_parallel_workers=num_parallel_workers) @@ -140,20 +138,18 @@ def create_dataset_pyfunc_multiproc(data_path, batch_size=32, num_op_parallel_wo # Define dataset with num_parallel_workers=8 for reasonable performance data1 = ds.MnistDataset(data_path, num_parallel_workers=8) - data1 = data1.map(operations=[py_vision.ToType(np.int32)], input_columns="label", + data1 = data1.map(operations=[vision.ToType(np.int32)], input_columns="label", num_parallel_workers=num_op_parallel_workers, python_multiprocessing=True, max_rowsize=max_rowsize) # Setup transforms list which include Python ops transforms_list = [ - py_vision.ToTensor(), lambda x: x, - py_vision.HWC2CHW(), - py_vision.RandomErasing(0.9, value='random'), - py_vision.Cutout(4, 2), + vision.HWC2CHW(), + vision.RandomErasing(0.9, value='random'), lambda y: y ] - compose_op = py_transforms.Compose(transforms_list) + compose_op = transforms.Compose(transforms_list) data1 = data1.map(operations=compose_op, input_columns="image", num_parallel_workers=num_op_parallel_workers, python_multiprocessing=True, max_rowsize=max_rowsize) diff --git a/tests/st/fl/albert/src/dataset.py b/tests/st/fl/albert/src/dataset.py index b2ab92ba2dc..05aaa9fc976 100644 --- a/tests/st/fl/albert/src/dataset.py +++ b/tests/st/fl/albert/src/dataset.py @@ -1,4 +1,4 @@ -# Copyright 2021 Huawei Technologies Co., Ltd +# Copyright 2021-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ import pickle import numpy as np import mindspore as ms import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as c_transforms +import mindspore.dataset.transforms as transforms class InputFeatures: @@ -140,7 +140,7 @@ def create_ms_dataset(data_list, label_list, max_seq_length, tokenizer, batch_si if do_shuffle: dataset = dataset.shuffle(buffer_size=10000) - type_cast_op = c_transforms.TypeCast(ms.int32) + type_cast_op = transforms.TypeCast(ms.int32) dataset = dataset.map(operations=[type_cast_op]) dataset = dataset.batch(batch_size=batch_size, drop_remainder=drop_remainder) return dataset diff --git a/tests/st/fl/cross_device_lenet/client/image_classfication_dataset_process.md b/tests/st/fl/cross_device_lenet/client/image_classfication_dataset_process.md index f9fc0cc55d3..a0a4167b177 100644 --- a/tests/st/fl/cross_device_lenet/client/image_classfication_dataset_process.md +++ b/tests/st/fl/cross_device_lenet/client/image_classfication_dataset_process.md @@ -309,9 +309,8 @@ import numpy as np import os import mindspore.dataset as ds - import mindspore.dataset.transforms.c_transforms as tC - import mindspore.dataset.vision.py_transforms as PV - import mindspore.dataset.transforms.py_transforms as PT + import mindspore.dataset.vision as vision + import mindspore.dataset.transforms as transforms import mindspore def mkdir(path): @@ -340,16 +339,16 @@ resize_height, resize_width = img_size[0], img_size[1] # 32 transform = [ - PV.Decode(), - PV.Grayscale(1), - PV.Resize(size=(resize_height, resize_width)), - PV.Grayscale(3), - PV.ToTensor(), + vision.Decode(True), + vision.Grayscale(1), + vision.Resize(size=(resize_height, resize_width)), + vision.Grayscale(3), + vision.ToTensor(), ] - compose = PT.Compose(transform) + compose = transforms.Compose(transform) # apply map operations on images - mnist_ds = mnist_ds.map(input_columns="label", operations=tC.TypeCast(mindspore.int32)) + mnist_ds = mnist_ds.map(input_columns="label", operations=transforms.TypeCast(mindspore.int32)) mnist_ds = mnist_ds.map(input_columns="image", operations=compose) # apply DatasetOps diff --git a/tests/st/fl/cross_device_lenet/client/image_classfication_dataset_process_en.md b/tests/st/fl/cross_device_lenet/client/image_classfication_dataset_process_en.md index 5433401cf32..44ff45be2d0 100644 --- a/tests/st/fl/cross_device_lenet/client/image_classfication_dataset_process_en.md +++ b/tests/st/fl/cross_device_lenet/client/image_classfication_dataset_process_en.md @@ -311,9 +311,8 @@ each user is 226.83, and the variance of the data volume of all users is 88.94. import numpy as np import os import mindspore.dataset as ds - import mindspore.dataset.transforms.c_transforms as tC - import mindspore.dataset.vision.py_transforms as PV - import mindspore.dataset.transforms.py_transforms as PT + import mindspore.dataset.vision as vision + import mindspore.dataset.transforms as transforms import mindspore def mkdir(path): @@ -342,16 +341,16 @@ each user is 226.83, and the variance of the data volume of all users is 88.94. resize_height, resize_width = img_size[0], img_size[1] # 32 transform = [ - PV.Decode(), - PV.Grayscale(1), - PV.Resize(size=(resize_height, resize_width)), - PV.Grayscale(3), - PV.ToTensor(), + vision.Decode(True), + vision.Grayscale(1), + vision.Resize(size=(resize_height, resize_width)), + vision.Grayscale(3), + vision.ToTensor(), ] - compose = PT.Compose(transform) + compose = transforms.Compose(transform) # apply map operations on images - mnist_ds = mnist_ds.map(input_columns="label", operations=tC.TypeCast(mindspore.int32)) + mnist_ds = mnist_ds.map(input_columns="label", operations=transforms.TypeCast(mindspore.int32)) mnist_ds = mnist_ds.map(input_columns="image", operations=compose) # apply DatasetOps diff --git a/tests/st/fl/cross_silo_faster_rcnn/src/dataset.py b/tests/st/fl/cross_silo_faster_rcnn/src/dataset.py index 46e485dd478..263c050e728 100644 --- a/tests/st/fl/cross_silo_faster_rcnn/src/dataset.py +++ b/tests/st/fl/cross_silo_faster_rcnn/src/dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,9 +23,10 @@ from numpy import random import cv2 import mmcv import mindspore.dataset as de -import mindspore.dataset.vision.c_transforms as C +import mindspore.dataset.vision as C from mindspore.mindrecord import FileWriter + def bbox_overlaps(bboxes1, bboxes2, mode='iou'): """Calculate the ious between each bbox of bboxes1 and bboxes2. diff --git a/tests/st/fl/cross_silo_femnist/test_cross_silo_femnist.py b/tests/st/fl/cross_silo_femnist/test_cross_silo_femnist.py index 759f3b0148d..b8b8b236e5b 100644 --- a/tests/st/fl/cross_silo_femnist/test_cross_silo_femnist.py +++ b/tests/st/fl/cross_silo_femnist/test_cross_silo_femnist.py @@ -1,4 +1,4 @@ -# Copyright 2021 Huawei Technologies Co., Ltd +# Copyright 2021-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,9 +24,8 @@ import mindspore.nn as nn from mindspore.common.initializer import TruncatedNormal from mindspore import Tensor import mindspore.dataset as ds -import mindspore.dataset.vision.py_transforms as PV -import mindspore.dataset.transforms.py_transforms as PT -import mindspore.dataset.transforms.c_transforms as tC +import mindspore.dataset.vision as vision +import mindspore.dataset.transforms as transforms from mindspore.train.serialization import save_checkpoint from mindspore.train.callback import Callback, FederatedLearningManager from mindspore.nn.metrics import Accuracy @@ -252,16 +251,16 @@ def create_dataset_from_folder(data_path, img_size, batch_size=32, repeat_size=1 resize_height, resize_width = img_size[0], img_size[1] transform = [ - PV.Decode(), - PV.Grayscale(1), - PV.Resize(size=(resize_height, resize_width)), - PV.Grayscale(3), - PV.ToTensor() + vision.Decode(True), + vision.Grayscale(1), + vision.Resize(size=(resize_height, resize_width)), + vision.Grayscale(3), + vision.ToTensor() ] - compose = PT.Compose(transform) + compose = transforms.Compose(transform) # apply map operations on images - mnist_ds = mnist_ds.map(input_columns="label", operations=tC.TypeCast(mindspore.int32)) + mnist_ds = mnist_ds.map(input_columns="label", operations=transforms.TypeCast(mindspore.int32)) mnist_ds = mnist_ds.map(input_columns="image", operations=compose) # apply DatasetOps diff --git a/tests/st/frontend_compile_cache/run_lenet_ps.py b/tests/st/frontend_compile_cache/run_lenet_ps.py index cb364c05774..5e6ae38b35d 100644 --- a/tests/st/frontend_compile_cache/run_lenet_ps.py +++ b/tests/st/frontend_compile_cache/run_lenet_ps.py @@ -18,8 +18,8 @@ import sys import mindspore.context as context import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.vision.c_transforms as CV +import mindspore.dataset.transforms as C +import mindspore.dataset.vision as CV import mindspore.nn as nn from mindspore.common import dtype as mstype from mindspore.dataset.vision import Inter diff --git a/tests/st/mem_reuse/resnet_cifar_memreuse.py b/tests/st/mem_reuse/resnet_cifar_memreuse.py index e4b12478ed4..6529006a1c9 100644 --- a/tests/st/mem_reuse/resnet_cifar_memreuse.py +++ b/tests/st/mem_reuse/resnet_cifar_memreuse.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,8 +21,8 @@ from resnet import resnet50 import mindspore.common.dtype as mstype import mindspore.context as context import mindspore.dataset as de -import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.vision.c_transforms as vision +import mindspore.dataset.transforms as C +import mindspore.dataset.vision as vision import mindspore.nn as nn from mindspore import Tensor from mindspore.communication.management import init diff --git a/tests/st/mem_reuse/resnet_cifar_normal.py b/tests/st/mem_reuse/resnet_cifar_normal.py index 5677a6d7306..f537046c6c7 100644 --- a/tests/st/mem_reuse/resnet_cifar_normal.py +++ b/tests/st/mem_reuse/resnet_cifar_normal.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,8 +21,8 @@ from resnet import resnet50 import mindspore.common.dtype as mstype import mindspore.context as context import mindspore.dataset as de -import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.vision.c_transforms as vision +import mindspore.dataset.transforms as C +import mindspore.dataset.vision as vision import mindspore.nn as nn from mindspore import Tensor from mindspore.communication.management import init diff --git a/tests/st/model_zoo_tests/transformer/test_transformer.py b/tests/st/model_zoo_tests/transformer/test_transformer.py index 3b7d413647a..b87f27b0b36 100644 --- a/tests/st/model_zoo_tests/transformer/test_transformer.py +++ b/tests/st/model_zoo_tests/transformer/test_transformer.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -25,7 +25,7 @@ from mindspore.train.model import Model from mindspore.train.loss_scale_manager import DynamicLossScaleManager from mindspore.train.callback import Callback import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as deC +import mindspore.dataset.transforms as deC from mindspore import context from easydict import EasyDict as edict from tests.models.official.nlp.transformer.src.transformer_model import TransformerConfig diff --git a/tests/st/model_zoo_tests/yolov3/src/dataset.py b/tests/st/model_zoo_tests/yolov3/src/dataset.py index 386551d0d46..f9078a6e5da 100644 --- a/tests/st/model_zoo_tests/yolov3/src/dataset.py +++ b/tests/st/model_zoo_tests/yolov3/src/dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,14 +22,14 @@ from matplotlib.colors import rgb_to_hsv, hsv_to_rgb from PIL import Image import mindspore.dataset as de from mindspore.mindrecord import FileWriter -import mindspore.dataset.vision.c_transforms as C +import mindspore.dataset.vision as C from src.config import ConfigYOLOV3ResNet18 -iter_cnt = 0 _NUM_BOXES = 50 np.random.seed(1) de.config.set_seed(1) + def preprocess_fn(image, box, is_training): """Preprocess function for dataset.""" config_anchors = [10, 13, 16, 30, 33, 23, 30, 61, 62, 45, 59, 119, 116, 90, 156, 198, 163, 326] @@ -46,7 +46,6 @@ def preprocess_fn(image, box, is_training): num_layers = anchors.shape[0] // 3 anchor_mask = [[6, 7, 8], [3, 4, 5], [0, 1, 2]] true_boxes = np.array(true_boxes, dtype='float32') - # input_shape = np.array([in_shape, in_shape], dtype='int32') input_shape = np.array(in_shape, dtype='int32') boxes_xy = (true_boxes[..., 0:2] + true_boxes[..., 2:4]) // 2. boxes_wh = true_boxes[..., 2:4] - true_boxes[..., 0:2] diff --git a/tests/st/model_zoo_tests/yolov3_darknet53/src/yolo_dataset.py b/tests/st/model_zoo_tests/yolov3_darknet53/src/yolo_dataset.py index 32a1bed44a2..67aa72bf02b 100644 --- a/tests/st/model_zoo_tests/yolov3_darknet53/src/yolo_dataset.py +++ b/tests/st/model_zoo_tests/yolov3_darknet53/src/yolo_dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ import cv2 from PIL import Image from pycocotools.coco import COCO import mindspore.dataset as de -import mindspore.dataset.vision.c_transforms as CV +import mindspore.dataset.vision as CV from src.distributed_sampler import DistributedSampler from src.transforms import reshape_fn, MultiScaleTrans @@ -45,7 +45,7 @@ def has_valid_annotation(anno): # if all boxes have close to zero area, there is no annotation if _has_only_empty_bbox(anno): return False - # keypoints task have a slight different critera for considering + # keypoints task have a slight different criteria for considering # if an annotation is valid if "keypoints" not in anno[0]: return True diff --git a/tests/st/networks/models/bert/bert_performance/test_bert_tdt_lossscale.py b/tests/st/networks/models/bert/bert_performance/test_bert_tdt_lossscale.py index 3d4daf53bc7..ab11c6c7c26 100644 --- a/tests/st/networks/models/bert/bert_performance/test_bert_tdt_lossscale.py +++ b/tests/st/networks/models/bert/bert_performance/test_bert_tdt_lossscale.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ import numpy as np import pytest import mindspore.common.dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.transforms as C from mindspore import context from mindspore import log as logger from mindspore.ops import operations as P diff --git a/tests/st/networks/models/bert/bert_performance/test_bert_thor.py b/tests/st/networks/models/bert/bert_performance/test_bert_thor.py index 2936488e0bf..bff33671ed0 100644 --- a/tests/st/networks/models/bert/bert_performance/test_bert_thor.py +++ b/tests/st/networks/models/bert/bert_performance/test_bert_thor.py @@ -1,4 +1,4 @@ -# Copyright 2020-2021 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -31,7 +31,7 @@ from mindspore.train.serialization import load_checkpoint, load_param_into_net from mindspore.nn.optim import thor from mindspore.train.model import Model from mindspore.train.train_thor import ConvertModelUtils -import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.transforms as C from tests.models.official.nlp.bert.src.bert_for_pre_training import BertNetworkWithLoss, BertTrainOneStepCell from tests.models.official.nlp.bert.src.utils import get_bert_thor_lr, get_bert_thor_damping diff --git a/tests/st/networks/models/bert/bert_precision/test_bert_tdt_lossscale.py b/tests/st/networks/models/bert/bert_precision/test_bert_tdt_lossscale.py index fc35c048745..7099da7f322 100644 --- a/tests/st/networks/models/bert/bert_precision/test_bert_tdt_lossscale.py +++ b/tests/st/networks/models/bert/bert_precision/test_bert_tdt_lossscale.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,7 +21,7 @@ import numpy as np import pytest import mindspore.common.dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.transforms as C from mindspore import context from mindspore import log as logger from mindspore.ops import operations as P diff --git a/tests/st/networks/models/bert/src/dataset.py b/tests/st/networks/models/bert/src/dataset.py index dc3abbbe9c9..6365e8c786c 100644 --- a/tests/st/networks/models/bert/src/dataset.py +++ b/tests/st/networks/models/bert/src/dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ Data operations, will be used in run_pretrain.py import os import mindspore.common.dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.transforms as C from mindspore import log as logger from .config import bert_net_cfg diff --git a/tests/st/networks/models/deeplabv3/src/md_dataset.py b/tests/st/networks/models/deeplabv3/src/md_dataset.py index dfbbc501446..9f324fa063e 100644 --- a/tests/st/networks/models/deeplabv3/src/md_dataset.py +++ b/tests/st/networks/models/deeplabv3/src/md_dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the License); # you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ import numpy as np from PIL import Image import mindspore.dataset as de -import mindspore.dataset.vision.c_transforms as C +import mindspore.dataset.vision as C from .ei_dataset import HwVocRawDataset from .utils import custom_transforms as tr diff --git a/tests/st/networks/models/resnet50/src/dataset.py b/tests/st/networks/models/resnet50/src/dataset.py index 764405f5db2..8109703cb68 100755 --- a/tests/st/networks/models/resnet50/src/dataset.py +++ b/tests/st/networks/models/resnet50/src/dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,8 +18,8 @@ import os import mindspore.common.dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.vision.c_transforms as C -import mindspore.dataset.transforms.c_transforms as C2 +import mindspore.dataset.vision as C +import mindspore.dataset.transforms as C2 def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): diff --git a/tests/st/networks/models/resnet50/src_thor/dataset.py b/tests/st/networks/models/resnet50/src_thor/dataset.py index 0f7c3adbfc1..875bb41c1ee 100644 --- a/tests/st/networks/models/resnet50/src_thor/dataset.py +++ b/tests/st/networks/models/resnet50/src_thor/dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -50,18 +50,18 @@ def create_dataset1(dataset_path, do_train, batch_size=32, train_image_size=224, trans = [] if do_train: trans += [ - ds.vision.c_transforms.RandomCrop((32, 32), (4, 4, 4, 4)), - ds.vision.c_transforms.RandomHorizontalFlip(prob=0.5) + ds.vision.RandomCrop((32, 32), (4, 4, 4, 4)), + ds.vision.RandomHorizontalFlip(prob=0.5) ] trans += [ - ds.vision.c_transforms.Resize((train_image_size, train_image_size)), - ds.vision.c_transforms.Rescale(1.0 / 255.0, 0.0), - ds.vision.c_transforms.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), - ds.vision.c_transforms.HWC2CHW() + ds.vision.Resize((train_image_size, train_image_size)), + ds.vision.Rescale(1.0 / 255.0, 0.0), + ds.vision.Normalize([0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010]), + ds.vision.HWC2CHW() ] - type_cast_op = ds.transforms.c_transforms.TypeCast(ms.int32) + type_cast_op = ds.transforms.TypeCast(ms.int32) data_set = data_set.map(operations=type_cast_op, input_columns="label", num_parallel_workers=get_num_parallel_workers(8)) @@ -117,18 +117,18 @@ def create_dataset2(dataset_path, do_train, batch_size=32, train_image_size=224, # define map operations if do_train: trans = [ - ds.vision.c_transforms.RandomCropDecodeResize(train_image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), - ds.vision.c_transforms.RandomHorizontalFlip(prob=0.5) + ds.vision.RandomCropDecodeResize(train_image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), + ds.vision.RandomHorizontalFlip(prob=0.5) ] else: trans = [ - ds.vision.c_transforms.Decode(), - ds.vision.c_transforms.Resize(256), - ds.vision.c_transforms.CenterCrop(eval_image_size) + ds.vision.Decode(), + ds.vision.Resize(256), + ds.vision.CenterCrop(eval_image_size) ] - trans_norm = [ds.vision.c_transforms.Normalize(mean=mean, std=std), ds.vision.c_transforms.HWC2CHW()] + trans_norm = [ds.vision.Normalize(mean=mean, std=std), ds.vision.HWC2CHW()] - type_cast_op = ds.transforms.c_transforms.TypeCast(ms.int32) + type_cast_op = ds.transforms.TypeCast(ms.int32) data_set = data_set.map(operations=trans, input_columns="image", num_parallel_workers=get_num_parallel_workers(12)) data_set = data_set.map(operations=trans_norm, input_columns="image", diff --git a/tests/st/networks/test_gpu_lenet.py b/tests/st/networks/test_gpu_lenet.py index c2e25ff5b21..2af01ab4f07 100644 --- a/tests/st/networks/test_gpu_lenet.py +++ b/tests/st/networks/test_gpu_lenet.py @@ -1,4 +1,4 @@ -# Copyright 2019 Huawei Technologies Co., Ltd +# Copyright 2019-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,8 +20,8 @@ import pytest import mindspore.context as context import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.vision.c_transforms as CV +import mindspore.dataset.transforms as C +import mindspore.dataset.vision as CV import mindspore.nn as nn from mindspore import Tensor, ParameterTuple from mindspore.common import dtype as mstype diff --git a/tests/st/networks/test_gradient_accumulation.py b/tests/st/networks/test_gradient_accumulation.py index b56d22493c2..5b0580649e4 100644 --- a/tests/st/networks/test_gradient_accumulation.py +++ b/tests/st/networks/test_gradient_accumulation.py @@ -3,8 +3,8 @@ import os import pytest import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as CT -import mindspore.dataset.vision.c_transforms as CV +import mindspore.dataset.transforms as CT +import mindspore.dataset.vision as CV import mindspore.nn as nn from mindspore import ParameterTuple from mindspore import context diff --git a/tests/st/ops/ascend/test_tdt_data_ms.py b/tests/st/ops/ascend/test_tdt_data_ms.py index b11ecbe8e7c..f551b0744d2 100644 --- a/tests/st/ops/ascend/test_tdt_data_ms.py +++ b/tests/st/ops/ascend/test_tdt_data_ms.py @@ -1,4 +1,4 @@ -# Copyright 2019 Huawei Technologies Co., Ltd +# Copyright 2019-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ import numpy as np import mindspore.context as context import mindspore.dataset as ds -import mindspore.dataset.vision.c_transforms as vision +import mindspore.dataset.vision as vision import mindspore.nn as nn from mindspore.common.api import _cell_graph_executor from mindspore.common.tensor import Tensor diff --git a/tests/st/probability/bnn_layers/dataset.py b/tests/st/probability/bnn_layers/dataset.py index 707410257c9..57da307ffa1 100644 --- a/tests/st/probability/bnn_layers/dataset.py +++ b/tests/st/probability/bnn_layers/dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,8 +17,8 @@ Produce the dataset """ import mindspore.dataset as ds -import mindspore.dataset.vision.c_transforms as CV -import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.vision as CV +import mindspore.dataset.transforms as C from mindspore.dataset.vision import Inter from mindspore import dtype as mstype diff --git a/tests/st/probability/dpn/test_gpu_svi_cvae.py b/tests/st/probability/dpn/test_gpu_svi_cvae.py index de232269668..6e312129af8 100644 --- a/tests/st/probability/dpn/test_gpu_svi_cvae.py +++ b/tests/st/probability/dpn/test_gpu_svi_cvae.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ import os from mindspore import dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.vision.c_transforms as CV +import mindspore.dataset.vision as CV import mindspore.nn as nn from mindspore import context, Tensor import mindspore.ops as ops diff --git a/tests/st/probability/dpn/test_gpu_svi_vae.py b/tests/st/probability/dpn/test_gpu_svi_vae.py index 47a9b7c8ad5..842376bf384 100644 --- a/tests/st/probability/dpn/test_gpu_svi_vae.py +++ b/tests/st/probability/dpn/test_gpu_svi_vae.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,7 +16,7 @@ import os from mindspore import dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.vision.c_transforms as CV +import mindspore.dataset.vision as CV import mindspore.nn as nn from mindspore import context, Tensor import mindspore.ops as ops diff --git a/tests/st/probability/dpn/test_gpu_vae_gan.py b/tests/st/probability/dpn/test_gpu_vae_gan.py index ab65aefa1d1..f6b8a6bfdd5 100644 --- a/tests/st/probability/dpn/test_gpu_vae_gan.py +++ b/tests/st/probability/dpn/test_gpu_vae_gan.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,7 +18,7 @@ The VAE interface can be called to construct VAE-GAN network. import os import mindspore.dataset as ds -import mindspore.dataset.vision.c_transforms as CV +import mindspore.dataset.vision as CV import mindspore.nn as nn from mindspore import context import mindspore.ops as ops diff --git a/tests/st/probability/toolbox/test_uncertainty.py b/tests/st/probability/toolbox/test_uncertainty.py index b406ad31cc6..b5e0cb1ff6c 100644 --- a/tests/st/probability/toolbox/test_uncertainty.py +++ b/tests/st/probability/toolbox/test_uncertainty.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +14,8 @@ # ============================================================================ """ test uncertainty toolbox """ import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.vision.c_transforms as CV +import mindspore.dataset.transforms as C +import mindspore.dataset.vision as CV import mindspore.nn as nn from mindspore import context, Tensor from mindspore import dtype as mstype diff --git a/tests/st/probability/transforms/dataset.py b/tests/st/probability/transforms/dataset.py index 707410257c9..57da307ffa1 100644 --- a/tests/st/probability/transforms/dataset.py +++ b/tests/st/probability/transforms/dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,8 +17,8 @@ Produce the dataset """ import mindspore.dataset as ds -import mindspore.dataset.vision.c_transforms as CV -import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.vision as CV +import mindspore.dataset.transforms as C from mindspore.dataset.vision import Inter from mindspore import dtype as mstype diff --git a/tests/st/probability/zhusuan/vae/utils.py b/tests/st/probability/zhusuan/vae/utils.py index 88812c0ec9d..c1227a7e119 100755 --- a/tests/st/probability/zhusuan/vae/utils.py +++ b/tests/st/probability/zhusuan/vae/utils.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,8 +19,8 @@ import numpy as np from mindspore.common import dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.transforms.vision.c_transforms as CV +import mindspore.dataset.transforms as C +import mindspore.dataset.vision as CV from mindspore.dataset.transforms.vision import Inter diff --git a/tests/st/profiler/test_profiler.py b/tests/st/profiler/test_profiler.py index 853c19695b1..275650d9280 100644 --- a/tests/st/profiler/test_profiler.py +++ b/tests/st/profiler/test_profiler.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -24,8 +24,8 @@ from mindspore import dataset as ds from mindspore import nn, Tensor, context from mindspore.nn.metrics import Accuracy from mindspore.nn.optim import Momentum -from mindspore.dataset.transforms import c_transforms as C -from mindspore.dataset.vision import c_transforms as CV +from mindspore.dataset.transforms import transforms as C +from mindspore.dataset.vision import transforms as CV from mindspore.dataset.vision import Inter from mindspore.common import dtype as mstype from mindspore.common.initializer import TruncatedNormal diff --git a/tests/st/ps/full_ps/test_full_ps_lenet.py b/tests/st/ps/full_ps/test_full_ps_lenet.py index 6c58d4d2c9c..05cb483c6d1 100644 --- a/tests/st/ps/full_ps/test_full_ps_lenet.py +++ b/tests/st/ps/full_ps/test_full_ps_lenet.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,8 +18,8 @@ import argparse import mindspore.context as context import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.vision.c_transforms as CV +import mindspore.dataset.transforms as C +import mindspore.dataset.vision as CV import mindspore.nn as nn from mindspore.common import dtype as mstype from mindspore.dataset.vision import Inter diff --git a/tests/st/ps/part_ps/test_ps_embedding_heterogeneous_conv2d_adam.py b/tests/st/ps/part_ps/test_ps_embedding_heterogeneous_conv2d_adam.py index 05a4a6d78c7..9649869728b 100644 --- a/tests/st/ps/part_ps/test_ps_embedding_heterogeneous_conv2d_adam.py +++ b/tests/st/ps/part_ps/test_ps_embedding_heterogeneous_conv2d_adam.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,8 +18,8 @@ import argparse import numpy as np import mindspore.context as context import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.vision.c_transforms as CV +import mindspore.dataset.transforms as C +import mindspore.dataset.vision as CV from mindspore.common import dtype as mstype from mindspore.dataset.vision import Inter from mindspore.common.tensor import Tensor diff --git a/tests/st/pynative/test_pynative_resnet50_ascend.py b/tests/st/pynative/test_pynative_resnet50_ascend.py index b3f5e5e7c48..39185079308 100644 --- a/tests/st/pynative/test_pynative_resnet50_ascend.py +++ b/tests/st/pynative/test_pynative_resnet50_ascend.py @@ -22,8 +22,8 @@ import pytest import mindspore.common.dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.vision.c_transforms as vision +import mindspore.dataset.transforms as C +import mindspore.dataset.vision as vision import mindspore.nn as nn import mindspore.ops.functional as F diff --git a/tests/st/pynative/test_pynative_resnet50_gpu.py b/tests/st/pynative/test_pynative_resnet50_gpu.py index 0a21049d2b9..df20d41a9bd 100644 --- a/tests/st/pynative/test_pynative_resnet50_gpu.py +++ b/tests/st/pynative/test_pynative_resnet50_gpu.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,8 +19,8 @@ import numpy as np import mindspore.common.dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.vision.c_transforms as vision +import mindspore.dataset.transforms as C +import mindspore.dataset.vision as vision import mindspore.nn as nn import mindspore.ops.functional as F diff --git a/tests/st/quantization/lenet_quant/dataset.py b/tests/st/quantization/lenet_quant/dataset.py index df9eecda1fe..164825686b2 100644 --- a/tests/st/quantization/lenet_quant/dataset.py +++ b/tests/st/quantization/lenet_quant/dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,8 +17,8 @@ Produce the dataset """ import mindspore.dataset as ds -import mindspore.dataset.vision.c_transforms as CV -import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.vision as CV +import mindspore.dataset.transforms as C from mindspore.dataset.vision import Inter from mindspore.common import dtype as mstype diff --git a/tests/st/quantization/mobilenetv2_quant/dataset.py b/tests/st/quantization/mobilenetv2_quant/dataset.py index 6dd158f98ca..ee2802fa86a 100644 --- a/tests/st/quantization/mobilenetv2_quant/dataset.py +++ b/tests/st/quantization/mobilenetv2_quant/dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,8 +17,8 @@ from functools import partial import mindspore.dataset as ds import mindspore.common.dtype as mstype -import mindspore.dataset.vision.c_transforms as C -import mindspore.dataset.transforms.c_transforms as C2 +import mindspore.dataset.vision as C +import mindspore.dataset.transforms as C2 def create_dataset(dataset_path, config, repeat_num=1, batch_size=32): diff --git a/tests/st/quantization/resnet50_quant/dataset.py b/tests/st/quantization/resnet50_quant/dataset.py index ec59a50d3d2..69a1f774819 100755 --- a/tests/st/quantization/resnet50_quant/dataset.py +++ b/tests/st/quantization/resnet50_quant/dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,8 +18,8 @@ from functools import partial import mindspore.common.dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C2 -import mindspore.dataset.vision.c_transforms as C +import mindspore.dataset.transforms as C2 +import mindspore.dataset.vision as C def create_dataset(dataset_path, config, repeat_num=1, batch_size=32): diff --git a/tests/st/summary/dataset.py b/tests/st/summary/dataset.py index ecea5061e0e..d253085322a 100644 --- a/tests/st/summary/dataset.py +++ b/tests/st/summary/dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,9 +17,9 @@ import os from mindspore import dataset as ds from mindspore.common import dtype as mstype -from mindspore.dataset.transforms import c_transforms as C +from mindspore.dataset.transforms import transforms as C from mindspore.dataset.vision import Inter -from mindspore.dataset.vision import c_transforms as CV +from mindspore.dataset.vision import transforms as CV def create_mnist_dataset(mode='train', num_samples=2, batch_size=2): diff --git a/tests/st/tbe_networks/resnet_cifar.py b/tests/st/tbe_networks/resnet_cifar.py index 470a1e8daac..fd52dc61d74 100644 --- a/tests/st/tbe_networks/resnet_cifar.py +++ b/tests/st/tbe_networks/resnet_cifar.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,8 +23,8 @@ import mindspore.ops.functional as F from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor from mindspore.train.serialization import load_checkpoint, load_param_into_net import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.vision.c_transforms as vision +import mindspore.dataset.transforms as C +import mindspore.dataset.vision as vision import mindspore.nn as nn from mindspore import Tensor from mindspore import context diff --git a/tests/st/tbe_networks/test_resnet_cifar_1p.py b/tests/st/tbe_networks/test_resnet_cifar_1p.py index d30b315918b..916a20eb208 100644 --- a/tests/st/tbe_networks/test_resnet_cifar_1p.py +++ b/tests/st/tbe_networks/test_resnet_cifar_1p.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,8 +20,8 @@ from resnet import resnet50 import mindspore.common.dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.vision.c_transforms as vision +import mindspore.dataset.transforms as C +import mindspore.dataset.vision as vision import mindspore.nn as nn import mindspore.ops.functional as F from mindspore import Tensor diff --git a/tests/st/tbe_networks/test_resnet_cifar_8p.py b/tests/st/tbe_networks/test_resnet_cifar_8p.py index 66298e5be31..5da14488fb3 100644 --- a/tests/st/tbe_networks/test_resnet_cifar_8p.py +++ b/tests/st/tbe_networks/test_resnet_cifar_8p.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -21,8 +21,8 @@ import numpy as np from resnet import resnet50 import mindspore.common.dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C -import mindspore.dataset.vision.c_transforms as vision +import mindspore.dataset.transforms as C +import mindspore.dataset.vision as vision import mindspore.nn as nn import mindspore.ops.functional as F from mindspore import Tensor diff --git a/tests/ut/python/dataset/test_get_image_num_channels.py b/tests/ut/python/dataset/test_get_image_num_channels.py index 2cd7f0bba88..477fdba5a5b 100644 --- a/tests/ut/python/dataset/test_get_image_num_channels.py +++ b/tests/ut/python/dataset/test_get_image_num_channels.py @@ -12,13 +12,15 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== - +""" +Test MindData vision utility get_image_num_channels +""" import numpy as np import pytest from PIL import Image -import mindspore.dataset.vision.utils as vision -import mindspore.dataset.vision.c_transforms as C +import mindspore.dataset.vision.utils as vision_utils +import mindspore.dataset.vision as vision from mindspore import log as logger @@ -30,8 +32,8 @@ def test_get_image_num_channels_output_array(): """ expect_output = 3 img = np.fromfile("../data/dataset/apple.jpg", dtype=np.uint8) - input_array = C.Decode()(img) - output = vision.get_image_num_channels(input_array) + input_array = vision.Decode()(img) + output = vision_utils.get_image_num_channels(input_array) assert expect_output == output @@ -44,7 +46,7 @@ def test_get_image_num_channels_output_img(): testdata = "../data/dataset/apple.jpg" img = Image.open(testdata) expect_channel = 3 - output_channel = vision.get_image_num_channels(img) + output_channel = vision_utils.get_image_num_channels(img) assert expect_channel == output_channel @@ -58,7 +60,7 @@ def test_get_image_num_channels_invalid_input(): def test_invalid_input(test_name, image, error, error_msg): logger.info("Test get_image_num_channels with wrong params: {0}".format(test_name)) with pytest.raises(error) as error_info: - vision.get_image_num_channels(image) + vision_utils.get_image_num_channels(image) assert error_msg in str(error_info.value) invalid_input = 1 diff --git a/tests/ut/python/dataset/test_get_image_size.py b/tests/ut/python/dataset/test_get_image_size.py index 44094ed33b2..558fc4775b3 100644 --- a/tests/ut/python/dataset/test_get_image_size.py +++ b/tests/ut/python/dataset/test_get_image_size.py @@ -12,52 +12,54 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== - +""" +Test MindData vision utility get_image_size +""" import numpy as np import pytest from PIL import Image -import mindspore.dataset.vision.utils as vision -import mindspore.dataset.vision.c_transforms as C +import mindspore.dataset.vision.utils as vision_utils +import mindspore.dataset.vision as vision from mindspore import log as logger def test_get_image_size_output_array(): """ - Feature: get_image_num_channels array(after Decode array.shape is HWC) - Description: Test get_image_num_channels + Feature: get_image_size + Description: Test get_image_size array Expectation: The returned result is as expected """ expect = [2268, 4032] img = np.fromfile("../data/dataset/apple.jpg", dtype=np.uint8) - input_array = C.Decode()(img) - output = vision.get_image_size(input_array) + input_array = vision.Decode()(img) + output = vision_utils.get_image_size(input_array) assert expect == output -def test_get_image_num_size_output_img(): +def test_get_image_size_output_img(): """ - Feature: get_image_num_channels img(Image.size is [H, W]) - Description: Test get_image_num_channels + Feature: get_image_size + Description: Test get_image_size image (Image.size is [H, W]) Expectation: The returned result is as expected """ expect = [2268, 4032] img = Image.open("../data/dataset/apple.jpg") - output_size = vision.get_image_size(img) + output_size = vision_utils.get_image_size(img) assert expect == output_size -def test_get_image_num_channels_invalid_input(): +def test_get_image_size_invalid_input(): """ - Feature: get_image_num_channels - Description: Test get_image_num_channels invalid input + Feature: get_image_size + Description: Test get_image_size invalid input Expectation: Correct error is raised as expected """ def test_invalid_input(test_name, image, error, error_msg): - logger.info("Test GetImageNumChannels with wrong params: {0}".format(test_name)) + logger.info("Test GetImageSize with wrong params: {0}".format(test_name)) with pytest.raises(error) as error_info: - vision.get_image_size(image) + vision_utils.get_image_size(image) assert error_msg in str(error_info.value) invalid_input = 1 @@ -71,5 +73,5 @@ def test_get_image_num_channels_invalid_input(): if __name__ == "__main__": test_get_image_size_output_array() - test_get_image_num_size_output_img() - test_get_image_num_channels_invalid_input() + test_get_image_size_output_img() + test_get_image_size_invalid_input() diff --git a/tests/ut/python/dataset/test_pyfunc_multiprocess.py b/tests/ut/python/dataset/test_pyfunc_multiprocess.py index 9ecf40f085c..fac6d75c0ae 100644 --- a/tests/ut/python/dataset/test_pyfunc_multiprocess.py +++ b/tests/ut/python/dataset/test_pyfunc_multiprocess.py @@ -80,7 +80,6 @@ def create_dataset_pyop_multiproc(num_parallel_workers=None, max_rowsize=16, bat lambda x: x, vision.HWC2CHW(), vision.RandomErasing(0.9, value='random'), - vision.CutOut(4, 2, is_hwc=False), lambda y: y ] compose_op = transforms.Compose(transforms_list) diff --git a/tests/ut/python/profiler/parser/test_minddata_analyzer.py b/tests/ut/python/profiler/parser/test_minddata_analyzer.py index dc899e289cb..de77029269d 100644 --- a/tests/ut/python/profiler/parser/test_minddata_analyzer.py +++ b/tests/ut/python/profiler/parser/test_minddata_analyzer.py @@ -1,4 +1,4 @@ -# Copyright 2021 Huawei Technologies Co., Ltd +# Copyright 2021-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,7 +22,7 @@ import numpy as np import pytest import mindspore.common.dtype as mstype import mindspore.dataset as ds -import mindspore.dataset.transforms.c_transforms as C +import mindspore.dataset.transforms as transforms import mindspore._c_dataengine as cde from mindspore.profiler.parser.minddata_analyzer import MinddataProfilingAnalyzer @@ -143,14 +143,16 @@ class TestMinddataProfilingAnalyzer: def test_analyze_basic(self, tmp_path): """ - Test MindData profiling analyze summary files exist with basic pipeline. - Also test basic content (subset of keys and values) from the returned summary result. + Feature: MindData Profiling Analyzer + Description: Test MindData profiling analyze summary files exist with basic pipeline. + Also test basic content (subset of keys and values) from the returned summary result. + Expectation: MindData Profiling Analyzer output is as expected """ # Create this basic and common linear pipeline # Generator -> Map -> Batch -> Repeat -> EpochCtrl data1 = ds.GeneratorDataset(self.mysource, ["col1"]) - type_cast_op = C.TypeCast(mstype.int32) + type_cast_op = transforms.TypeCast(mstype.int32) data1 = data1.map(operations=type_cast_op, input_columns="col1") data1 = data1.batch(16) data1 = data1.repeat(2) @@ -206,13 +208,15 @@ class TestMinddataProfilingAnalyzer: def test_analyze_sequential_pipelines_invalid(self, tmp_path): """ - Test invalid scenario in which MinddataProfilingAnalyzer is called for two sequential pipelines. + Feature: MindData Profiling Analyzer + Description: Test invalid scenario in which MinddataProfilingAnalyzer is called for two sequential pipelines. + Expectation: MindData Profiling Analyzer output in each pipeline is as expected """ # Create the pipeline # Generator -> Map -> Batch -> EpochCtrl data1 = ds.GeneratorDataset(self.mysource, ["col1"]) - type_cast_op = C.TypeCast(mstype.int32) + type_cast_op = transforms.TypeCast(mstype.int32) data1 = data1.map(operations=type_cast_op, input_columns="col1") data1 = data1.batch(64) diff --git a/tests/ut/python/train/dataset.py b/tests/ut/python/train/dataset.py index 6ee84e3cc72..2f6b53e693a 100644 --- a/tests/ut/python/train/dataset.py +++ b/tests/ut/python/train/dataset.py @@ -1,4 +1,4 @@ -# Copyright 2020 Huawei Technologies Co., Ltd +# Copyright 2020-2022 Huawei Technologies Co., Ltd # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,13 +17,14 @@ import os from mindspore import dataset as ds from mindspore.common import dtype as mstype -from mindspore.dataset.transforms import c_transforms as C +import mindspore.dataset.transforms as C from mindspore.dataset.vision import Inter -from mindspore.dataset.vision import c_transforms as CV +import mindspore.dataset.vision as CV from mindspore import nn, Tensor from mindspore.common.initializer import Normal from mindspore.ops import operations as P + def create_mnist_dataset(mode='train', num_samples=2, batch_size=2): """create dataset for train or test""" mnist_path = '/home/workspace/mindspore_dataset/mnist' diff --git a/tests/ut/python/utils/test_export.py b/tests/ut/python/utils/test_export.py index ea1732ce362..934e80c434e 100644 --- a/tests/ut/python/utils/test_export.py +++ b/tests/ut/python/utils/test_export.py @@ -1,10 +1,25 @@ +# Copyright 2020-2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test export""" import os import numpy as np import mindspore.nn as nn import mindspore.dataset as ds -import mindspore.dataset.vision.c_transforms as CV -import mindspore.dataset.transforms.c_transforms as CT +import mindspore.dataset.vision as CV +import mindspore.dataset.transforms as CT from mindspore.dataset.vision import Inter from mindspore import context from mindspore.common import dtype as mstype