From a9972a7def05f581ae9fe027b601ba15f7cda32b Mon Sep 17 00:00:00 2001 From: jinyaohui Date: Sat, 29 Aug 2020 10:34:52 +0800 Subject: [PATCH] optim pylint --- tests/st/auto_parallel/test_expand_loss.py | 1 - .../test_resnet50_expand_loss.py | 1 - ...rain_and_test_multinpu_ci_data_parallel.py | 8 +-- .../models/bert/test_bert_graph_kernel.py | 9 ++- tests/st/networks/test_gpu_alexnet.py | 2 - tests/st/networks/test_gpu_lstm.py | 1 - .../ops/ascend/test_aicpu_ops/test_poisson.py | 1 - tests/st/ops/cpu/test_slice_grad_op.py | 5 +- tests/st/ops/cpu/test_slice_op.py | 8 ++- tests/st/ops/custom_ops_tbe/cus_add3.py | 2 + tests/st/summary/test_cpu_summary.py | 2 +- .../python/dataset/test_minddataset_padded.py | 67 ++++++++++++------- tests/ut/python/dataset/test_sampler.py | 3 +- .../ut/python/dataset/test_serdes_dataset.py | 5 +- tests/ut/python/ops/test_control_ops.py | 47 ++++++++----- tests/ut/python/ops/test_math_ops.py | 3 +- tests/ut/python/ops/test_ops.py | 9 +-- tests/ut/python/train/test_amp.py | 11 ++- tests/ut/python/utils/test_initializer.py | 4 +- 19 files changed, 113 insertions(+), 76 deletions(-) diff --git a/tests/st/auto_parallel/test_expand_loss.py b/tests/st/auto_parallel/test_expand_loss.py index efae4f0ba97..2844ff4015f 100644 --- a/tests/st/auto_parallel/test_expand_loss.py +++ b/tests/st/auto_parallel/test_expand_loss.py @@ -13,7 +13,6 @@ # limitations under the License. # ============================================================================ import os -import pytest def test_expand_loss(): diff --git a/tests/st/auto_parallel/test_resnet50_expand_loss.py b/tests/st/auto_parallel/test_resnet50_expand_loss.py index 8400a2d855d..0fe58302301 100644 --- a/tests/st/auto_parallel/test_resnet50_expand_loss.py +++ b/tests/st/auto_parallel/test_resnet50_expand_loss.py @@ -13,7 +13,6 @@ # limitations under the License. # ============================================================================ import os -import pytest def test_expand_loss(): diff --git a/tests/st/model_zoo_tests/wide_and_deep/train_and_test_multinpu_ci_data_parallel.py b/tests/st/model_zoo_tests/wide_and_deep/train_and_test_multinpu_ci_data_parallel.py index eaae9dfff35..b76f9d28ef2 100644 --- a/tests/st/model_zoo_tests/wide_and_deep/train_and_test_multinpu_ci_data_parallel.py +++ b/tests/st/model_zoo_tests/wide_and_deep/train_and_test_multinpu_ci_data_parallel.py @@ -14,7 +14,6 @@ # ============================================================================ """train_multinpu.""" - import os import sys import numpy as np @@ -35,7 +34,6 @@ context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL, mirr init() - def get_WideDeep_net(config): WideDeep_net = WideDeepModel(config) loss_net = NetWithLossClass(WideDeep_net, config) @@ -48,6 +46,7 @@ class ModelBuilder(): """ ModelBuilder """ + def __init__(self): pass @@ -101,14 +100,13 @@ def test_train_eval(): print("=====" * 5 + "model.eval() initialized: {}".format(out)) model.train(epochs, ds_train, callbacks=[TimeMonitor(ds_train.get_dataset_size()), eval_callback, callback, ckpoint_cb]) - expect_out0 = [0.792634,0.799862,0.803324] - expect_out6 = [0.796580,0.803908,0.807262] + expect_out0 = [0.792634, 0.799862, 0.803324] + expect_out6 = [0.796580, 0.803908, 0.807262] if get_rank() == 0: assert np.allclose(eval_callback.eval_values, expect_out0) if get_rank() == 6: assert np.allclose(eval_callback.eval_values, expect_out6) - if __name__ == "__main__": test_train_eval() diff --git a/tests/st/networks/models/bert/test_bert_graph_kernel.py b/tests/st/networks/models/bert/test_bert_graph_kernel.py index 576c7a32c5c..23075ddcbdc 100644 --- a/tests/st/networks/models/bert/test_bert_graph_kernel.py +++ b/tests/st/networks/models/bert/test_bert_graph_kernel.py @@ -16,8 +16,10 @@ """train bert network without lossscale""" import os -import pytest + import numpy as np +from src.bert_for_pre_training import BertNetworkWithLoss, BertTrainOneStepWithLossScaleCell +from src.bert_model import BertConfig import mindspore.common.dtype as mstype import mindspore.dataset.engine.datasets as de @@ -25,14 +27,11 @@ import mindspore.dataset.transforms.c_transforms as C from mindspore import context from mindspore import log as logger from mindspore.common.tensor import Tensor +from mindspore.nn import learning_rate_schedule as lr_schedules from mindspore.nn.optim import Lamb from mindspore.train.callback import Callback from mindspore.train.loss_scale_manager import DynamicLossScaleManager from mindspore.train.model import Model -from mindspore.nn import learning_rate_schedule as lr_schedules -from src.bert_for_pre_training import BertNetworkWithLoss, BertTrainOneStepWithLossScaleCell -from src.bert_model import BertConfig - DATA_DIR = ["/home/workspace/mindspore_dataset/bert/example/examples.tfrecord"] SCHEMA_DIR = "/home/workspace/mindspore_dataset/bert/example/datasetSchema.json" diff --git a/tests/st/networks/test_gpu_alexnet.py b/tests/st/networks/test_gpu_alexnet.py index 7a55006571e..a037d1cef10 100644 --- a/tests/st/networks/test_gpu_alexnet.py +++ b/tests/st/networks/test_gpu_alexnet.py @@ -23,10 +23,8 @@ import pytest import mindspore.context as context import mindspore.nn as nn from mindspore import Tensor -from mindspore.common.initializer import initializer from mindspore.nn import TrainOneStepCell, WithLossCell from mindspore.nn.optim import Momentum -from mindspore.ops import operations as P context.set_context(mode=context.GRAPH_MODE, device_target="GPU") diff --git a/tests/st/networks/test_gpu_lstm.py b/tests/st/networks/test_gpu_lstm.py index bc59b7e3872..c1146ed0924 100644 --- a/tests/st/networks/test_gpu_lstm.py +++ b/tests/st/networks/test_gpu_lstm.py @@ -21,7 +21,6 @@ import mindspore.nn as nn from mindspore import Tensor from mindspore.common.initializer import initializer from mindspore.common.parameter import Parameter -from mindspore.nn import Dense from mindspore.nn import TrainOneStepCell, WithLossCell from mindspore.nn.optim import Momentum from mindspore.ops import operations as P diff --git a/tests/st/ops/ascend/test_aicpu_ops/test_poisson.py b/tests/st/ops/ascend/test_aicpu_ops/test_poisson.py index 7720d303d68..05fefb486c9 100644 --- a/tests/st/ops/ascend/test_aicpu_ops/test_poisson.py +++ b/tests/st/ops/ascend/test_aicpu_ops/test_poisson.py @@ -18,7 +18,6 @@ import mindspore.context as context import mindspore.nn as nn from mindspore import Tensor from mindspore.ops import operations as P -from mindspore.common import dtype as mstype context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") diff --git a/tests/st/ops/cpu/test_slice_grad_op.py b/tests/st/ops/cpu/test_slice_grad_op.py index a3d56f0f052..4b4dee794e2 100644 --- a/tests/st/ops/cpu/test_slice_grad_op.py +++ b/tests/st/ops/cpu/test_slice_grad_op.py @@ -54,6 +54,7 @@ def test_slice_grad(): print("output:\n", output) assert (output.asnumpy() == expect).all() + class SliceGrad2(nn.Cell): def __init__(self): super(SliceGrad2, self).__init__() @@ -62,6 +63,7 @@ class SliceGrad2(nn.Cell): def construct(self, dy, x): return self.slicegrad(dy, x, (0, 1, 0), (2, 2, 2)) + @pytest.mark.level0 @pytest.mark.platform_x86_cpu @pytest.mark.env_onecard @@ -71,10 +73,11 @@ def test_slice_grad2(): grad = SliceGrad2() output = grad(dy, x) print("output:\n", output) - expect = [[[0., 0.], [2., 3.], [4., 5.]], + expect = [[[0., 0.], [2., 3.], [4., 5.]], [[0., 0.], [8., 9.], [10., 11.]]] assert (output.asnumpy() == expect).all() + if __name__ == '__main__': test_slice_grad() test_slice_grad2() diff --git a/tests/st/ops/cpu/test_slice_op.py b/tests/st/ops/cpu/test_slice_op.py index e927c31689e..bf35cf4a07a 100644 --- a/tests/st/ops/cpu/test_slice_op.py +++ b/tests/st/ops/cpu/test_slice_op.py @@ -21,10 +21,10 @@ import mindspore.nn as nn from mindspore import Tensor from mindspore.common import dtype as mstype from mindspore.ops import operations as P -from mindspore.ops.operations import _grad_ops as G context.set_context(mode=context.GRAPH_MODE, device_target='CPU') + class Slice(nn.Cell): def __init__(self): super(Slice, self).__init__() @@ -33,6 +33,7 @@ class Slice(nn.Cell): def construct(self, x): return self.slice(x, (0, 1, 0), (2, 1, 3)) + @pytest.mark.level0 @pytest.mark.platform_x86_cpu @pytest.mark.env_onecard @@ -47,6 +48,7 @@ def test_slice(): print("output:\n", output) assert (output.asnumpy() == expect).all() + class Slice2(nn.Cell): def __init__(self): super(Slice2, self).__init__() @@ -55,12 +57,13 @@ class Slice2(nn.Cell): def construct(self, x): return self.slice(x, (1, 0, 0), (1, 2, 3)) + @pytest.mark.level0 @pytest.mark.platform_x86_cpu @pytest.mark.env_onecard def test_slice2(): x = Tensor(np.arange(3 * 2 * 3).reshape(3, 2, 3), mstype.float32) - expect = [[[6., 7., 8.], + expect = [[[6., 7., 8.], [9., 10., 11.]]] slice_op = Slice2() @@ -68,6 +71,7 @@ def test_slice2(): print("output:\n", output) assert (output.asnumpy() == expect).all() + if __name__ == '__main__': test_slice() test_slice2() diff --git a/tests/st/ops/custom_ops_tbe/cus_add3.py b/tests/st/ops/custom_ops_tbe/cus_add3.py index a534be3eae4..80070764a8b 100644 --- a/tests/st/ops/custom_ops_tbe/cus_add3.py +++ b/tests/st/ops/custom_ops_tbe/cus_add3.py @@ -14,9 +14,11 @@ # ============================================================================ from mindspore.ops import prim_attr_register, PrimitiveWithInfer + # sum = input1 + input2 + const_bias class CusAdd3(PrimitiveWithInfer): """Custom add3 definition""" + @prim_attr_register def __init__(self, const_bias=0.0): self.init_prim_io_names(inputs=['input1', 'input2'], outputs=['sum3']) diff --git a/tests/st/summary/test_cpu_summary.py b/tests/st/summary/test_cpu_summary.py index e2f6e8f616f..52704b11b11 100644 --- a/tests/st/summary/test_cpu_summary.py +++ b/tests/st/summary/test_cpu_summary.py @@ -24,8 +24,8 @@ import mindspore.context as context import mindspore.nn as nn from mindspore import Tensor from mindspore.ops import operations as P -from tests.summary_utils import SummaryReader from mindspore.train.summary.summary_record import SummaryRecord +from tests.summary_utils import SummaryReader context.set_context(mode=context.GRAPH_MODE, device_target='CPU') diff --git a/tests/ut/python/dataset/test_minddataset_padded.py b/tests/ut/python/dataset/test_minddataset_padded.py index a05879ab019..aeea897f6c2 100644 --- a/tests/ut/python/dataset/test_minddataset_padded.py +++ b/tests/ut/python/dataset/test_minddataset_padded.py @@ -16,17 +16,15 @@ This is the test module for mindrecord """ import collections -import json -import numpy as np import os -import pytest import re import string +import numpy as np +import pytest + import mindspore.dataset as ds -import mindspore.dataset.transforms.vision.c_transforms as vision from mindspore import log as logger -from mindspore.dataset.transforms.vision import Inter from mindspore.mindrecord import FileWriter FILES_NUM = 4 @@ -52,9 +50,9 @@ def add_and_remove_cv_file(): writer = FileWriter(CV_FILE_NAME, FILES_NUM) data = get_data(CV_DIR_NAME) cv_schema_json = {"id": {"type": "int32"}, - "file_name": {"type": "string"}, - "label": {"type": "int32"}, - "data": {"type": "bytes"}} + "file_name": {"type": "string"}, + "label": {"type": "int32"}, + "data": {"type": "bytes"}} writer.add_schema(cv_schema_json, "img_schema") writer.add_index(["file_name", "label"]) writer.write_raw_data(data) @@ -85,14 +83,14 @@ def add_and_remove_nlp_file(): writer = FileWriter(NLP_FILE_NAME, FILES_NUM) data = [x for x in get_nlp_data(NLP_FILE_POS, NLP_FILE_VOCAB, 10)] nlp_schema_json = {"id": {"type": "string"}, "label": {"type": "int32"}, - "rating": {"type": "float32"}, - "input_ids": {"type": "int64", - "shape": [-1]}, - "input_mask": {"type": "int64", - "shape": [1, -1]}, - "segment_ids": {"type": "int64", - "shape": [2, -1]} - } + "rating": {"type": "float32"}, + "input_ids": {"type": "int64", + "shape": [-1]}, + "input_mask": {"type": "int64", + "shape": [1, -1]}, + "segment_ids": {"type": "int64", + "shape": [2, -1]} + } writer.set_header_size(1 << 14) writer.set_page_size(1 << 15) writer.add_schema(nlp_schema_json, "nlp_schema") @@ -110,6 +108,7 @@ def add_and_remove_nlp_file(): os.remove("{}".format(x)) os.remove("{}.db".format(x)) + def test_cv_minddataset_reader_basic_padded_samples(add_and_remove_cv_file): """tutorial for cv minderdataset.""" columns_list = ["label", "file_name", "data"] @@ -130,7 +129,7 @@ def test_cv_minddataset_reader_basic_padded_samples(add_and_remove_cv_file): if item['label'] == -1: num_padded_iter += 1 assert item['file_name'] == bytes(padded_sample['file_name'], - encoding='utf8') + encoding='utf8') assert item['label'] == padded_sample['label'] assert (item['data'] == np.array(list(padded_sample['data']))).all() num_iter += 1 @@ -177,6 +176,7 @@ def test_cv_minddataset_partition_padded_samples(add_and_remove_cv_file): partitions(5, 5, 3) partitions(9, 8, 2) + def test_cv_minddataset_partition_padded_samples_multi_epoch(add_and_remove_cv_file): """tutorial for cv minddataset.""" columns_list = ["data", "file_name", "label"] @@ -248,6 +248,7 @@ def test_cv_minddataset_partition_padded_samples_multi_epoch(add_and_remove_cv_f partitions(5, 5, 3) partitions(9, 8, 2) + def test_cv_minddataset_partition_padded_samples_no_dividsible(add_and_remove_cv_file): """tutorial for cv minddataset.""" columns_list = ["data", "file_name", "label"] @@ -273,6 +274,7 @@ def test_cv_minddataset_partition_padded_samples_no_dividsible(add_and_remove_cv with pytest.raises(RuntimeError): partitions(4, 1) + def test_cv_minddataset_partition_padded_samples_dataset_size_no_divisible(add_and_remove_cv_file): columns_list = ["data", "file_name", "label"] @@ -291,8 +293,10 @@ def test_cv_minddataset_partition_padded_samples_dataset_size_no_divisible(add_a num_padded=num_padded) with pytest.raises(RuntimeError): data_set.get_dataset_size() == 3 + partitions(4, 1) + def test_cv_minddataset_partition_padded_samples_no_equal_column_list(add_and_remove_cv_file): columns_list = ["data", "file_name", "label"] @@ -314,9 +318,11 @@ def test_cv_minddataset_partition_padded_samples_no_equal_column_list(add_and_re logger.info("-------------- len(item[data]): {} ------------------------".format(len(item["data"]))) logger.info("-------------- item[data]: {} -----------------------------".format(item["data"])) logger.info("-------------- item[file_name]: {} ------------------------".format(item["file_name"])) + with pytest.raises(Exception, match="padded_sample cannot match columns_list."): partitions(4, 2) + def test_cv_minddataset_partition_padded_samples_no_column_list(add_and_remove_cv_file): data = get_data(CV_DIR_NAME) padded_sample = data[0] @@ -336,9 +342,11 @@ def test_cv_minddataset_partition_padded_samples_no_column_list(add_and_remove_c logger.info("-------------- len(item[data]): {} ------------------------".format(len(item["data"]))) logger.info("-------------- item[data]: {} -----------------------------".format(item["data"])) logger.info("-------------- item[file_name]: {} ------------------------".format(item["file_name"])) + with pytest.raises(Exception, match="padded_sample is specified and requires columns_list as well."): partitions(4, 2) + def test_cv_minddataset_partition_padded_samples_no_num_padded(add_and_remove_cv_file): columns_list = ["data", "file_name", "label"] data = get_data(CV_DIR_NAME) @@ -357,9 +365,11 @@ def test_cv_minddataset_partition_padded_samples_no_num_padded(add_and_remove_cv logger.info("-------------- len(item[data]): {} ------------------------".format(len(item["data"]))) logger.info("-------------- item[data]: {} -----------------------------".format(item["data"])) logger.info("-------------- item[file_name]: {} ------------------------".format(item["file_name"])) + with pytest.raises(Exception, match="padded_sample is specified and requires num_padded as well."): partitions(4, 2) + def test_cv_minddataset_partition_padded_samples_no_padded_samples(add_and_remove_cv_file): columns_list = ["data", "file_name", "label"] data = get_data(CV_DIR_NAME) @@ -378,18 +388,18 @@ def test_cv_minddataset_partition_padded_samples_no_padded_samples(add_and_remov logger.info("-------------- len(item[data]): {} ------------------------".format(len(item["data"]))) logger.info("-------------- item[data]: {} -----------------------------".format(item["data"])) logger.info("-------------- item[file_name]: {} ------------------------".format(item["file_name"])) + with pytest.raises(Exception, match="num_padded is specified but padded_sample is not."): partitions(4, 2) - def test_nlp_minddataset_reader_basic_padded_samples(add_and_remove_nlp_file): columns_list = ["input_ids", "id", "rating"] data = [x for x in get_nlp_data(NLP_FILE_POS, NLP_FILE_VOCAB, 10)] padded_sample = data[0] padded_sample['id'] = "-1" - padded_sample['input_ids'] = np.array([-1,-1,-1,-1], dtype=np.int64) + padded_sample['input_ids'] = np.array([-1, -1, -1, -1], dtype=np.int64) padded_sample['rating'] = 1.0 num_readers = 4 @@ -406,7 +416,9 @@ def test_nlp_minddataset_reader_basic_padded_samples(add_and_remove_nlp_file): for item in data_set.create_dict_iterator(): logger.info("-------------- item[id]: {} ------------------------".format(item["id"])) logger.info("-------------- item[rating]: {} --------------------".format(item["rating"])) - logger.info("-------------- item[input_ids]: {}, shape: {} -----------------".format(item["input_ids"], item["input_ids"].shape)) + logger.info("-------------- item[input_ids]: {}, shape: {} -----------------".format( + item["input_ids"], + item["input_ids"].shape)) if item['id'] == bytes('-1', encoding='utf-8'): num_padded_iter += 1 assert item['id'] == bytes(padded_sample['id'], encoding='utf-8') @@ -420,13 +432,14 @@ def test_nlp_minddataset_reader_basic_padded_samples(add_and_remove_nlp_file): partitions(5, 5, 3) partitions(9, 8, 2) + def test_nlp_minddataset_reader_basic_padded_samples_multi_epoch(add_and_remove_nlp_file): columns_list = ["input_ids", "id", "rating"] data = [x for x in get_nlp_data(NLP_FILE_POS, NLP_FILE_VOCAB, 10)] padded_sample = data[0] padded_sample['id'] = "-1" - padded_sample['input_ids'] = np.array([-1,-1,-1,-1], dtype=np.int64) + padded_sample['input_ids'] = np.array([-1, -1, -1, -1], dtype=np.int64) padded_sample['rating'] = 1.0 num_readers = 4 repeat_size = 3 @@ -451,7 +464,9 @@ def test_nlp_minddataset_reader_basic_padded_samples_multi_epoch(add_and_remove_ for item in data_set.create_dict_iterator(): logger.info("-------------- item[id]: {} ------------------------".format(item["id"])) logger.info("-------------- item[rating]: {} --------------------".format(item["rating"])) - logger.info("-------------- item[input_ids]: {}, shape: {} -----------------".format(item["input_ids"], item["input_ids"].shape)) + logger.info("-------------- item[input_ids]: {}, shape: {} -----------------".format( + item["input_ids"], + item["input_ids"].shape)) if item['id'] == bytes('-1', encoding='utf-8'): num_padded_iter += 1 assert item['id'] == bytes(padded_sample['id'], encoding='utf-8') @@ -488,7 +503,7 @@ def test_nlp_minddataset_reader_basic_padded_samples_check_whole_reshuffle_resul padded_sample = {} padded_sample['id'] = "-1" - padded_sample['input_ids'] = np.array([-1,-1,-1,-1], dtype=np.int64) + padded_sample['input_ids'] = np.array([-1, -1, -1, -1], dtype=np.int64) padded_sample['rating'] = 1.0 num_readers = 4 repeat_size = 3 @@ -512,14 +527,15 @@ def test_nlp_minddataset_reader_basic_padded_samples_check_whole_reshuffle_resul logger.info("-------------- item[id]: {} ------------------------".format(item["id"])) logger.info("-------------- item[rating]: {} --------------------".format(item["rating"])) logger.info("-------------- item[input_ids]: {}, shape: {} -----------------" - .format(item["input_ids"], item["input_ids"].shape)) + .format(item["input_ids"], item["input_ids"].shape)) if item['id'] == bytes('-1', encoding='utf-8'): num_padded_iter += 1 assert item['id'] == bytes(padded_sample['id'], encoding='utf-8') assert (item['input_ids'] == padded_sample['input_ids']).all() assert (item['rating'] == padded_sample['rating']).all() # save epoch result - epoch_result[partition_id][int(inner_num_iter / dataset_size)][inner_num_iter % dataset_size] = item["id"] + epoch_result[partition_id][int(inner_num_iter / dataset_size)][inner_num_iter % dataset_size] = item[ + "id"] num_iter += 1 inner_num_iter += 1 assert epoch_result[partition_id][0] not in (epoch_result[partition_id][1], epoch_result[partition_id][2]) @@ -651,6 +667,7 @@ def inputs(vectors, maxlen=50): segment = [0] * maxlen return input_, mask, segment + if __name__ == '__main__': test_cv_minddataset_reader_basic_padded_samples(add_and_remove_cv_file) test_cv_minddataset_partition_padded_samples(add_and_remove_cv_file) diff --git a/tests/ut/python/dataset/test_sampler.py b/tests/ut/python/dataset/test_sampler.py index a7ec89c2092..e8ce43beec7 100644 --- a/tests/ut/python/dataset/test_sampler.py +++ b/tests/ut/python/dataset/test_sampler.py @@ -216,6 +216,7 @@ def test_sampler_chain(): assert test_config(5, 3) == [3] assert test_config(5, 4) == [4] + def test_add_sampler_invalid_input(): manifest_file = "../data/dataset/testManifestData/test5trainimgs.json" _ = {(172876, 0): 0, (54214, 0): 1, (54214, 1): 2, (173673, 0): 3, (64631, 1): 4} @@ -231,7 +232,7 @@ def test_add_sampler_invalid_input(): sampler = ds.SequentialSampler() with pytest.raises(ValueError) as info: - data2 = ds.ManifestDataset(manifest_file, sampler=sampler, num_samples=20) + data2 = ds.ManifestDataset(manifest_file, sampler=sampler, num_samples=20) assert "Conflicting arguments during sampler assignments" in str(info.value) diff --git a/tests/ut/python/dataset/test_serdes_dataset.py b/tests/ut/python/dataset/test_serdes_dataset.py index d041f44bc18..e8a5bbafa2a 100644 --- a/tests/ut/python/dataset/test_serdes_dataset.py +++ b/tests/ut/python/dataset/test_serdes_dataset.py @@ -19,7 +19,10 @@ import filecmp import glob import json import os + import numpy as np +from test_minddataset_sampler import add_and_remove_cv_file, get_data, CV_DIR_NAME, CV_FILE_NAME +from util import config_get_set_num_parallel_workers import mindspore.dataset as ds import mindspore.dataset.transforms.c_transforms as c @@ -27,8 +30,6 @@ import mindspore.dataset.transforms.vision.c_transforms as vision from mindspore import log as logger from mindspore.dataset.transforms.vision import Inter -from test_minddataset_sampler import add_and_remove_cv_file, get_data, CV_DIR_NAME, CV_FILE_NAME -from util import config_get_set_num_parallel_workers def test_imagefolder(remove_json_files=True): """ diff --git a/tests/ut/python/ops/test_control_ops.py b/tests/ut/python/ops/test_control_ops.py index 00c653ce880..c7ce5b1204f 100644 --- a/tests/ut/python/ops/test_control_ops.py +++ b/tests/ut/python/ops/test_control_ops.py @@ -29,7 +29,6 @@ from mindspore.common import ms_function context.set_context(mode=context.GRAPH_MODE) - grad_by_list = C.GradOperation(get_by_list=True) grad_all = C.GradOperation(get_all=True) grad_all_with_sens = C.GradOperation(get_all=True, sens_param=True) @@ -123,6 +122,7 @@ def test_if_none(): net = Net(z) assert np.all(net(x, y).asnumpy() == y.asnumpy()) + def test_if_str_is_not_none_right(): class Net(nn.Cell): def __init__(self, z: str): @@ -455,8 +455,10 @@ def test_parser_switch_layer_switch_in_bprop(): super(OneInputBprop, self).__init__() self.op = P.ReLU() self.funcs = funcs + def construct(self, i, x): - return self.op(x) + return self.op(x) + def bprop(self, i, x, out, dout): return i, self.funcs[i](x, dout) @@ -475,6 +477,7 @@ def test_parser_switch_layer_switch_in_bprop(): def construct(self, x, y): return self.op(x, y) + func1 = Add() func2 = Mul() funcs = (func1, func2) @@ -572,6 +575,7 @@ def test_switch_layer_env_eliminate(): weights = self.weights grad = self.grad_op(self.net, weights)(x, index) return grad + net = Net() net2 = NetGrad(net) x = Tensor(np.ones((3, 1, 12, 12)), ms.float32) @@ -601,6 +605,7 @@ def test_switch_layer_single_layer(): weights = self.weights grad = self.grad_op(self.net, weights)(x, index) return grad + net = Net() net2 = NetGrad(net) x = Tensor(np.ones((3, 1, 12, 12)), ms.float32) @@ -638,6 +643,7 @@ def test_if_nested_compile(): else: res = self.squre(self.value) return res + x = Tensor(1.0, dtype=ms.float32) y = Tensor(2.0, dtype=ms.float32) net = Net() @@ -660,6 +666,7 @@ def test_if_inside_for(): else: res = res - y return res + c1 = Tensor(1, dtype=ms.int32) c2 = Tensor(1, dtype=ms.int32) net = Net() @@ -671,6 +678,7 @@ def test_while_in_while(): c2 = Tensor(2, dtype=ms.int32) c3 = Tensor(3, dtype=ms.int32) c4 = Tensor(4, dtype=ms.int32) + @ms_function def while_in_while(x, y, z, u): out = c4 @@ -683,6 +691,7 @@ def test_while_in_while(): out = out + 3 return out + while_in_while(c1, c2, c3, c4) @@ -692,6 +701,7 @@ def test_tensor_cond(): super(Net, self).__init__() self.t = Tensor(np.array(0, np.bool)) self.t1 = Tensor(np.array([True], np.bool)) + def construct(self, x, y): t = 0 if self.t: @@ -703,18 +713,19 @@ def test_tensor_cond(): else: t = t + x * y return t - - + x = Tensor(np.ones([6, 8, 10], np.int32)) y = Tensor(np.ones([6, 8, 10], np.int32)) net = Net() out = net(x, y) + def test_tensor_cond_exception(): class Net(nn.Cell): def __init__(self): super(Net, self).__init__() self.t = Tensor(np.array([True, False], np.bool)) + def construct(self, x, y): t = 0 if self.t: @@ -722,19 +733,20 @@ def test_tensor_cond_exception(): else: t = t - x / y return t - - + x = Tensor(np.ones([6, 8, 10], np.int32)) y = Tensor(np.ones([6, 8, 10], np.int32)) net = Net() with pytest.raises(ValueError): out = net(x, y) + def test_while_scalar(): class Net(nn.Cell): def __init__(self): super(Net, self).__init__() self.x = 10 + def construct(self, x, y): i = 0 t = 0 @@ -742,17 +754,20 @@ def test_while_scalar(): t = t + x + y i = i + 1 return t + net = Net() x = Tensor(np.ones([6, 8, 10], np.int32)) y = Tensor(np.ones([6, 8, 10], np.int32)) out = net(x, y) + def test_while_tensor(): class Net(nn.Cell): def __init__(self): super(Net, self).__init__() self.t = Tensor(np.ones([6, 8, 10], np.int32)) self.count = Tensor(np.array([10], np.int32)) + def construct(self, x, y): i = 0 t = self.t @@ -760,6 +775,7 @@ def test_while_tensor(): t = t + x + y i = i + 1 return t + net = Net() x = Tensor(np.ones([6, 8, 10], np.int32)) y = Tensor(np.ones([6, 8, 10], np.int32)) @@ -770,7 +786,7 @@ def test_large_for_loop(): class Net(nn.Cell): def __init__(self): super(Net, self).__init__() - self.flatten = P.ReLU() #nn.Flatten() + self.flatten = P.ReLU() # nn.Flatten() def construct(self, x): for elem in range(1, 1900): @@ -791,7 +807,7 @@ def test_large_for_loop_with_continue_break(): class Net(nn.Cell): def __init__(self): super(Net, self).__init__() - self.flatten = P.ReLU() #nn.Flatten() + self.flatten = P.ReLU() # nn.Flatten() def construct(self, x): idx = 0 @@ -854,7 +870,7 @@ def test_tensor_all_construct_lack_branch(): if input1.all(): return self.logicaland(input1, input2) while input1.any(): - return self.logicalor(input1, input2) + return self.logicalor(input1, input2) # NOTICE: here missing return statement, default return None input_np_1 = np.random.choice([True], size=(2, 3, 4, 5)) @@ -891,28 +907,29 @@ def test_parser_switch_layer_func_primitive(): def test_recursive_call(): class Net(nn.Cell): """ Net definition """ - + def __init__(self): super(Net, self).__init__() self.fc = nn.Dense(10, 10) # padding=0 - #self.net2 = Net2() - + # self.net2 = Net2() + def construct(self, x): net2 = Net2() x = net2(x) out = self.fc(x) return out - + class Net2(nn.Cell): def __init__(self): super(Net2, self).__init__() self.net = Net() self.fc = nn.Dense(10, 10) + def construct(self, x): x = self.net(x) out = self.fc(x) return out - + context.set_context(mode=context.GRAPH_MODE, save_graphs=False) old_max_call_depth = context.get_context('max_call_depth') context.set_context(max_call_depth=80) @@ -949,7 +966,6 @@ def test_switch_layer_shape_join_failed(): funcs = (func1, func2) - net = AddFuncNet(funcs, func3) inp = Tensor(np.random.randn(2, 3, 4, 5).astype(np.float32)) @@ -980,7 +996,6 @@ def test_switch_layer_dtype_join_failed(): x = self.op(x) return x - func1 = nn.ReLU() func2 = Cast(mstype.int32) funcs = (func1, func2) diff --git a/tests/ut/python/ops/test_math_ops.py b/tests/ut/python/ops/test_math_ops.py index 1113d9eeb60..0c84bd1ef5f 100755 --- a/tests/ut/python/ops/test_math_ops.py +++ b/tests/ut/python/ops/test_math_ops.py @@ -14,8 +14,8 @@ # ============================================================================ """ test math ops """ import functools + import numpy as np -import pytest import mindspore as ms import mindspore.context as context @@ -31,6 +31,7 @@ from ....mindspore_test_framework.pipeline.forward.compile_forward \ import pipeline_for_compile_forward_ge_graph_for_case_by_case_config from ....mindspore_test_framework.pipeline.forward.verify_exception \ import pipeline_for_verify_exception_for_case_by_case_config + context.set_context(mode=context.GRAPH_MODE) # pylint: disable=W0613 diff --git a/tests/ut/python/ops/test_ops.py b/tests/ut/python/ops/test_ops.py index c794ed9c366..fa2d18b3bb5 100755 --- a/tests/ut/python/ops/test_ops.py +++ b/tests/ut/python/ops/test_ops.py @@ -35,7 +35,6 @@ from ....mindspore_test_framework.pipeline.gradient.compile_gradient \ import pipeline_for_compile_grad_ge_graph_for_case_by_case_config from ....ops_common import convert - grad_all_with_sens = C.GradOperation(get_all=True, sens_param=True) @@ -266,6 +265,7 @@ class ScatterNdSub(nn.Cell): out = self.scatter_nd_sub(self.ref, indices, updates) return out + class ScatterNdAdd(nn.Cell): """ScatterNdAdd net definition""" @@ -311,7 +311,7 @@ class ScatterDiv(nn.Cell): def __init__(self, ref_shape, dtype=np.float32, use_locking=False): super(ScatterDiv, self).__init__() self.scatter_div = P.ScatterDiv(use_locking) - self.ref = Parameter(Tensor(np.ones(ref_shape, dtype)*10), name="ref") + self.ref = Parameter(Tensor(np.ones(ref_shape, dtype) * 10), name="ref") def construct(self, indices, updates): out = self.scatter_div(self.ref, indices, updates) @@ -633,7 +633,7 @@ class CTCGreedyDecoderNet(nn.Cell): self.assert_op = P.Assert(300) def construct(self, inputs, sequence_length): - out = self.ctc_greedy_decoder(inputs,sequence_length) + out = self.ctc_greedy_decoder(inputs, sequence_length) self.assert_op(True, (out[0], out[1], out[2], out[3])) return out[2] @@ -711,12 +711,13 @@ class BasicLSTMCellNet(nn.Cell): def construct(self, x, h, c, w, b): return self.lstm(x, h, c, w, b) + class EditDistance(nn.Cell): def __init__(self, hypothesis_shape, truth_shape, normalize=True): super(EditDistance, self).__init__() self.edit_distance = P.EditDistance(normalize) self.hypothesis_shape = hypothesis_shape - self.truth_shape =truth_shape + self.truth_shape = truth_shape def construct(self, hypothesis_indices, hypothesis_values, truth_indices, truth_values): return self.edit_distance(hypothesis_indices, hypothesis_values, self.hypothesis_shape, diff --git a/tests/ut/python/train/test_amp.py b/tests/ut/python/train/test_amp.py index 0cba6bfa082..074056f3267 100644 --- a/tests/ut/python/train/test_amp.py +++ b/tests/ut/python/train/test_amp.py @@ -20,13 +20,11 @@ import mindspore.context as context from mindspore import Tensor from mindspore import amp from mindspore import nn -from mindspore.train import Model -from mindspore.context import ParallelMode -from mindspore.common import dtype as mstype -from ....dataset_mock import MindData -from mindspore.parallel._auto_parallel_context import auto_parallel_context from mindspore.communication.management import init -from tests.ut.python.model.resnet import resnet50 +from mindspore.context import ParallelMode +from mindspore.train import Model +from ....dataset_mock import MindData + def setup_module(module): _ = module @@ -144,6 +142,7 @@ def test_compile_model_train_O2(): # not actual run, the metrics step will fail, check if compile ok. model.eval(dataset) + def test_compile_model_train_O2_parallel(): dataset_types = (np.float32, np.float32) dataset_shapes = ((16, 16), (16, 16)) diff --git a/tests/ut/python/utils/test_initializer.py b/tests/ut/python/utils/test_initializer.py index 44d12fa94d4..1d2498b9595 100644 --- a/tests/ut/python/utils/test_initializer.py +++ b/tests/ut/python/utils/test_initializer.py @@ -141,6 +141,7 @@ def test_init_abnormal(): with py.raises(TypeError): init.initializer([''], [5, 4], ms.float32) + def test_initializer_reinit(): weights = init.initializer("XavierUniform", shape=(10, 1, 10, 10), dtype=ms.float16) assert weights.dtype == ms.float16 @@ -152,7 +153,8 @@ def test_initializer_reinit(): weights = init.initializer(weights, (10, 1)) assert weights.dtype == ms.float16 assert weights.shape == (10, 1) - + + def test_init_xavier_uniform(): """ test_init_xavier_uniform """ gain = 1.2