From d774e446112e948714cbc0966d0e96a74f9f32ad Mon Sep 17 00:00:00 2001 From: z00478463 Date: Thu, 21 May 2020 10:41:53 +0800 Subject: [PATCH 01/11] for second-order codes --- .../st/networks/thor_test/config_imagenet.py | 44 ++ tests/st/networks/thor_test/crossentropy.py | 23 + .../thor_test/cus_ops/batch_matmul_impl.py | 67 ++ .../thor_test/cus_ops/cholesky_trsm.py | 50 ++ .../thor_test/cus_ops/cus_batch_matmul.py | 53 ++ .../thor_test/cus_ops/cus_cholesky_trsm.py | 21 + .../thor_test/cus_ops/cus_fused_abs_max1.py | 28 + .../networks/thor_test/cus_ops/cus_img2col.py | 51 ++ .../thor_test/cus_ops/cus_matmul_cube.py | 52 ++ .../cus_ops/cus_matmul_cube_dense_left.py | 38 ++ .../cus_matmul_cube_fracz_right_mul.py | 38 ++ .../thor_test/cus_ops/cus_matrix_combine.py | 40 ++ .../thor_test/cus_ops/cus_transpose02314.py | 46 ++ .../thor_test/cus_ops/fused_abs_max1.py | 53 ++ .../thor_test/cus_ops/img2col_impl.py | 72 ++ .../cus_ops/matmul_cube_dense_left.py | 109 +++ .../matmul_cube_fracz_left_cast_impl.py | 107 +++ .../matmul_cube_fracz_right_mul_impl.py | 121 ++++ .../thor_test/cus_ops/matmul_cube_impl.py | 119 ++++ .../thor_test/cus_ops/matrix_combine_impl.py | 49 ++ .../thor_test/cus_ops/transpose02314_impl.py | 48 ++ .../st/networks/thor_test/dataset_imagenet.py | 79 +++ tests/st/networks/thor_test/lr_generator.py | 118 ++++ tests/st/networks/thor_test/model/' | 342 ++++++++++ .../thor_test/model/dataset_helper.py | 183 +++++ .../thor_test/model/grad_reducer_thor.py | 178 +++++ .../st/networks/thor_test/model/model_thor.py | 633 ++++++++++++++++++ tests/st/networks/thor_test/model/resnet.py | 342 ++++++++++ tests/st/networks/thor_test/model/thor.py | 195 ++++++ .../st/networks/thor_test/model/thor_layer.py | 450 +++++++++++++ .../thor_test/run_distribute_train_new.sh | 56 ++ tests/st/networks/thor_test/train.py | 151 +++++ 32 files changed, 3956 insertions(+) create mode 100644 tests/st/networks/thor_test/config_imagenet.py create mode 100644 tests/st/networks/thor_test/crossentropy.py create mode 100644 tests/st/networks/thor_test/cus_ops/batch_matmul_impl.py create mode 100644 tests/st/networks/thor_test/cus_ops/cholesky_trsm.py create mode 100644 tests/st/networks/thor_test/cus_ops/cus_batch_matmul.py create mode 100644 tests/st/networks/thor_test/cus_ops/cus_cholesky_trsm.py create mode 100644 tests/st/networks/thor_test/cus_ops/cus_fused_abs_max1.py create mode 100644 tests/st/networks/thor_test/cus_ops/cus_img2col.py create mode 100644 tests/st/networks/thor_test/cus_ops/cus_matmul_cube.py create mode 100644 tests/st/networks/thor_test/cus_ops/cus_matmul_cube_dense_left.py create mode 100644 tests/st/networks/thor_test/cus_ops/cus_matmul_cube_fracz_right_mul.py create mode 100644 tests/st/networks/thor_test/cus_ops/cus_matrix_combine.py create mode 100644 tests/st/networks/thor_test/cus_ops/cus_transpose02314.py create mode 100644 tests/st/networks/thor_test/cus_ops/fused_abs_max1.py create mode 100644 tests/st/networks/thor_test/cus_ops/img2col_impl.py create mode 100644 tests/st/networks/thor_test/cus_ops/matmul_cube_dense_left.py create mode 100644 tests/st/networks/thor_test/cus_ops/matmul_cube_fracz_left_cast_impl.py create mode 100644 tests/st/networks/thor_test/cus_ops/matmul_cube_fracz_right_mul_impl.py create mode 100644 tests/st/networks/thor_test/cus_ops/matmul_cube_impl.py create mode 100644 tests/st/networks/thor_test/cus_ops/matrix_combine_impl.py create mode 100644 tests/st/networks/thor_test/cus_ops/transpose02314_impl.py create mode 100644 tests/st/networks/thor_test/dataset_imagenet.py create mode 100644 tests/st/networks/thor_test/lr_generator.py create mode 100644 tests/st/networks/thor_test/model/' create mode 100644 tests/st/networks/thor_test/model/dataset_helper.py create mode 100644 tests/st/networks/thor_test/model/grad_reducer_thor.py create mode 100644 tests/st/networks/thor_test/model/model_thor.py create mode 100644 tests/st/networks/thor_test/model/resnet.py create mode 100644 tests/st/networks/thor_test/model/thor.py create mode 100644 tests/st/networks/thor_test/model/thor_layer.py create mode 100644 tests/st/networks/thor_test/run_distribute_train_new.sh create mode 100644 tests/st/networks/thor_test/train.py diff --git a/tests/st/networks/thor_test/config_imagenet.py b/tests/st/networks/thor_test/config_imagenet.py new file mode 100644 index 00000000000..3158dfaddd4 --- /dev/null +++ b/tests/st/networks/thor_test/config_imagenet.py @@ -0,0 +1,44 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +network config setting, will be used in train.py and eval.py +""" +from easydict import EasyDict as ed +config = ed({ + "class_num": 1000, + "batch_size": 32, + "loss_scale": 128, + "momentum": 0.9, + "weight_decay": 5e-4, + "epoch_size": 50, + "buffer_size": 1000, + "image_height": 224, + "image_width": 224, + "save_checkpoint": True, + "save_checkpoint_steps": 5004, + "keep_checkpoint_max": 20, + "save_checkpoint_path": "./", + "lr_init": 0.01, + "lr_end": 0.00001, + "lr_max": 0.1, + "warmup_epochs": 0, + "lr_decay_mode": "cosine", + "label_smooth": 1, + "label_smooth_factor": 0.1, + "lr": 0.1, + "T_max": 90, + "eta_min": 0, + "frequency": 278 +}) diff --git a/tests/st/networks/thor_test/crossentropy.py b/tests/st/networks/thor_test/crossentropy.py new file mode 100644 index 00000000000..5746051285d --- /dev/null +++ b/tests/st/networks/thor_test/crossentropy.py @@ -0,0 +1,23 @@ +from mindspore.nn.loss.loss import _Loss +from mindspore.ops import operations as P +from mindspore.ops import functional as F +from mindspore import Tensor +from mindspore.common import dtype as mstype +import mindspore.nn as nn + +class CrossEntropy(_Loss): + def __init__(self, smooth_factor=0., num_classes=1000): + super(CrossEntropy, self).__init__() + self.onehot = P.OneHot() + self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) + self.off_value = Tensor(1.0 * smooth_factor / (num_classes -1), mstype.float32) + #self.cast = P.Cast() + self.ce = nn.SoftmaxCrossEntropyWithLogits() + self.mean = P.ReduceMean(False) + def construct(self, logit, label): + #one_hot_label = self.onehot(self.cast(label, mstype.int32), + # F.shape(logit)[1], self.on_value, self.off_value)、 + one_hot_label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) + loss = self.ce(logit, one_hot_label) + loss = self.mean(loss, 0) + return loss diff --git a/tests/st/networks/thor_test/cus_ops/batch_matmul_impl.py b/tests/st/networks/thor_test/cus_ops/batch_matmul_impl.py new file mode 100644 index 00000000000..5af96b712eb --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/batch_matmul_impl.py @@ -0,0 +1,67 @@ +from te import tik +from topi.cce import util +from mindspore.ops.op_info_register import op_info_register + +@op_info_register("""{ + "op_name": "CusBatchMatMul", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "batchmatmul.so", + "compute_cost": 10, + "kernel_name": "CusBatchMatMul", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float32" + ], + "format": [ + "DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float32" + ], + "format": [ + "DefaultFormat" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32" + ], + "format": [ + "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") + + + + + +def CusBatchMatMul(input_x1, input_x2, output, transpose_a=False, transpose_b=True, kernel_name="batchmatmul"): + + return diff --git a/tests/st/networks/thor_test/cus_ops/cholesky_trsm.py b/tests/st/networks/thor_test/cus_ops/cholesky_trsm.py new file mode 100644 index 00000000000..6ed324dd0e6 --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/cholesky_trsm.py @@ -0,0 +1,50 @@ +from te import tik +from topi.cce import util +from mindspore.ops.op_info_register import op_info_register +@op_info_register("""{ + "op_name": "CusCholeskyTrsm", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "choleskytrsm.so", + "compute_cost": 10, + "kernel_name": "CusCholeskyTrsm", + "partial_flag": true, + "attr": [ + + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float32" + ], + "format": [ + "DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32" + ], + "format": [ + "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") + + +def CusCholeskyTrsm(input_x,output, kernel_name): + return diff --git a/tests/st/networks/thor_test/cus_ops/cus_batch_matmul.py b/tests/st/networks/thor_test/cus_ops/cus_batch_matmul.py new file mode 100644 index 00000000000..c58a15c88e1 --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/cus_batch_matmul.py @@ -0,0 +1,53 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.ops import prim_attr_register, PrimitiveWithInfer +from mindspore import Tensor +import mindspore as ms +from mindspore.ops.composite import multitype_ops as C +# y = x^2 +class CusBatchMatMul(PrimitiveWithInfer): + """CusMatMulCube definition""" + @prim_attr_register + def __init__(self): + """init CusMatMulCube""" + self.init_prim_io_names(inputs=['x1', 'x2'], outputs=['y']) + # self.transpose_a = transpose_a + # self.transpose_b = transpose_b + from .batch_matmul_impl import CusBatchMatMul + + def get_bprop(self): + def bprop(x1, x2, out, dout): + return (C.zeros_like(x1),C.zeros_like(x2)) + return bprop + + def infer_shape(self, data1_shape, data2_shape): + #shape = [1, data1_shape[1], data2_shape[2], 16, 16] + #return shape + # if self.transpose_a == True: + # k1, m = data1_shape + # else: + # m, k1 = data1_shape + # if self.transpose_b == True: + # n, k2 = data2_shape + # else: + # k2, n = data2_shape + # assert k1==k2 + # shape = [m, n] + return data1_shape + + def infer_dtype(self, data1_dtype, data2_dtype): + return data1_dtype + # return ms.common.dtype.tensor_type(getattr(ms, "float32")) diff --git a/tests/st/networks/thor_test/cus_ops/cus_cholesky_trsm.py b/tests/st/networks/thor_test/cus_ops/cus_cholesky_trsm.py new file mode 100644 index 00000000000..ce5f052c9da --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/cus_cholesky_trsm.py @@ -0,0 +1,21 @@ +import numpy as np +from mindspore.ops import prim_attr_register, PrimitiveWithInfer +from mindspore import Tensor + +class CusCholeskyTrsm(PrimitiveWithInfer): + """CusCholeskyTrsm definition""" + @prim_attr_register + def __init__(self): + """init CusCholeskyTrsm""" + self.init_prim_io_names(inputs=['x1'], outputs=['y']) + from .cholesky_trsm import CusCholeskyTrsm + + def infer_shape(self, data1_shape): + m,n = data1_shape + if m >= 128: + return [m//128,128,128] + else: + return [1,64,64] + + def infer_dtype(self, data1_dtype): + return data1_dtype diff --git a/tests/st/networks/thor_test/cus_ops/cus_fused_abs_max1.py b/tests/st/networks/thor_test/cus_ops/cus_fused_abs_max1.py new file mode 100644 index 00000000000..41a1e993cea --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/cus_fused_abs_max1.py @@ -0,0 +1,28 @@ +import numpy as np +from mindspore.ops import prim_attr_register, PrimitiveWithInfer +from mindspore import Tensor +from mindspore.ops.composite import multitype_ops as C + +class CusFusedAbsMax1(PrimitiveWithInfer): + """CusCholeskyTrsm definition""" + @prim_attr_register + def __init__(self, origin_shape = [-1,-1]): + """init CusCholeskyTrsm""" + self.init_prim_io_names(inputs=['x1'], outputs=['y']) + from .fused_abs_max1 import CusFusedAbsMax1 + self.origin_shape = origin_shape + + def get_bprop(self): + def bprop(x, out, dout): + return (C.zeros_like(x),) + return bprop + + def infer_shape(self, data1_shape): + if len(data1_shape) == 2: + return [1,] + else: + return [32, 64] + # return [128,128] + + def infer_dtype(self, data1_dtype): + return data1_dtype diff --git a/tests/st/networks/thor_test/cus_ops/cus_img2col.py b/tests/st/networks/thor_test/cus_ops/cus_img2col.py new file mode 100644 index 00000000000..5a1249938ff --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/cus_img2col.py @@ -0,0 +1,51 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import numpy as np +from mindspore.ops import prim_attr_register, PrimitiveWithInfer +from mindspore import Tensor +from mindspore.ops.composite import multitype_ops as C + +class CusImg2Col(PrimitiveWithInfer): + """CusImg2Col definition""" + @prim_attr_register + def __init__(self, ksizes, strides, dilates = (1, 1, 1, 1), mode="NC1HWC0"): + """init CusImg2Col""" + self.init_prim_io_names(inputs=['x1'], outputs=['y']) + self.ksizes = ksizes + self.strides = strides + self.dilates = dilates + self.mode = mode + from .img2col_impl import CusImg2Col + + def get_bprop(self): + def bprop(x, out, dout): + return (C.zeros_like(x),) + return bprop + + def infer_shape(self, data1_shape): + bs, c, h, w = data1_shape + _, stride_h, stride_w, _ = self.strides + _, k_w, k_h, _ = self.ksizes + # assert m == n + c0 = 16 + c1 = c // 16 + if c1 == 0: + c1 = 1 + shape = [bs * int(h // stride_h) * int(w // stride_w), k_w * k_h * c1 * c0] + return shape + + def infer_dtype(self, data1_dtype): + return data1_dtype diff --git a/tests/st/networks/thor_test/cus_ops/cus_matmul_cube.py b/tests/st/networks/thor_test/cus_ops/cus_matmul_cube.py new file mode 100644 index 00000000000..1b9d295b1c7 --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/cus_matmul_cube.py @@ -0,0 +1,52 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.ops import prim_attr_register, PrimitiveWithInfer +from mindspore import Tensor +import mindspore as ms +from mindspore.ops.composite import multitype_ops as C +# y = x^2 +class CusMatMulCube(PrimitiveWithInfer): + """CusMatMulCube definition""" + @prim_attr_register + def __init__(self, transpose_a=False, transpose_b=False): + """init CusMatMulCube""" + self.init_prim_io_names(inputs=['x1', 'x2'], outputs=['y']) + self.transpose_a = transpose_a + self.transpose_b = transpose_b + from .matmul_cube_impl import CusMatMulCube + + def get_bprop(self): + def bprop(x1, x2, out, dout): + return (C.zeros_like(x1),C.zeros_like(x2)) + return bprop + + def infer_shape(self, data1_shape, data2_shape): + #shape = [1, data1_shape[1], data2_shape[2], 16, 16] + #return shape + if self.transpose_a == True: + k1, m = data1_shape + else: + m, k1 = data1_shape + if self.transpose_b == True: + n, k2 = data2_shape + else: + k2, n = data2_shape + assert k1==k2 + shape = [m, n] + return shape + + def infer_dtype(self, data1_dtype, data2_dtype): + return ms.common.dtype.tensor_type(getattr(ms, "float32")) diff --git a/tests/st/networks/thor_test/cus_ops/cus_matmul_cube_dense_left.py b/tests/st/networks/thor_test/cus_ops/cus_matmul_cube_dense_left.py new file mode 100644 index 00000000000..71b77910ecc --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/cus_matmul_cube_dense_left.py @@ -0,0 +1,38 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.ops import prim_attr_register, PrimitiveWithInfer +from mindspore import Tensor +import mindspore as ms +from mindspore.ops.composite import multitype_ops as C +# y = x^2 +class CusMatMulCubeDenseLeft(PrimitiveWithInfer): + """CusMatMulCube definition""" + @prim_attr_register + def __init__(self): + """init CusMatMulCube""" + self.init_prim_io_names(inputs=['x1', 'x2'], outputs=['y']) + from .matmul_cube_dense_left import CusMatMulCubeDenseLeft + + def get_bprop(self): + def bprop(x1, x2, out, dout): + return (C.zeros_like(x1),C.zeros_like(x2)) + return bprop + + def infer_shape(self, data1_shape, data2_shape): + return data2_shape + + def infer_dtype(self, data1_dtype, data2_dtype): + return ms.common.dtype.tensor_type(getattr(ms, "float16")) diff --git a/tests/st/networks/thor_test/cus_ops/cus_matmul_cube_fracz_right_mul.py b/tests/st/networks/thor_test/cus_ops/cus_matmul_cube_fracz_right_mul.py new file mode 100644 index 00000000000..20b278e1fc2 --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/cus_matmul_cube_fracz_right_mul.py @@ -0,0 +1,38 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.ops import prim_attr_register, PrimitiveWithInfer +from mindspore import Tensor +import mindspore as ms +from mindspore.ops.composite import multitype_ops as C +# y = x^2 +class CusMatMulCubeFraczRightMul(PrimitiveWithInfer): + """CusMatMulCubeFraczRightMul definition""" + @prim_attr_register + def __init__(self): + """init CusMatMulCubeFraczRightMul""" + self.init_prim_io_names(inputs=['x1', 'x2', 'x3'], outputs=['y']) + from .matmul_cube_fracz_right_mul_impl import CusMatMulCubeFraczRightMul + + def get_bprop(self): + def bprop(x1, x2, x3, out, dout): + return (C.zeros_like(x1),C.zeros_like(x2),C.zeros_like(x3)) + return bprop + + def infer_shape(self, data1_shape, data2_shape, data3_shape): + return data1_shape + + def infer_dtype(self, data1_dtype, data2_dtype, data3_dtype): + return ms.common.dtype.tensor_type(getattr(ms, "float32")) diff --git a/tests/st/networks/thor_test/cus_ops/cus_matrix_combine.py b/tests/st/networks/thor_test/cus_ops/cus_matrix_combine.py new file mode 100644 index 00000000000..66230e0a176 --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/cus_matrix_combine.py @@ -0,0 +1,40 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +import numpy as np +from mindspore.ops import prim_attr_register, PrimitiveWithInfer +from mindspore import Tensor +import mindspore as ms +from mindspore.ops.composite import multitype_ops as C +# y = x^2 +class CusMatrixCombine(PrimitiveWithInfer): + """CusMatMulCube definition""" + @prim_attr_register + def __init__(self): + """init CusMatMulCube""" + self.init_prim_io_names(inputs=['x'], outputs=['y']) + from .matrix_combine_impl import CusMatrixCombine + def get_bprop(self): + def bprop(x, out, dout): + return (C.zeros_like(x),) + return bprop + + def infer_shape(self, data_shape): + a, b, c = data_shape + shape = [a*b, a*c] + + return shape + + def infer_dtype(self, data_dtype): + return data_dtype diff --git a/tests/st/networks/thor_test/cus_ops/cus_transpose02314.py b/tests/st/networks/thor_test/cus_ops/cus_transpose02314.py new file mode 100644 index 00000000000..90051022f8e --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/cus_transpose02314.py @@ -0,0 +1,46 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + + +import numpy as np +from mindspore.ops import prim_attr_register, PrimitiveWithInfer +from mindspore import Tensor +from mindspore.ops.composite import multitype_ops as C + +class CusTranspose02314(PrimitiveWithInfer): + """CusTranspose02314 definition""" + @prim_attr_register + def __init__(self): + """init CusTranspose02314""" + self.init_prim_io_names(inputs=['x1'], outputs=['y']) + from .transpose02314_impl import CusTranspose02314 + + def get_bprop(self): + def bprop(x, out, dout): + return (C.zeros_like(x),) + return bprop + + def infer_shape(self, data1_shape): + assert len(data1_shape) == 4 + n, c, h, w = data1_shape + c0 = 16 + c1 = c // 16 + shape = (n * h * w, c1 * c0) + # axis_0, axis_1, axis_2, axis_3, axis_4 = data1_shape + # shape = (axis_0, axis_2, axis_3, axis_1, axis_4) + return shape + + def infer_dtype(self, data1_dtype): + return data1_dtype diff --git a/tests/st/networks/thor_test/cus_ops/fused_abs_max1.py b/tests/st/networks/thor_test/cus_ops/fused_abs_max1.py new file mode 100644 index 00000000000..e11347ed64f --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/fused_abs_max1.py @@ -0,0 +1,53 @@ +from te import tik +from topi.cce import util +from mindspore.ops.op_info_register import op_info_register +@op_info_register("""{ + "op_name": "CusFusedAbsMax1", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "fusedabsmax1.so", + "compute_cost": 10, + "kernel_name": "CusFusedAbsMax1", + "partial_flag": true, + "attr": [ + { + "name": "origin_shape", + "param_type": "required", + "type": "listInt", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float32" + ], + "format": [ + "DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32" + ], + "format": [ + "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") +def CusFusedAbsMax1(input_x, output, origin_shape = None, kernel_name="fused_abs_max1"): + return diff --git a/tests/st/networks/thor_test/cus_ops/img2col_impl.py b/tests/st/networks/thor_test/cus_ops/img2col_impl.py new file mode 100644 index 00000000000..10e4a3acae2 --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/img2col_impl.py @@ -0,0 +1,72 @@ +from te import tik +from topi.cce import util +from mindspore.ops.op_info_register import op_info_register +@op_info_register("""{ + "op_name": "CusImg2ColNC1HWC0", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "img2colnc1hwc0.so", + "compute_cost": 10, + "kernel_name": "CusImg2ColNC1HWC0", + "partial_flag": true, + "attr": [ + { + "name": "ksizes", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "strides", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "dilates", + "param_type": "required", + "type": "listInt", + "value": "all" + }, + { + "name": "padding", + "param_type": "required", + "type": "str", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "FRACTAL_NZ" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") + +def CusImg2ColNC1HWC0(input_x, output, ksizes, strides, dilates, padding, kernel_name="img2col"): + return diff --git a/tests/st/networks/thor_test/cus_ops/matmul_cube_dense_left.py b/tests/st/networks/thor_test/cus_ops/matmul_cube_dense_left.py new file mode 100644 index 00000000000..eaa7f9b1ec7 --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/matmul_cube_dense_left.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- +""" +copyright 2019 Huawei Technologies Co., Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License == distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +matmul +""" +from __future__ import absolute_import + +import te.lang.cce +import te.platform.cce_params as cce +from te.platform.fusion_manager import fusion_manager +from te import tvm +from topi import generic +from topi.cce import util + +from impl.matmul_vector import matmul_vector_cce + +from te import tik +from mindspore.ops.op_info_register import op_info_register +# General limitation of the size for input shape: 2**31 +SHAPE_SIZE_LIMIT = 2147483648 +NoneType = type(None) + +@op_info_register("""{ + "op_name": "CusMatMulCubeDenseLeft", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "matmulcubedenseleft.so", + "compute_cost": 10, + "kernel_name": "CusMatMulCubeDenseLeft", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16" + ], + "format": [ + "FRACTAL_NZ" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16" + ], + "format": [ + "DefaultFormat" + ], + "name": "x3", + "need_compile": false, + "param_type": "optional", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "FRACTAL_NZ" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") + +@util.check_input_type(dict, dict, (dict, NoneType), dict, bool, bool, str) +def CusMatMulCubeDenseLeft(input_x1, input_x2, bias=None, output_y={}, trans_a=False, trans_b=False, kernel_name="matmulcube"): + return + diff --git a/tests/st/networks/thor_test/cus_ops/matmul_cube_fracz_left_cast_impl.py b/tests/st/networks/thor_test/cus_ops/matmul_cube_fracz_left_cast_impl.py new file mode 100644 index 00000000000..e0126a02dcc --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/matmul_cube_fracz_left_cast_impl.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- +""" +copyright 2019 Huawei Technologies Co., Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License == distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +matmul +""" +from __future__ import absolute_import + +import te.platform.cce_params as cce +from te import tvm +from topi.cce import util + +from te import tik +from mindspore.ops.op_info_register import op_info_register + +# General limitation of the size for input shape: 2**31 +SHAPE_SIZE_LIMIT = 2147483648 +NoneType = type(None) + + +@op_info_register("""{ + "op_name": "CusMatMulCubeFraczLeftCast", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "matmulcubefraczleftcast.so", + "compute_cost": 10, + "kernel_name": "CusMatMulCubeFraczLeftCast", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float32" + ], + "format": [ + "FracZ" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16" + ], + "format": [ + "DefaultFormat" + ], + "name": "x3", + "need_compile": false, + "param_type": "optional", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "FracZ" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") + +# pylint: disable=locally-disabled,too-many-arguments, too-many-locals, too-many-statements +@util.check_input_type(dict, dict, (dict, NoneType), dict, bool, bool, str) +def CusMatMulCubeFraczLeftCast(input_x1, input_x2, bias=None, output_y={}, trans_a=False, trans_b=False, + kernel_name="CusMatMulCubeFraczLeftCast"): + return diff --git a/tests/st/networks/thor_test/cus_ops/matmul_cube_fracz_right_mul_impl.py b/tests/st/networks/thor_test/cus_ops/matmul_cube_fracz_right_mul_impl.py new file mode 100644 index 00000000000..c3589bb4d89 --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/matmul_cube_fracz_right_mul_impl.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- +""" +copyright 2019 Huawei Technologies Co., Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License == distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +matmul +""" +from __future__ import absolute_import + +import te.lang.cce +import te.platform.cce_params as cce +from te.platform.fusion_manager import fusion_manager +from te import tvm +from topi import generic +from topi.cce import util +from te import tik +from impl.matmul_vector import matmul_vector_cce +from mindspore.ops.op_info_register import op_info_register +# General limitation of the size for input shape: 2**31 +SHAPE_SIZE_LIMIT = 2147483648 +NoneType = type(None) + + +@op_info_register("""{ + "op_name": "CusMatMulCubeFraczRightMul", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "matmulcubefraczrightmul.so", + "compute_cost": 10, + "kernel_name": "CusMatMulCubeFraczRightMul", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "FracZ" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16" + ], + "format": [ + "DefaultFormat" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float32" + ], + "format": [ + "DefaultFormat" + ], + "name": "x3", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 3, + "dtype": [ + "float16" + ], + "format": [ + "DefaultFormat" + ], + "name": "x4", + "need_compile": false, + "param_type": "optional", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32" + ], + "format": [ + "FracZ" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") + +def CusMatMulCubeFraczRightMul(input_x1, input_x2, input_x3, bias=None, output_y={}, trans_a=False, trans_b=False, kernel_name="matmulcube"): + return + + diff --git a/tests/st/networks/thor_test/cus_ops/matmul_cube_impl.py b/tests/st/networks/thor_test/cus_ops/matmul_cube_impl.py new file mode 100644 index 00000000000..4cb12751d71 --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/matmul_cube_impl.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding:utf-8 -*- +""" +copyright 2019 Huawei Technologies Co., Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License == distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +matmul +""" +from __future__ import absolute_import + +import te.lang.cce +import te.platform.cce_params as cce +from te import tvm +from topi import generic +from topi.cce import util + +from impl.matmul_vector import matmul_vector_cce +from mindspore.ops.op_info_register import op_info_register + +# General limitation of the size for input shape: 2**31 +SHAPE_SIZE_LIMIT = 2147483648 +NoneType = type(None) + +@op_info_register("""{ + "op_name": "CusMatMulCube", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "matmulcube.so", + "compute_cost": 10, + "kernel_name": "CusMatMulCube", + "partial_flag": true, + "attr": [ + { + "name": "transpose_a", + "param_type": "required", + "type": "bool", + "value": "all" + }, + { + "name": "transpose_b", + "param_type": "required", + "type": "bool", + "value": "all" + } + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "FRACTAL_NZ" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 1, + "dtype": [ + "float16" + ], + "format": [ + "FRACTAL_NZ" + ], + "name": "x2", + "need_compile": false, + "param_type": "required", + "shape": "all" + }, + { + "index": 2, + "dtype": [ + "float16" + ], + "format": [ + "DefaultFormat" + ], + "name": "x3", + "need_compile": false, + "param_type": "optional", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32" + ], + "format": [ + "FRACTAL_NZ" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") + +# pylint: disable=locally-disabled,too-many-arguments, too-many-locals, too-many-statements +@util.check_input_type(dict, dict, (dict, NoneType), dict, bool, bool, str) +def CusMatMulCube(input_x1, input_x2, bias=None, output_y={}, trans_a=False, trans_b=False, kernel_name="matmulcube"): + return diff --git a/tests/st/networks/thor_test/cus_ops/matrix_combine_impl.py b/tests/st/networks/thor_test/cus_ops/matrix_combine_impl.py new file mode 100644 index 00000000000..e9cead18fb9 --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/matrix_combine_impl.py @@ -0,0 +1,49 @@ +from te import tik +from topi.cce import util +from mindspore.ops.op_info_register import op_info_register +@op_info_register("""{ + "op_name": "CusMatrixCombine", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "matrixcombine.so", + "compute_cost": 10, + "kernel_name": "CusMatrixCombine", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float32" + ], + "format": [ + "DefaultFormat" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float32" + ], + "format": [ + "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") + + +def CusMatrixCombine(input_x, output,kernel_name="matrix_combine"): + return diff --git a/tests/st/networks/thor_test/cus_ops/transpose02314_impl.py b/tests/st/networks/thor_test/cus_ops/transpose02314_impl.py new file mode 100644 index 00000000000..be4ee94d869 --- /dev/null +++ b/tests/st/networks/thor_test/cus_ops/transpose02314_impl.py @@ -0,0 +1,48 @@ +from te import tik +from topi.cce import util +from mindspore.ops.op_info_register import op_info_register +@op_info_register("""{ + "op_name": "CusTranspose02314", + "imply_type": "TBE", + "fusion_type": "OPAQUE", + "async_flag": false, + "binfile_name": "transpose02314.so", + "compute_cost": 10, + "kernel_name": "CusTranspose02314", + "partial_flag": true, + "attr": [ + ], + "inputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "NC1HWC0" + ], + "name": "x1", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ], + "outputs": [ + { + "index": 0, + "dtype": [ + "float16" + ], + "format": [ + "DefaultFormat" + ], + "name": "y", + "need_compile": false, + "param_type": "required", + "shape": "all" + } + ] +}""") + +def CusTranspose02314(input_x, output, kernel_name="transpose021354"): + return diff --git a/tests/st/networks/thor_test/dataset_imagenet.py b/tests/st/networks/thor_test/dataset_imagenet.py new file mode 100644 index 00000000000..3febbd76ccb --- /dev/null +++ b/tests/st/networks/thor_test/dataset_imagenet.py @@ -0,0 +1,79 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +create train or eval dataset. +""" +import os +import mindspore.common.dtype as mstype +import mindspore.dataset.engine as de +import mindspore.dataset.transforms.vision.c_transforms as V_C +import mindspore.dataset.transforms.c_transforms as C2 +from config_imagenet import config + +def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): + """ + create a train or eval dataset + Args: + dataset_path(string): the path of dataset. + do_train(bool): whether dataset is used for train or eval. + repeat_num(int): the repeat times of dataset. Default: 1 + batch_size(int): the batch size of dataset. Default: 32 + Returns: + dataset + """ + + device_num = int(os.getenv("RANK_SIZE")) + rank_id = int(os.getenv("RANK_ID")) + + if device_num == 1: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False) + else: + ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, + num_shards=device_num, shard_id=rank_id) + + image_size = 224 + mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] + std = [0.229 * 255, 0.224 * 255, 0.225 * 255] + if do_train: + transform_img = [ + V_C.RandomCropDecodeResize(image_size, scale=(0.08, 1.0), ratio=(0.75, 1.333)), + V_C.RandomHorizontalFlip(prob=0.5), + V_C.Normalize(mean=mean, std=std), + V_C.HWC2CHW() + ] + else: + transform_img = [ + V_C.Decode(), + V_C.Resize((256, 256)), + V_C.CenterCrop(image_size), + V_C.Normalize(mean=mean, std=std), + V_C.HWC2CHW() + ] + #type_cast_op = C2.TypeCast(mstype.float16) + type_cast_op = C2.TypeCast(mstype.int32) + + ds = ds.map(input_columns="image", operations=transform_img, num_parallel_workers=8) + ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) + + # apply shuffle operations + # ds = ds.shuffle(buffer_size=config.buffer_size) + + # apply batch operations + ds = ds.batch(batch_size, drop_remainder=True) + + # apply dataset repeat operation + ds = ds.repeat(repeat_num) + + return ds diff --git a/tests/st/networks/thor_test/lr_generator.py b/tests/st/networks/thor_test/lr_generator.py new file mode 100644 index 00000000000..3536b0f89eb --- /dev/null +++ b/tests/st/networks/thor_test/lr_generator.py @@ -0,0 +1,118 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""learning rate generator""" +import numpy as np +import math + +def linear_warmup_lr(current_step, warmup_steps, base_lr, init_lr): + lr_inc = (float(base_lr) - float(init_lr)) / float(warmup_steps) + lr = float(init_lr) + lr_inc * current_step + return lr + +def cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0, num_periods=0.5): + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch) + warmup_steps = int(warmup_epochs * steps_per_epoch) + decay_steps = total_steps - warmup_steps + lr_each_step = [] + for i in range(total_steps): + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + # linear_decay = (total_steps - i) / decay_steps + cosine_decay = 0.5 * (1 + math.cos(math.pi * i / decay_steps)) + decayed = cosine_decay + lr = base_lr * decayed + lr_each_step.append(lr) + return np.array(lr_each_step).astype(np.float32) + +def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0, num_periods=0.5): + base_lr = lr + warmup_init_lr = 0 + total_steps = int(max_epoch * steps_per_epoch * 0.99) + warmup_steps = int(warmup_epochs * steps_per_epoch) + decay_steps = total_steps - warmup_steps + lr_each_step = [] + for i in range(total_steps): + if i < warmup_steps: + lr = linear_warmup_lr(i + 1, warmup_steps, base_lr, warmup_init_lr) + else: + linear_decay = (total_steps - i) / decay_steps + cosine_decay = 0.5 * (1 + math.cos(math.pi * 2 * num_periods * i / decay_steps)) + decayed = linear_decay * cosine_decay + lr = base_lr * decayed + 0.000005 + lr_each_step.append(lr) + return np.array(lr_each_step).astype(np.float32) + +def get_lr(global_step, lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch, lr_decay_mode): + """ + generate learning rate array + + Args: + global_step(int): total steps of the training + lr_init(float): init learning rate + lr_end(float): end learning rate + lr_max(float): max learning rate + warmup_epochs(int): number of warmup epochs + total_epochs(int): total epoch of training + steps_per_epoch(int): steps of one epoch + lr_decay_mode(string): learning rate decay mode, including steps, poly or default + + Returns: + np.array, learning rate array + """ + lr_each_step = [] + total_steps = steps_per_epoch * total_epochs + warmup_steps = steps_per_epoch * warmup_epochs + if lr_decay_mode == 'steps': + decay_epoch_index = [0.3 * total_steps, 0.6 * total_steps, 0.8 * total_steps] + for i in range(total_steps): + if i < decay_epoch_index[0]: + lr = lr_max + elif i < decay_epoch_index[1]: + lr = lr_max * 0.1 + elif i < decay_epoch_index[2]: + lr = lr_max * 0.01 + else: + lr = lr_max * 0.001 + lr_each_step.append(lr) + elif lr_decay_mode == 'poly': + if warmup_steps != 0: + inc_each_step = (float(lr_max) - float(lr_init)) / float(warmup_steps) + else: + inc_each_step = 0 + for i in range(total_steps): + if i < warmup_steps: + lr = float(lr_init) + inc_each_step * float(i) + else: + base = (1.0 - (float(i) - float(warmup_steps)) / (float(total_steps) - float(warmup_steps))) + lr = float(lr_max) * base * base + if lr < 0.0: + lr = 0.0 + lr_each_step.append(lr) + else: + for i in range(total_steps): + if i < warmup_steps: + lr = lr_init + (lr_max - lr_init) * i / warmup_steps + else: + lr = lr_max - (lr_max - lr_end) * (i - warmup_steps) / (total_steps - warmup_steps) + lr_each_step.append(lr) + + current_step = global_step + lr_each_step = np.array(lr_each_step).astype(np.float32) + learning_rate = lr_each_step[current_step:] + + return learning_rate diff --git a/tests/st/networks/thor_test/model/' b/tests/st/networks/thor_test/model/' new file mode 100644 index 00000000000..d77239dbf29 --- /dev/null +++ b/tests/st/networks/thor_test/model/' @@ -0,0 +1,342 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ResNet.""" +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from second_order.thor_layer import Conv2d_Thor, Dense_Thor +import math + +def calculate_gain(nonlinearity, param=None): + linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d'] + if nonlinearity in linear_fns or nonlinearity == 'sigmoid': + return 1 + elif nonlinearity == 'tanh': + return 5.0 / 3 + elif nonlinearity == 'relu': + return math.sqrt(2.0) + elif nonlinearity == 'leaky_relu': + if param is None: + negative_slope = 0.01 + elif not isinstance(param, bool) and isinstance(param, int) or isinstance(param, float): + # True/False are instances of int, hence check above + negative_slope = param + else: + raise ValueError("negative_slope {} not a valid number".format(param)) + return math.sqrt(2.0 / (1 + negative_slope ** 2)) + else: + raise ValueError("Unsupported nonlinearity {}".format(nonlinearity)) + +def _calculate_fan_in_and_fan_out(tensor): + dimensions = len(tensor) + if dimensions < 2: + raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions") + + if dimensions == 2: # Linear + fan_in = tensor[1] + fan_out = tensor[0] + else: + num_input_fmaps = tensor[1] + num_output_fmaps = tensor[0] + receptive_field_size = 1 + if dimensions > 2: + receptive_field_size = tensor[2] * tensor[3] + fan_in = num_input_fmaps * receptive_field_size + fan_out = num_output_fmaps * receptive_field_size + return fan_in, fan_out + +def _calculate_correct_fan(tensor, mode): + mode = mode.lower() + valid_modes = ['fan_in', 'fan_out'] + if mode not in valid_modes: + raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes)) + + fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) + return fan_in if mode == 'fan_in' else fan_out + +def kaiming_normal(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu'): + fan = _calculate_correct_fan(inputs_shape, mode) + gain = calculate_gain(nonlinearity, a) + std = gain / math.sqrt(fan) + return np.random.normal(0, std, size=inputs_shape).astype(np.float32) + +def kaiming_uniform(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu'): + fan = _calculate_correct_fan(inputs_shape, mode) + gain = calculate_gain(nonlinearity, a) + std = gain / math.sqrt(fan) + bound = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation + return np.random.uniform(-bound, bound, size=inputs_shape).astype(np.float32) + +def _conv3x3(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): + weight_shape = (out_channel, in_channel, 3, 3) + weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) + return Conv2d_Thor(in_channel, out_channel, + kernel_size=3, stride=stride, padding=0, pad_mode='same', weight_init=weight, + damping=damping, loss_scale=loss_scale, frequency=frequency) + # return nn.Conv2d(in_channel, out_channel, + # kernel_size=3, stride=stride, padding=0, pad_mode='same', weight_init=weight) + +def _conv1x1(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): + weight_shape = (out_channel, in_channel, 1, 1) + weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) + return Conv2d_Thor(in_channel, out_channel, + kernel_size=1, stride=stride, padding=0, pad_mode='same', weight_init=weight, + damping=damping, loss_scale=loss_scale, frequency=frequency) + +def _conv7x7(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): + weight_shape = (out_channel, in_channel, 7, 7) + weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) + return Conv2d_Thor(in_channel, out_channel, + kernel_size=7, stride=stride, padding=0, pad_mode='same', weight_init=weight, + damping=damping, loss_scale=loss_scale, frequency=frequency) + +def _bn(channel): + return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, + gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) + +def _bn_last(channel): + return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, + gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) + +def _fc(in_channel, out_channel, damping, loss_scale, frequency): + weight_shape = (out_channel, in_channel) + weight = Tensor(kaiming_uniform(weight_shape, a = math.sqrt(5)) + return Dense_Thor(in_channel, out_channel, has_bias=False, weight_init=weight, bias_init=0, + damping=damping, loss_scale=loss_scale, frequency=frequency) + +class ResidualBlock(nn.Cell): + """ + ResNet V1 residual block definition. + + Args: + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. Default: 1. + + Returns: + Tensor, output tensor. + + Examples: + >>> ResidualBlock(3, 256, stride=2) + """ + expansion = 4 + + def __init__(self, + in_channel, + out_channel, + stride=1, + damping=0.03, + loss_scale=1, + frequency=278): + super(ResidualBlock, self).__init__() + + channel = out_channel // self.expansion + self.conv1 = _conv1x1(in_channel, channel, stride=1, damping=damping, loss_scale=loss_scale, frequency=frequency) + self.bn1 = _bn(channel) + + self.conv2 = _conv3x3(channel, channel, stride=stride, damping=damping, loss_scale=loss_scale, frequency=frequency) + self.bn2 = _bn(channel) + + self.conv3 = _conv1x1(channel, out_channel, stride=1, damping=damping, loss_scale=loss_scale, frequency=frequency) + self.bn3 = _bn_last(out_channel) + + self.relu = nn.ReLU() + + self.down_sample = False + + if stride != 1 or in_channel != out_channel: + self.down_sample = True + self.down_sample_layer = None + + if self.down_sample: + self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channel, out_channel, stride, + damping=damping, loss_scale=loss_scale, frequency=frequency), + _bn(out_channel)]) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.down_sample: + identity = self.down_sample_layer(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet(nn.Cell): + """ + ResNet architecture. + + Args: + block (Cell): Block for network. + layer_nums (list): Numbers of block in different layers. + in_channels (list): Input channel in each layer. + out_channels (list): Output channel in each layer. + strides (list): Stride size in each layer. + num_classes (int): The number of classes that the training images are belonging to. + Returns: + Tensor, output tensor. + + Examples: + >>> ResNet(ResidualBlock, + >>> [3, 4, 6, 3], + >>> [64, 256, 512, 1024], + >>> [256, 512, 1024, 2048], + >>> [1, 2, 2, 2], + >>> 10) + """ + + def __init__(self, + block, + layer_nums, + in_channels, + out_channels, + strides, + num_classes, + damping, + loss_scale, + frequency): + super(ResNet, self).__init__() + + if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: + raise ValueError("the length of layer_num, in_channels, out_channels list must be 4!") + + self.conv1 = _conv7x7(3, 64, stride=2, damping=damping, loss_scale=loss_scale, frequency=frequency) + self.bn1 = _bn(64) + self.relu = P.ReLU() + self.maxpool = P.MaxPoolWithArgmax(padding="same", ksize=3, strides=2) + + self.layer1 = self._make_layer(block, + layer_nums[0], + in_channel=in_channels[0], + out_channel=out_channels[0], + stride=strides[0], + damping=damping, + loss_scale=loss_scale, + frequency=frequency) + self.layer2 = self._make_layer(block, + layer_nums[1], + in_channel=in_channels[1], + out_channel=out_channels[1], + stride=strides[1], + damping=damping, + loss_scale=loss_scale, + frequency=frequency) + self.layer3 = self._make_layer(block, + layer_nums[2], + in_channel=in_channels[2], + out_channel=out_channels[2], + stride=strides[2],damping=damping, + loss_scale=loss_scale, + frequency=frequency) + self.layer4 = self._make_layer(block, + layer_nums[3], + in_channel=in_channels[3], + out_channel=out_channels[3], + stride=strides[3], + damping=damping, + loss_scale=loss_scale, + frequency=frequency) + + self.mean = P.ReduceMean(keep_dims=True) + self.flatten = nn.Flatten() + self.end_point = _fc(out_channels[3], num_classes, damping=damping, loss_scale=loss_scale, frequency=frequency) + + def _make_layer(self, block, layer_num, in_channel, out_channel, stride, + damping, loss_scale, frequency): + """ + Make stage network of ResNet. + + Args: + block (Cell): Resnet block. + layer_num (int): Layer number. + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. + + Returns: + SequentialCell, the output layer. + + Examples: + >>> _make_layer(ResidualBlock, 3, 128, 256, 2) + """ + layers = [] + + resnet_block = block(in_channel, out_channel, stride=stride, + damping=damping, loss_scale=loss_scale, frequency=frequency) + layers.append(resnet_block) + + for _ in range(1, layer_num): + resnet_block = block(out_channel, out_channel, stride=1, + damping=damping, loss_scale=loss_scale, frequency=frequency) + layers.append(resnet_block) + + return nn.SequentialCell(layers) + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + c1, argmax = self.maxpool(x) + + c2 = self.layer1(c1) + c3 = self.layer2(c2) + c4 = self.layer3(c3) + c5 = self.layer4(c4) + + out = self.mean(c5, (2, 3)) + out = self.flatten(out) + out = self.end_point(out) + + return out + + +def resnet50(class_num=10, damping=0.03, loss_scale=1, frequency=278): + """ + Get ResNet50 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet50 neural network. + + Examples: + >>> net = resnet50(10) + """ + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num, + damping, + loss_scale, + frequency) diff --git a/tests/st/networks/thor_test/model/dataset_helper.py b/tests/st/networks/thor_test/model/dataset_helper.py new file mode 100644 index 00000000000..36e4ce107a8 --- /dev/null +++ b/tests/st/networks/thor_test/model/dataset_helper.py @@ -0,0 +1,183 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Dataset help for minddata dataset""" +from mindspore._checkparam import check_bool +from mindspore import context +from mindspore.train.parallel_utils import ParallelMode +from mindspore.train._utils import _exec_datagraph, _get_types_and_shapes, _to_tensor, \ + _construct_tensor_list, _to_full_shapes, _to_full_tensor +from mindspore.nn.wrap import GetNextSingleOp +from mindspore.parallel._utils import _get_device_num, _get_global_rank, _get_parallel_mode + + +class DatasetHelper: + """ + Help function to use the Minddata dataset. + + According to different context, change the iter of dataset, to use the same for loop in different context. + + Note: + The iter of DatasetHelper will give one epoch data. + + Args: + dataset (DataSet): The dataset. + dataset_sink_mode (bool): If true use GetNext to fetch the data, or else feed the data from host. + Default: True. + + Examples: + >>> dataset_helper = DatasetHelper(dataset) + >>> for inputs in dataset_helper: + >>> outputs = network(*inputs) + """ + def __init__(self, dataset, first_order_iter=0, dataset_sink_mode=True): + check_bool(dataset_sink_mode) + + iterclass = _DatasetIterGE + if not dataset_sink_mode: + iterclass = _DatasetIterFeed + elif not context.get_context("enable_ge"): + if context.get_context("enable_loop_sink"): + iterclass = _DatasetIterMSLoopSink + else: + iterclass = _DatasetIterMS + + self.iter = iterclass(dataset, first_order_iter) + + def __iter__(self): + return self.iter.__iter__() + + # A temp solution for loop sink. Delete later + def types_shapes(self): + """Get the types and shapes from dataset on current config.""" + return self.iter.types_shapes() + + def loop_size(self): + """Get loop_size for every iteration.""" + return self.iter.loop_size + + +class _DatasetIter: + """Base iter for dataset help""" + def __init__(self, dataset): + self.loop_size = 1 + if not hasattr(dataset, '__ME_INITED__'): + if not hasattr(dataset, '__loop_size__'): + self.loop_size = dataset.get_dataset_size() + else: + self.loop_size = dataset.__loop_size__ + dataset.__ME_INITED__ = _exec_datagraph(dataset, self.loop_size).queue_name + + self.ind = 0 + self.dataset = dataset + dataset_types, dataset_shapes = _get_types_and_shapes(dataset) + self.dataset_types, self.dataset_shapes = dataset_types, dataset_shapes + # for self._parallel_mode equal to semi_auto_parallel or auto_parallel, use a complete tensor to + # compile, and slice tensor to run. The batch dimension of tensors for compile is device_number + # times the batch dimension of tensors for run + if _get_parallel_mode() in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): + device_num = _get_device_num() + self.dataset_shapes = _to_full_shapes(dataset_shapes, device_num) + + def __iter__(self): + self.ind = 0 + return self + + def __next__(self): + if self.ind >= self.loop_count: + raise StopIteration() + self.ind += 1 + return self.op() + + def types_shapes(self): + return self.dataset_types, self.dataset_shapes + + def get_loop_count(self, dataset): + loop_count = 1 + if hasattr(dataset, '__loop_size__'): + loop_size = dataset.__loop_size__ + loop_count = int(dataset.get_dataset_size()/loop_size) + return loop_count + + +class _DatasetIterMSLoopSink(_DatasetIter): + """Iter for context (enable_loop_sink=True)""" + def __init__(self, dataset, first_order_iter): + super(_DatasetIterMSLoopSink, self).__init__(dataset) + # self.loop_count = self.get_loop_count(dataset) + loop_size = dataset.__loop_size__ + first_order_iter + self.loop_count = int(dataset.get_dataset_size()/loop_size) * 2 + + def op(): + return tuple() + self.op = op + + +class _DatasetIterMS(_DatasetIter): + """Iter for context (enable_loop_sink=False)""" + def __init__(self, dataset, first_order_order): + super(_DatasetIterMS, self).__init__(dataset) + self.loop_count = dataset.get_dataset_size() + self.loop_size = 1 + queue_name = dataset.__ME_INITED__ + self.op = GetNextSingleOp(self.dataset_types, self.dataset_shapes, queue_name) + + +class _DatasetIterGE(_DatasetIter): + """Iter for ge""" + def __init__(self, dataset): + super(_DatasetIterGE, self).__init__(dataset) + self.loop_count = self.get_loop_count(dataset) + parallel_mode = _get_parallel_mode() + self.need_to_full = parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL) + batch_expand_num = 1 + if self.need_to_full: + batch_expand_num = _get_device_num() + tensor_list_run = _construct_tensor_list(self.dataset_types, self.dataset_shapes, batch_expand_num) + + def op(): + return tensor_list_run + self.op = op + + +class _DatasetIterFeed: + """Iter for feed data""" + def __init__(self, dataset, first_order_order): + self.dataset = dataset + self.device_num = _get_device_num() + self.global_rank = _get_global_rank() + self.repeat_count = dataset.get_repeat_count() + self.repeat_ind = 0 + self.loop_count = dataset.get_dataset_size() + self.ind = 0 + + parallel_mode = context.get_auto_parallel_context("parallel_mode") + self.need_to_full = parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL) + + def __iter__(self): + if self.repeat_ind % self.repeat_count == 0: + self.iter = self.dataset.__iter__() + + self.repeat_ind += 1 + self.ind = 0 + return self + + def __next__(self): + if self.ind >= self.loop_count: + raise StopIteration() + self.ind += 1 + data = self.iter.__next__() + if self.need_to_full: + return _to_full_tensor(data, self.device_num, self.global_rank) + return _to_tensor(data) diff --git a/tests/st/networks/thor_test/model/grad_reducer_thor.py b/tests/st/networks/thor_test/model/grad_reducer_thor.py new file mode 100644 index 00000000000..0540ebb287e --- /dev/null +++ b/tests/st/networks/thor_test/model/grad_reducer_thor.py @@ -0,0 +1,178 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from mindspore.nn.cell import Cell +from mindspore.communication.management import GlobalComm, get_group_size +from mindspore.ops import functional as F, composite as C, operations as P +from mindspore.ops.operations.comm_ops import AllReduce, ReduceOp +import mindspore.common.dtype as mstype +from mindspore.communication import create_group + +reduce_opt = C.MultitypeFuncGraph("reduce_opt") + +_all_reduce_A = AllReduce() + +def _init_optimizer_allreduce(group): + global _all_reduce_A + _all_reduce_A = AllReduce(ReduceOp.SUM, GlobalComm.WORLD_COMM_GROUP) + _all_reduce_A.add_prim_attr('fusion', group) + +@reduce_opt.register("Function", "Number", "Tensor") +def _tensors_allreduce_mean(mul, degree, grad): + degree = F.scalar_cast(degree, F.dtype(grad)) + grad = _all_reduce_A(grad) + cast_op = P.Cast() + return mul(grad, cast_op(F.scalar_to_array(1.0/degree), F.dtype(grad))) + +@reduce_opt.register("Bool", "Tensor") +def _tensors_allreduce(allreduce_filter, grad): + if allreduce_filter: + return _all_reduce_A(grad) + return grad + +_get_datatype = C.MultitypeFuncGraph("_get_datatype") + +@_get_datatype.register("Tensor") +def _tensors_get_datatype(grad): + """ + Acquire gradient datatype. + + Args: + grad (Tensor): The gradient tensor before operation. + + Returns: + mstype, the datatype of gradient. + """ + return F.dtype(grad) + + +_cast_datatype = C.MultitypeFuncGraph("_cast_datatype") + + +@_cast_datatype.register("TypeType", "Tensor") +def _tensors_cast_datatype(datatype, grad): + """ + Cast gradient to datatype. + + Args: + datatype (mstype): the destination datatype of gradient. + grad (Tensor): The gradient tensor before operation. + + Returns: + Tensor, the gradient tensor after operation. + """ + return F.cast(grad, datatype) + + +class DistributedGradReducerThor(Cell): + """ + A distributed optimizer. + + Constructs a gradient reducer Cell, which applies communication and average operations on + single-process gradient values. + + Args: + parameters (list): the parameters to be updated. + mean (bool): When mean is true, the mean coefficient (degree) would apply on gradients. Default: False. + degree (int): The mean coefficient. Usually it equals to device number. Default: None. + + Raises: + ValueError: If degree is not a int or less than 0. + + Examples: + >>> from mindspore.communication import init, get_group_size + >>> from mindspore.ops import composite as C + >>> from mindspore.ops import operations as P + >>> from mindspore.ops import functional as F + >>> from mindspore import context + >>> from mindspore import nn + >>> from mindspore import ParallelMode, ParameterTuple + >>> + >>> device_id = int(os.environ["DEVICE_ID"]) + >>> context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=True, + >>> device_id=int(device_id), enable_hccl=True) + >>> init() + >>> context.reset_auto_parallel_context() + >>> context.set_auto_parallel_context(parallel_mode=ParallelMode.DATA_PARALLEL) + >>> + >>> + >>> class TrainingWrapper(nn.Cell): + >>> def __init__(self, network, optimizer, sens=1.0): + >>> super(TrainingWrapper, self).__init__(auto_prefix=False) + >>> self.network = network + >>> self.network.add_flags(defer_inline=True) + >>> self.weights = ParameterTuple(network.trainable_params()) + >>> self.optimizer = optimizer + >>> self.grad = C.GradOperation('grad', get_by_list=True, sens_param=True) + >>> self.sens = sens + >>> self.reducer_flag = False + >>> self.grad_reducer = None + >>> self.parallel_mode = context.get_auto_parallel_context("parallel_mode") + >>> if self.parallel_mode in [ParallelMode.DATA_PARALLEL, + >>> ParallelMode.HYBRID_PARALLEL]: + >>> self.reducer_flag = True + >>> if self.reducer_flag: + >>> mean = context.get_auto_parallel_context("mirror_mean") + >>> if mean.get_device_num_is_set(): + >>> degree = context.get_auto_parallel_context("device_num") + >>> else: + >>> degree = get_group_size() + >>> self.grad_reducer = nn.DistributedGradReducer(optimizer.parameters, mean, degree) + >>> + >>> def construct(self, *args): + >>> weights = self.weights + >>> loss = self.network(*args) + >>> sens = P.Fill()(P.DType()(loss), P.Shape()(loss), self.sens) + >>> grads = self.grad(self.network, weights)(*args, sens) + >>> if self.reducer_flag: + >>> # apply grad reducer on grads + >>> grads = self.grad_reducer(grads) + >>> return F.depend(loss, self.optimizer(grads)) + >>> + >>> network = Net() + >>> optimizer = nn.Momentum(network.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> train_cell = TrainingWrapper(network, optimizer) + >>> inputs = Tensor(np.ones([16, 16]).astype(np.float32)) + >>> label = Tensor(np.zeros([16, 16]).astype(np.float32)) + >>> grads = train_cell(inputs, label) + """ + + def __init__(self, parameters, group, mean=True, degree=None): + super(DistributedGradReducerThor, self).__init__(auto_prefix=False) + self.hyper_map = C.HyperMap() + self.mul = P.Mul() + if degree is None: + self.degree = get_group_size() + else: + if not isinstance(degree, int) or degree <= 0: + raise ValueError("Parameter 'degree' in DistributedGradReducer should large than 0 and be int") + self.degree = degree + self.mean = mean + self.allreduce_filter = tuple(x.layerwise_parallel is False for x in parameters) + _init_optimizer_allreduce(group) + + def construct(self, grads): + # In some circumstances, the data precision of grads could be mixed with float16 and float32. Thus, the + # result of AllReduce is unreliable. To solve the problem, grads should be cast to float32 before AllReduce, + # and cast back after the operation. + datatypes = self.hyper_map(F.partial(_get_datatype), grads) + grads = self.hyper_map(F.partial(_cast_datatype, mstype.float32), grads) + + if self.mean: + new_grad = self.hyper_map(F.partial(reduce_opt, self.mul, self.degree), grads) + else: + new_grad = self.hyper_map(F.partial(reduce_opt), self.allreduce_filter, grads) + + new_grad = self.hyper_map(F.partial(_cast_datatype), datatypes, new_grad) + return new_grad diff --git a/tests/st/networks/thor_test/model/model_thor.py b/tests/st/networks/thor_test/model/model_thor.py new file mode 100644 index 00000000000..623a799e240 --- /dev/null +++ b/tests/st/networks/thor_test/model/model_thor.py @@ -0,0 +1,633 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Model.""" +import numpy as np +import mindspore.nn as nn +from mindspore import log as logger +from mindspore.common.tensor import Tensor +from mindspore.nn.metrics import get_metrics +from mindspore._checkparam import check_input_data, check_output_data, check_int_positive, check_bool +from mindspore.train.callback import _InternalCallbackParam, RunContext, _build_callbacks +from mindspore import context +from mindspore.parallel._utils import _get_parallel_mode, _get_device_num, _get_global_rank, \ + _get_parameter_broadcast, _device_number_check, _parameter_broadcast_check +from mindspore.nn.metrics import Loss +from mindspore.nn.wrap import WithLossCell, WithEvalCell, \ + DataWrapper +from mindspore.nn.wrap.cell_wrapper import _VirtualDatasetCell +from mindspore.train.parallel_utils import ParallelMode +from mindspore.common import dtype as mstype +from second_order.dataset_helper import DatasetHelper +from mindspore.train import amp + +from mindspore.common.dtype import pytype_to_dtype +from mindspore._c_expression import init_exec_dataset +from mindspore.common.parameter import Parameter + +def _convert_type(types): + """ + Convert from numpy type to tensor type. + + Args: + types (list): Numpy type list of element in dataset. + + Returns: + list, list of element in dataset. + """ + ms_types = [] + for np_type in types: + ms_type = pytype_to_dtype(np_type) + ms_types.append(ms_type) + return ms_types + +def _get_types_and_shapes(dataset): + """Get dataset types and shapes.""" + dataset_types = _convert_type(dataset.output_types()) + dataset_shapes = dataset.output_shapes() + return dataset_types, dataset_shapes + +def _exec_datagraph(exec_dataset, dataset_size, phase='dataset'): + """Initialize and execute the dataset graph.""" + batch_size = exec_dataset.get_batch_size() + input_indexs = exec_dataset.input_indexs + + # transform data format + dataset_types, dataset_shapes = _get_types_and_shapes(exec_dataset) + init_exec_dataset(exec_dataset.__ME_INITED__, + dataset_size, + batch_size, + dataset_types, + dataset_shapes, + input_indexs, + phase=phase) + + +class Model: + """ + High-Level API for Training or Testing. + + `Model` groups layers into an object with training and inference features. + + Args: + network (Cell): The training or testing network. + loss_fn (Cell): Objective function, if loss_fn is None, the + network should contain the logic of loss and grads calculation, and the logic + of parallel if needed. Default: None. + optimizer (Cell): Optimizer for updating the weights. Default: None. + metrics (Union[dict, set]): Dict or set of metrics to be evaluated by the model during + training and testing. eg: {'accuracy', 'recall'}. Default: None. + eval_network (Cell): Network for evaluation. If not defined, `network` and `loss_fn` would be wrapped as + `eval_network`. Default: None. + eval_indexes (list): In case of defining the `eval_network`, if `eval_indexes` is None, all outputs of + `eval_network` would be passed to metrics, otherwise `eval_indexes` must contain three + elements, representing the positions of loss value, predict value and label, the loss + value would be passed to `Loss` metric, predict value and label would be passed to other + metric. Default: None. + amp_level (str): Option for argument `level` in `mindspore.amp.build_train_network`, level for mixed + precision training. Supports [O0, O2]. Default: "O0". + + - O0: Do not change. + - O2: Cast network to float16, keep batchnorm run in float32, using dynamic loss scale. + + loss_scale_manager (Union[None, LossScaleManager]): If None, not scale the loss, or else + scale the loss by LossScaleManager. If it is set, overwrite the level setting. It's a eyword argument. + e.g. Use `loss_scale_manager=None` to set the value. + keep_batchnorm_fp32 (bool): Keep Batchnorm run in `float32`. If set, overwrite the level setting. Default: True. + + Examples: + >>> class Net(nn.Cell): + >>> def __init__(self): + >>> super(Net, self).__init__() + >>> self.conv = nn.Conv2d(3, 64, 3, has_bias=False, weight_init='normal') + >>> self.bn = nn.BatchNorm2d(64) + >>> self.relu = nn.ReLU() + >>> self.flatten = nn.Flatten() + >>> self.fc = nn.Dense(64*224*224, 12) # padding=0 + >>> + >>> def construct(self, x): + >>> x = self.conv(x) + >>> x = self.bn(x) + >>> x = self.relu(x) + >>> x = self.flatten(x) + >>> out = self.fc(x) + >>> return out + >>> + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None) + >>> dataset = get_dataset() + >>> model.train(2, dataset) + """ + + def __init__(self, network, loss_fn=None, optimizer=None, metrics=None, eval_network=None, + eval_indexes=None, amp_level="O0", frequency=278, **kwargs): + self._network = network + self._loss_fn = loss_fn + self._optimizer = optimizer + self._loss_scale_manager = None + self._loss_scale_manager_set = False + self._keep_bn_fp32 = True + self._frequency = frequency + self._check_kwargs(kwargs) + if 'keep_batchnorm_fp32' in kwargs: + self._keep_bn_fp32 = kwargs['keep_batchnorm_fp32'] + if 'loss_scale_manager' in kwargs: + self._loss_scale_manager = kwargs['loss_scale_manager'] + self._loss_scale_manager_set = True + self._amp_level = amp_level + self._parallel_mode = _get_parallel_mode() + self._device_number = _get_device_num() + self._global_rank = _get_global_rank() + self._parameter_broadcast = _get_parameter_broadcast() + + self._train_network = self._build_train_network() + self._build_eval_network(metrics, eval_network, eval_indexes) + self._build_predict_network() + + def _check_kwargs(self, kwargs): + for arg in kwargs: + if arg not in ['loss_scale_manager', 'keep_batchnorm_fp32']: + raise ValueError(f"Unsupport arg '{arg}'") + + def _build_train_network(self): + """Build train network""" + network = self._network + if self._optimizer: + if self._loss_scale_manager_set: + network = amp.build_train_network(network, + self._optimizer, + self._loss_fn, + level=self._amp_level, + loss_scale_manager=self._loss_scale_manager, + keep_batchnorm_fp32=self._keep_bn_fp32) + else: + network = amp.build_train_network(network, + self._optimizer, + self._loss_fn, + level=self._amp_level, + keep_batchnorm_fp32=self._keep_bn_fp32) + elif self._loss_fn: + network = nn.WithLossCell(network, self._loss_fn) + # If need to check if loss_fn is not None, but optimizer is None + return network + + def _build_eval_network(self, metrics, eval_network, eval_indexes): + """Build the network for evaluation.""" + self._metric_fns = get_metrics(metrics) + if not self._metric_fns: + return + + if eval_network is not None: + if eval_indexes is not None and not (isinstance(eval_indexes, list) and len(eval_indexes) == 3): + raise ValueError("Eval_indexes must be a list or None. If eval_indexes is a list, length of it \ + must be three. But got {}".format(eval_indexes)) + + self._eval_network = eval_network + self._eval_indexes = eval_indexes + else: + if self._loss_fn is None: + raise ValueError("loss_fn can not be None.") + self._eval_network = nn.WithEvalCell(self._network, self._loss_fn) + self._eval_indexes = [0, 1, 2] + + def _build_predict_network(self): + """Build the network for prediction.""" + self._predict_network = self._network + if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): + self._predict_network = _VirtualDatasetCell(self._network) + + def _clear_metrics(self): + """Clear metrics local values.""" + for metric in self._metric_fns.values(): + metric.clear() + + def _update_metrics(self, outputs): + """Update metrics local values.""" + if not isinstance(outputs, tuple): + raise ValueError("The `outputs` is not tuple.") + + if self._eval_indexes is not None and len(outputs) < 3: + raise ValueError("The length of `outputs` must be greater than or equal to 3, \ + but got {}".format(len(outputs))) + + for metric in self._metric_fns.values(): + if self._eval_indexes is None: + metric.update(*outputs) + else: + if isinstance(metric, Loss): + metric.update(outputs[self._eval_indexes[0]]) + else: + metric.update(outputs[self._eval_indexes[1]], outputs[self._eval_indexes[2]]) + + def _get_metrics(self): + """Get metrics local values.""" + metrics = dict() + for key, value in self._metric_fns.items(): + metrics[key] = value.eval() + return metrics + + def _get_scaling_sens(self): + """get the scaling sens""" + scaling_sens = 1 + if self._loss_scale_manager is not None: + scaling_sens = self._loss_scale_manager.get_loss_scale() + if self._parallel_mode == ParallelMode.DATA_PARALLEL: + scaling_sens /= self._device_number + return scaling_sens + + def _train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=True): + """ + Training. + + Args: + epoch (int): Total number of iterations on the data. + train_dataset (Dataset): A training dataset iterator. If there is no + loss_fn, a tuple with multiply data (data1, data2, data3, ...) will be + returned and passed to the network. Otherwise, a tuple (data, label) will + be returned, and the data and label are passed to the network and loss + function respectively. + callbacks (list): List of callback object. Callbacks which should be executed while training. Default: None. + dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. + Configure pynative mode, the training process will be performed with + dataset not sink. + """ + epoch = check_int_positive(epoch) + self._train_network.set_train() + + if self._parameter_broadcast: + self._train_network.set_broadcast_flag() + + # build callback list + list_callback = _build_callbacks(callbacks) + cb_params = _InternalCallbackParam() + cb_params.train_network = self._train_network + cb_params.epoch_num = epoch + cb_params.batch_num = train_dataset.get_dataset_size() + cb_params.mode = "train" + cb_params.loss_fn = self._loss_fn + cb_params.optimizer = self._optimizer + cb_params.parallel_mode = self._parallel_mode + cb_params.device_number = self._device_number + cb_params.train_dataset = train_dataset + cb_params.list_callback = list_callback + + if dataset_sink_mode: + if context.get_context("mode") == context.PYNATIVE_MODE: + logger.warning("The pynative mode cannot support dataset sink mode currently." + "So the training process will be performed with dataset not sink.") + self._train_process(epoch, train_dataset, list_callback, cb_params) + else: + self._train_dataset_sink_process(epoch, train_dataset, list_callback, cb_params) + else: + self._train_process(epoch, train_dataset, list_callback, cb_params) + + + def _train_dataset_sink_process(self, epoch, train_dataset, list_callback=None, cb_params=None): + """ + Training process. The data would be passed to network through dataset channel. + + Args: + epoch (int): Total number of iterations on the data. + train_dataset (Dataset): A training dataset iterator. If there is no + loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be + returned and passed to the network. Otherwise, a tuple (data, label) should + be returned, and the data and label are passed to the network and loss + function respectively. + list_callback (_ListCallback): Executor of callback list. Default: None. + cb_params (_InternalCallbackParam): Callback parameters. Default: None. + """ + # remove later to deal with loop sink + iter_first_order = 277 + iter_second_order = 1 + train_dataset.__loop_size__ = iter_second_order + need_wrap = False + if not hasattr(train_dataset, '__ME_INITED__') and context.get_context("enable_loop_sink") \ + and not context.get_context("enable_ge"): + need_wrap = True + + dataset_helper = DatasetHelper(train_dataset, iter_first_order) + # remove later to deal with loop sink + if need_wrap: + self._train_network = nn.DataWrapper(self._train_network, *(dataset_helper.types_shapes()), + train_dataset.__ME_INITED__) + cb_params.train_network = self._train_network + self._train_network.set_train() + + cb_params.cur_step_num = 0 + loop_size = dataset_helper.loop_size() + run_context = RunContext(cb_params) + list_callback.begin(run_context) + + # used to stop training for early stop, such as stopAtTIme or stopATStep + should_stop = False + has_do_train1_dataset = False + checkpoint_branch_one = True + for i in range(epoch): + cb_params.cur_epoch_num = i + 1 + list_callback.epoch_begin(run_context) + + # for data sink dataset_helper only iter once, other wise iter epoch_size times. + for inputs in dataset_helper: + list_callback.step_begin(run_context) + if checkpoint_branch_one: + cb_params.cur_step_num += loop_size + self._train_network.set_second_order(True) + self._train_network.phase = 'train0' + else: + cb_params.cur_step_num += iter_first_order + self._train_network.set_second_order(False) + self._train_network.phase = 'train1' + if not has_do_train1_dataset: + _exec_datagraph(train_dataset, iter_first_order, phase='train1_dataset') + has_do_train1_dataset = True + checkpoint_branch_one = not checkpoint_branch_one + outputs = self._train_network(*inputs) + cb_params.net_outputs = outputs + list_callback.step_end(run_context) + + list_callback.epoch_end(run_context) + should_stop = should_stop or run_context.get_stop_requested() + if should_stop: + break + + list_callback.end(run_context) + + def _train_process(self, epoch, train_dataset, list_callback=None, cb_params=None): + """ + Training process. The data would be passed to network directly. + + Args: + epoch (int): Total number of iterations on the data. + train_dataset (Dataset): A training dataset iterator. If there is no + loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be + returned and passed to the network. Otherwise, a tuple (data, label) should + be returned, and the data and label are passed to the network and loss + function respectively. + list_callback (_ListCallback): Executor of callback list. Default: None. + cb_params (_InternalCallbackParam): Callback parameters. Default: None. + """ + dataset_helper = DatasetHelper(train_dataset, dataset_sink_mode=False) + cb_params.cur_step_num = 0 + run_context = RunContext(cb_params) + _callback_wrapper(list_callback, run_context, "begin") + # used to stop training for early stop, such as stopAtTIme or stopATStep + should_stop = False + + for i in range(epoch): + cb_params.cur_epoch_num = i + 1 + + _callback_wrapper(list_callback, run_context, "epoch_begin") + + for next_element in dataset_helper: + len_element = len(next_element) + if self._loss_fn and len_element != 2: + raise ValueError("when loss_fn is not None, train_dataset should" + "return two elements, but got {}".format(len_element)) + cb_params.cur_step_num += 1 + _callback_wrapper(list_callback, run_context, "step_begin") + + overflow = False + if self._loss_scale_manager and self._loss_scale_manager.get_drop_overflow_update(): + scaling_sens = self._get_scaling_sens() + next_element = tuple(next_element) + (Tensor(scaling_sens, mstype.float32),) + + outputs = self._train_network(*next_element) + cb_params.net_outputs = outputs + if self._loss_scale_manager and self._loss_scale_manager.get_drop_overflow_update(): + _, overflow, _ = outputs + overflow = np.all(overflow.asnumpy()) + self._loss_scale_manager.update_loss_scale(overflow) + + _callback_wrapper(list_callback, run_context, "step_end") + should_stop = should_stop or run_context.get_stop_requested() + if should_stop: + break + + train_dataset.reset() + + _callback_wrapper(list_callback, run_context, "epoch_end") + should_stop = should_stop or run_context.get_stop_requested() + if should_stop: + break + + _callback_wrapper(list_callback, run_context, "end") + + def train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=True): + """ + Training API where the iteration is controlled by python front-end. + + When setting pynative mode, the training process will be performed with dataset not sink. + + Note: + CPU is not supported when dataset_sink_mode is true. + If dataset_sink_mode is True, epoch of training should be equal to the count of repeat + operation in dataset processing. Otherwise, errors could occur since the amount of data + is not the amount training requires. + If dataset_sink_mode is True, data will be sent to device. If device is Ascend, features + of data will be transferred one by one. The limitation of data transmission per time is 256M. + + Args: + epoch (int): Total number of iterations on the data. + train_dataset (Dataset): A training dataset iterator. If there is no + loss_fn, a tuple with multiply data (data1, data2, data3, ...) should be + returned and passed to the network. Otherwise, a tuple (data, label) should + be returned, and the data and label are passed to the network and loss + function respectively. + callbacks (list): List of callback object. Callbacks which should be excuted while training. Default: None. + dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. + Configure pynative mode, the training process will be performed with + dataset not sink. + + + Examples: + >>> dataset = get_dataset() + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> loss_scale_manager = FixedLossScaleManager() + >>> optim = Momentum(params=net.trainable_params(), learning_rate=0.1, momentum=0.9) + >>> model = Model(net, loss_fn=loss, optimizer=optim, metrics=None, loss_scale_manager=loss_scale_manager) + >>> model.train(2, dataset) + """ + repeat_count = train_dataset.get_repeat_count() + if epoch != repeat_count and dataset_sink_mode is True: + logger.warning(f"The epoch_size {epoch} is not the same with dataset repeat_count {repeat_count}") + check_bool(dataset_sink_mode) + _device_number_check(self._parallel_mode, self._device_number) + _parameter_broadcast_check(self._parallel_mode, self._parameter_broadcast) + + if context.get_context("device_target") in ["CPU", "GPU"] and context.get_context("enable_loop_sink"): + raise ValueError("CPU and GPU can't support loop sink, please set enable_loop_sink=False.") + + self._train(epoch, + train_dataset, + callbacks=callbacks, + dataset_sink_mode=dataset_sink_mode) + + def _eval_dataset_sink_process(self, valid_dataset, list_callback=None, cb_params=None): + """ + Evaluation. The data would be passed to network through dataset channel. + + Args: + valid_dataset (Dataset): Dataset to evaluate the model. + list_callback (ListCallback): Executor of callback list. Default: None. + cb_params (_InternalCallbackParam): Callback parameters. Default: None. + + Returns: + Dict, returns the loss value & metrics values for the model in test mode. + """ + _device_number_check(self._parallel_mode, self._device_number) + + run_context = RunContext(cb_params) + + # remove later to deal with loop sink + need_wrap = False + if not hasattr(valid_dataset, '__ME_INITED__') and context.get_context("enable_loop_sink") \ + and not context.get_context("enable_ge"): + need_wrap = True + + valid_dataset.__loop_size__ = 1 + dataset_helper = DatasetHelper(valid_dataset) + + # remove later to deal with loop sink + if need_wrap: + self._eval_network = nn.DataWrapper(self._eval_network, *(dataset_helper.types_shapes()), + valid_dataset.__ME_INITED__) + self._eval_network.set_train(mode=False) + self._eval_network.phase = 'eval' + list_callback.begin(run_context) + + for inputs in dataset_helper: + cb_params.cur_step_num += 1 + list_callback.step_begin(run_context) + + outputs = self._eval_network(*inputs) + + cb_params.net_outputs = outputs + list_callback.step_end(run_context) + self._update_metrics(outputs) + + metrics = self._get_metrics() + cb_params.metrics = metrics + list_callback.end(run_context) + + return metrics + + def _eval_process(self, valid_dataset, list_callback=None, cb_params=None): + """ + Evaluation. The data would be passed to network directly. + + Args: + valid_dataset (Dataset): Dataset to evaluate the model. + list_callback (ListCallback): Executor of callback list. Default: None. + cb_params (_InternalCallbackParam): Callback parameters. Default: None. + + Returns: + Dict, returns the loss value & metrics values for the model in test mode. + """ + run_context = RunContext(cb_params) + list_callback.begin(run_context) + + dataset_helper = DatasetHelper(valid_dataset, dataset_sink_mode=False) + for next_element in dataset_helper: + cb_params.cur_step_num += 1 + list_callback.step_begin(run_context) + outputs = self._eval_network(*next_element) + cb_params.net_outputs = outputs + list_callback.step_end(run_context) + self._update_metrics(outputs) + + metrics = self._get_metrics() + cb_params.metrics = metrics + list_callback.end(run_context) + return metrics + + def eval(self, valid_dataset, callbacks=None, dataset_sink_mode=True): + """ + Evaluation API where the iteration is controlled by python front-end. + + Configure to pynative mode, the evaluation will be performed with dataset non-sink mode. + + Note: + CPU is not supported when dataset_sink_mode is true. + If dataset_sink_mode is True, data will be sent to device. If device is Ascend, features + of data will be transferred one by one. The limitation of data transmission per time is 256M. + + Args: + valid_dataset (Dataset): Dataset to evaluate the model. + callbacks (list): List of callback object. Callbacks which should be excuted + while training. Default: None. + dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. + + Returns: + Dict, returns the loss value & metrics values for the model in test mode. + + Examples: + >>> dataset = get_dataset() + >>> net = Net() + >>> loss = nn.SoftmaxCrossEntropyWithLogits() + >>> model = Model(net, loss_fn=loss, optimizer=None, metrics={'acc'}) + >>> model.eval(dataset) + """ + check_bool(dataset_sink_mode) + if not self._metric_fns: + raise ValueError("metric fn can not be None or empty.") + + list_callback = _build_callbacks(callbacks) + cb_params = _InternalCallbackParam() + cb_params.eval_network = self._eval_network + cb_params.valid_dataset = valid_dataset + cb_params.batch_num = valid_dataset.get_dataset_size() + cb_params.mode = "eval" + cb_params.cur_step_num = 0 + + self._eval_network.set_train(mode=False) + self._eval_network.phase = 'eval' + + self._clear_metrics() + + if dataset_sink_mode: + return self._eval_dataset_sink_process(valid_dataset, list_callback, cb_params) + return self._eval_process(valid_dataset, list_callback, cb_params) + + def predict(self, *predict_data): + """ + Generates output predictions for the input samples. + + Data could be single tensor, or list of tensor, tuple of tensor. + + Note: + Batch data should be put together in one tensor. + + Args: + predict_data (Tensor): Tensor of predict data. can be array, list or tuple. + + Returns: + Tensor, array(s) of predictions. + + Examples: + >>> input_data = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]), mindspore.float32) + >>> model = Model(Net()) + >>> model.predict(input_data) + """ + self._predict_network.set_train(False) + check_input_data(*predict_data, data_class=Tensor) + result = self._predict_network(*predict_data) + + check_output_data(result) + return result + + +__all__ = ["Model"] diff --git a/tests/st/networks/thor_test/model/resnet.py b/tests/st/networks/thor_test/model/resnet.py new file mode 100644 index 00000000000..d77239dbf29 --- /dev/null +++ b/tests/st/networks/thor_test/model/resnet.py @@ -0,0 +1,342 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""ResNet.""" +import numpy as np +import mindspore.nn as nn +from mindspore.ops import operations as P +from mindspore.common.tensor import Tensor +from second_order.thor_layer import Conv2d_Thor, Dense_Thor +import math + +def calculate_gain(nonlinearity, param=None): + linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d'] + if nonlinearity in linear_fns or nonlinearity == 'sigmoid': + return 1 + elif nonlinearity == 'tanh': + return 5.0 / 3 + elif nonlinearity == 'relu': + return math.sqrt(2.0) + elif nonlinearity == 'leaky_relu': + if param is None: + negative_slope = 0.01 + elif not isinstance(param, bool) and isinstance(param, int) or isinstance(param, float): + # True/False are instances of int, hence check above + negative_slope = param + else: + raise ValueError("negative_slope {} not a valid number".format(param)) + return math.sqrt(2.0 / (1 + negative_slope ** 2)) + else: + raise ValueError("Unsupported nonlinearity {}".format(nonlinearity)) + +def _calculate_fan_in_and_fan_out(tensor): + dimensions = len(tensor) + if dimensions < 2: + raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions") + + if dimensions == 2: # Linear + fan_in = tensor[1] + fan_out = tensor[0] + else: + num_input_fmaps = tensor[1] + num_output_fmaps = tensor[0] + receptive_field_size = 1 + if dimensions > 2: + receptive_field_size = tensor[2] * tensor[3] + fan_in = num_input_fmaps * receptive_field_size + fan_out = num_output_fmaps * receptive_field_size + return fan_in, fan_out + +def _calculate_correct_fan(tensor, mode): + mode = mode.lower() + valid_modes = ['fan_in', 'fan_out'] + if mode not in valid_modes: + raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes)) + + fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) + return fan_in if mode == 'fan_in' else fan_out + +def kaiming_normal(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu'): + fan = _calculate_correct_fan(inputs_shape, mode) + gain = calculate_gain(nonlinearity, a) + std = gain / math.sqrt(fan) + return np.random.normal(0, std, size=inputs_shape).astype(np.float32) + +def kaiming_uniform(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu'): + fan = _calculate_correct_fan(inputs_shape, mode) + gain = calculate_gain(nonlinearity, a) + std = gain / math.sqrt(fan) + bound = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation + return np.random.uniform(-bound, bound, size=inputs_shape).astype(np.float32) + +def _conv3x3(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): + weight_shape = (out_channel, in_channel, 3, 3) + weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) + return Conv2d_Thor(in_channel, out_channel, + kernel_size=3, stride=stride, padding=0, pad_mode='same', weight_init=weight, + damping=damping, loss_scale=loss_scale, frequency=frequency) + # return nn.Conv2d(in_channel, out_channel, + # kernel_size=3, stride=stride, padding=0, pad_mode='same', weight_init=weight) + +def _conv1x1(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): + weight_shape = (out_channel, in_channel, 1, 1) + weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) + return Conv2d_Thor(in_channel, out_channel, + kernel_size=1, stride=stride, padding=0, pad_mode='same', weight_init=weight, + damping=damping, loss_scale=loss_scale, frequency=frequency) + +def _conv7x7(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): + weight_shape = (out_channel, in_channel, 7, 7) + weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) + return Conv2d_Thor(in_channel, out_channel, + kernel_size=7, stride=stride, padding=0, pad_mode='same', weight_init=weight, + damping=damping, loss_scale=loss_scale, frequency=frequency) + +def _bn(channel): + return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, + gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) + +def _bn_last(channel): + return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, + gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) + +def _fc(in_channel, out_channel, damping, loss_scale, frequency): + weight_shape = (out_channel, in_channel) + weight = Tensor(kaiming_uniform(weight_shape, a = math.sqrt(5)) + return Dense_Thor(in_channel, out_channel, has_bias=False, weight_init=weight, bias_init=0, + damping=damping, loss_scale=loss_scale, frequency=frequency) + +class ResidualBlock(nn.Cell): + """ + ResNet V1 residual block definition. + + Args: + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. Default: 1. + + Returns: + Tensor, output tensor. + + Examples: + >>> ResidualBlock(3, 256, stride=2) + """ + expansion = 4 + + def __init__(self, + in_channel, + out_channel, + stride=1, + damping=0.03, + loss_scale=1, + frequency=278): + super(ResidualBlock, self).__init__() + + channel = out_channel // self.expansion + self.conv1 = _conv1x1(in_channel, channel, stride=1, damping=damping, loss_scale=loss_scale, frequency=frequency) + self.bn1 = _bn(channel) + + self.conv2 = _conv3x3(channel, channel, stride=stride, damping=damping, loss_scale=loss_scale, frequency=frequency) + self.bn2 = _bn(channel) + + self.conv3 = _conv1x1(channel, out_channel, stride=1, damping=damping, loss_scale=loss_scale, frequency=frequency) + self.bn3 = _bn_last(out_channel) + + self.relu = nn.ReLU() + + self.down_sample = False + + if stride != 1 or in_channel != out_channel: + self.down_sample = True + self.down_sample_layer = None + + if self.down_sample: + self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channel, out_channel, stride, + damping=damping, loss_scale=loss_scale, frequency=frequency), + _bn(out_channel)]) + self.add = P.TensorAdd() + + def construct(self, x): + identity = x + + out = self.conv1(x) + out = self.bn1(out) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out) + out = self.relu(out) + + out = self.conv3(out) + out = self.bn3(out) + + if self.down_sample: + identity = self.down_sample_layer(identity) + + out = self.add(out, identity) + out = self.relu(out) + + return out + + +class ResNet(nn.Cell): + """ + ResNet architecture. + + Args: + block (Cell): Block for network. + layer_nums (list): Numbers of block in different layers. + in_channels (list): Input channel in each layer. + out_channels (list): Output channel in each layer. + strides (list): Stride size in each layer. + num_classes (int): The number of classes that the training images are belonging to. + Returns: + Tensor, output tensor. + + Examples: + >>> ResNet(ResidualBlock, + >>> [3, 4, 6, 3], + >>> [64, 256, 512, 1024], + >>> [256, 512, 1024, 2048], + >>> [1, 2, 2, 2], + >>> 10) + """ + + def __init__(self, + block, + layer_nums, + in_channels, + out_channels, + strides, + num_classes, + damping, + loss_scale, + frequency): + super(ResNet, self).__init__() + + if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: + raise ValueError("the length of layer_num, in_channels, out_channels list must be 4!") + + self.conv1 = _conv7x7(3, 64, stride=2, damping=damping, loss_scale=loss_scale, frequency=frequency) + self.bn1 = _bn(64) + self.relu = P.ReLU() + self.maxpool = P.MaxPoolWithArgmax(padding="same", ksize=3, strides=2) + + self.layer1 = self._make_layer(block, + layer_nums[0], + in_channel=in_channels[0], + out_channel=out_channels[0], + stride=strides[0], + damping=damping, + loss_scale=loss_scale, + frequency=frequency) + self.layer2 = self._make_layer(block, + layer_nums[1], + in_channel=in_channels[1], + out_channel=out_channels[1], + stride=strides[1], + damping=damping, + loss_scale=loss_scale, + frequency=frequency) + self.layer3 = self._make_layer(block, + layer_nums[2], + in_channel=in_channels[2], + out_channel=out_channels[2], + stride=strides[2],damping=damping, + loss_scale=loss_scale, + frequency=frequency) + self.layer4 = self._make_layer(block, + layer_nums[3], + in_channel=in_channels[3], + out_channel=out_channels[3], + stride=strides[3], + damping=damping, + loss_scale=loss_scale, + frequency=frequency) + + self.mean = P.ReduceMean(keep_dims=True) + self.flatten = nn.Flatten() + self.end_point = _fc(out_channels[3], num_classes, damping=damping, loss_scale=loss_scale, frequency=frequency) + + def _make_layer(self, block, layer_num, in_channel, out_channel, stride, + damping, loss_scale, frequency): + """ + Make stage network of ResNet. + + Args: + block (Cell): Resnet block. + layer_num (int): Layer number. + in_channel (int): Input channel. + out_channel (int): Output channel. + stride (int): Stride size for the first convolutional layer. + + Returns: + SequentialCell, the output layer. + + Examples: + >>> _make_layer(ResidualBlock, 3, 128, 256, 2) + """ + layers = [] + + resnet_block = block(in_channel, out_channel, stride=stride, + damping=damping, loss_scale=loss_scale, frequency=frequency) + layers.append(resnet_block) + + for _ in range(1, layer_num): + resnet_block = block(out_channel, out_channel, stride=1, + damping=damping, loss_scale=loss_scale, frequency=frequency) + layers.append(resnet_block) + + return nn.SequentialCell(layers) + + def construct(self, x): + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + c1, argmax = self.maxpool(x) + + c2 = self.layer1(c1) + c3 = self.layer2(c2) + c4 = self.layer3(c3) + c5 = self.layer4(c4) + + out = self.mean(c5, (2, 3)) + out = self.flatten(out) + out = self.end_point(out) + + return out + + +def resnet50(class_num=10, damping=0.03, loss_scale=1, frequency=278): + """ + Get ResNet50 neural network. + + Args: + class_num (int): Class number. + + Returns: + Cell, cell instance of ResNet50 neural network. + + Examples: + >>> net = resnet50(10) + """ + return ResNet(ResidualBlock, + [3, 4, 6, 3], + [64, 256, 512, 1024], + [256, 512, 1024, 2048], + [1, 2, 2, 2], + class_num, + damping, + loss_scale, + frequency) diff --git a/tests/st/networks/thor_test/model/thor.py b/tests/st/networks/thor_test/model/thor.py new file mode 100644 index 00000000000..33d11c018fd --- /dev/null +++ b/tests/st/networks/thor_test/model/thor.py @@ -0,0 +1,195 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""momentum""" +import numpy as np +from mindspore.ops import functional as F, composite as C, operations as P +from mindspore.common.parameter import Parameter +from mindspore.common.tensor import Tensor +import mindspore.common.dtype as mstype +from mindspore.nn.optim.optimizer import Optimizer +from mindspore.common.parameter import ParameterTuple +from mindspore.parallel._utils import _get_device_num, _get_parallel_mode, _get_mirror_mean +from mindspore.common.initializer import initializer +from model.grad_reducer_thor import DistributedGradReducerThor +from cus_ops.cus_matmul_cube_fracz_right_mul import CusMatMulCubeFraczRightMul +from cus_ops.cus_fused_abs_max1 import CusFusedAbsMax1 +from cus_ops.cus_matmul_cube_fracz_left_cast import CusMatMulCubeFraczLeftCast +from cus_ops.cus_matmul_cube_dense_left import CusMatMulCubeDenseLeft +from cus_ops.cus_matmul_cube_dense_right import CusMatMulCubeDenseRight + +momentum_opt = C.MultitypeFuncGraph("momentum_opt") +@momentum_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor") +def _tensor_run_opt_ext(opt, learning_rate, momentum, gradient, weight, moment): + """Apply momentum optimizer to the weight parameter using Tensor.""" + success = True + success = F.depend(success, opt(weight, moment, learning_rate, gradient, momentum)) + return success + +op_add = P.AddN() +apply_decay = C.MultitypeFuncGraph("apply_decay") + +@apply_decay.register("Number", "Bool", "Tensor", "Tensor") +def _tensor_apply_decay(weight_decay, if_apply, weight, gradient): + """Get grad with weight_decay.""" + if if_apply: + return op_add((weight * weight_decay, gradient)) + return gradient + +class THOR(Optimizer): + def __init__(self, params, learning_rate, momentum, matrix_A, matrix_G, A_inv_max, G_inv_max, weight_decay=0.0, loss_scale=1.0, + decay_filter=lambda x: x.name not in []): + super(THOR, self).__init__(learning_rate, params, weight_decay, loss_scale) + if isinstance(momentum, float) and momentum < 0.0: + raise ValueError("momentum should be at least 0.0, but got momentum {}".format(momentum)) + self.momentum = Parameter(Tensor(momentum, mstype.float32), name="momentum") + self.params = self.parameters + self.moments = self.params.clone(prefix="moments", init='zeros') + self.hyper_map = C.HyperMap() + self.opt = P.ApplyMomentum() + self.matrix_A = ParameterTuple(matrix_A) + self.matrix_G = ParameterTuple(matrix_G) + self.A_inv_max = ParameterTuple(A_inv_max) + self.G_inv_max = ParameterTuple(G_inv_max) + self.cube_matmul_left = CusMatMulCubeFraczLeftCast() + self.cube_matmul_left_fc = CusMatMulCubeDenseLeft() + self.cube_matmul_right_fc = CusMatMulCubeDenseRight() + self.cube_matmul_right_mul = CusMatMulCubeFraczRightMul() + self.transpose = P.Transpose() + self.shape = P.Shape() + self.reshape = P.Reshape() + self.mul = P.Mul() + self.weight_idx = [] + for i in range(len(self.params)): + if "conv" in self.params[i].name or "end_point" in self.params[i].name: + self.weight_idx.append(i) + self.weight_idx.append(len(self.params)) + self.feature_map = [1.0 / 12544, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, + 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, 1.0 / 3136, + 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, + 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, 1.0 / 784, + 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, + 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, 1.0 / 196, + 1.0 / 196, 1.0 / 196, 1.0 / 196, + 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, 1.0 / 49, + 1.0] + mean = _get_mirror_mean() + degree = _get_device_num() + self.grad_reducer_Amax = DistributedGradReducerThor(self.parameters, 2, mean, degree) + self.grad_reducer_Gmax = DistributedGradReducerThor(self.parameters, 5, mean, degree) + self.grad_reducer_A = DistributedGradReducerThor(self.parameters, 3, mean, degree) + self.grad_reducer_G = DistributedGradReducerThor(self.parameters, 4, mean, degree) + self.matrix_A_inv = () + self.matrix_G_inv = () + self.matrix_max_inv = () + + for i in range(54): + self.matrix_max_inv = self.matrix_max_inv + (Parameter(initializer(1, [1], mstype.float32), name="matrix_max"+str(i), requires_grad=False), ) + self.log = P.Log() + self.exp = P.Exp() + self.sqrt = P.Sqrt() + self.matrix_max_inv = ParameterTuple(self.matrix_max_inv) + self.assign = P.Assign() + self.cast = P.Cast() + self.thor = True + self.weight_decay = weight_decay * loss_scale + self.decay_flags = tuple(decay_filter(x) for x in self.parameters) + + def construct(self, gradients): + params = self.params + moments = self.moments + if self.thor: + matrix_A_allreduce = () + matrix_G_allreduce = () + matrix_A_max_allreduce = () + matrix_G_max_allreduce = () + for i in range(54): + g = gradients[i * 3] + matrix_A = self.matrix_A[i] + matrix_G = self.matrix_G[i] + A_max = self.A_inv_max[i] + G_max = self.G_inv_max[i] + matrix_A = F.depend(matrix_A, g) + matrix_G = F.depend(matrix_G, g) + A_max = F.depend(A_max, g) + G_max = F.depend(G_max, g) + matrix_A_allreduce = matrix_A_allreduce + (matrix_A, ) + matrix_G_allreduce = matrix_G_allreduce + (matrix_G, ) + matrix_A_max_allreduce = matrix_A_max_allreduce + (A_max, ) + matrix_G_max_allreduce = matrix_G_max_allreduce + (G_max,) + matrix_A_allreduce = self.grad_reducer_A(matrix_A_allreduce) + matrix_G_allreduce = self.grad_reducer_G(matrix_G_allreduce) + matrix_A_max_allreduce = self.grad_reducer_Amax(matrix_A_max_allreduce) + matrix_G_max_allreduce = self.grad_reducer_Gmax(matrix_G_max_allreduce) + new_grads = () + for i in range(54): + g = gradients[i * 3] + temp_a = matrix_A_allreduce[i] + temp_g = matrix_G_allreduce[i] + temp_a = self.cast(temp_a, mstype.float32) + temp_g = self.cast(temp_g, mstype.float32) + matrix_A_inv_max = self.log(matrix_A_max_allreduce[i]) + matrix_A_inv_max = self.mul(matrix_A_inv_max, -1) + matrix_A_inv_max = self.exp(matrix_A_inv_max) + temp_a = self.mul(temp_a, matrix_A_inv_max) + matrix_G_inv_max = self.log(matrix_G_max_allreduce[i]) + matrix_G_inv_max = self.mul(matrix_G_inv_max, -1) + matrix_G_inv_max = self.exp(matrix_G_inv_max) + temp_g = self.mul(temp_g, matrix_G_inv_max) + temp_max = self.mul(matrix_A_max_allreduce[i], matrix_G_max_allreduce[i]) + temp_max = self.mul(temp_max, self.feature_map[i]) + if i == 53: + g = self.cube_matmul_left_fc(temp_g, g) + g = self.cube_matmul_right_fc(g, temp_a, temp_max) + else: + g = self.cube_matmul_left(temp_g, g) + g = self.cube_matmul_right_mul(g, temp_a, temp_max) + fake_A = self.assign(self.matrix_A[i], temp_a) + fake_G = self.assign(self.matrix_G[i], temp_g) + fake_max = self.assign(self.matrix_max_inv[i], temp_max) + g = F.depend(g, fake_A) + g = F.depend(g, fake_G) + g = F.depend(g, fake_max) + if i == 53: + new_grads = new_grads + (g,) + else: + new_grads = new_grads + (g, gradients[i * 3 + 1], gradients[i * 3 + 2]) + gradients = new_grads + else: + new_grads = () + for i in range(54): + g = gradients[i * 3] + matrix_A = self.matrix_A[i] + matrix_G = self.matrix_G[i] + matrix_max = self.matrix_max_inv[i] + matrix_A = F.depend(matrix_A, g) + matrix_G = F.depend(matrix_G, g) + matrix_max = F.depend(matrix_max, g) + if i == 53: + g = self.cube_matmul_left_fc(matrix_G, g) + g = self.cube_matmul_right_fc(g, matrix_A, matrix_max) + new_grads = new_grads + (g,) + else: + g = self.cube_matmul_left(matrix_G, g) + g =self.cube_matmul_right_mul(g, matrix_A, matrix_max) + new_grads = new_grads + (g, gradients[i * 3 + 1], gradients[i * 3 + 2]) + gradients = new_grads + + if self.weight_decay > 0: + gradients = self.hyper_map(F.partial(apply_decay, self.weight_decay), self.decay_flags, + params, gradients) + gradients = self.scale_grad(gradients) + lr = self.get_lr() + success = self.hyper_map(F.partial(momentum_opt, self.opt, lr, self.momentum), gradients, params, moments) + return success diff --git a/tests/st/networks/thor_test/model/thor_layer.py b/tests/st/networks/thor_test/model/thor_layer.py new file mode 100644 index 00000000000..70c207b7e2d --- /dev/null +++ b/tests/st/networks/thor_test/model/thor_layer.py @@ -0,0 +1,450 @@ +import numpy as np +import mindspore as ms +import mindspore.common.dtype as mstype +from mindspore.ops import operations as P +from mindspore.common.parameter import Parameter +from mindspore.common.initializer import initializer +from mindspore._checkparam import check_bool, twice, check_int_positive +from mindspore.nn.cell import Cell +from mindspore.ops import functional as F +from mindspore.common.tensor import Tensor +from mindspore.nn.layer.activation import get_activation +from mindspore._extends import cell_attr_register +from cus_ops.cus_matmul_cube import CusMatMulCube +from cus_ops.cus_matrix_combine import CusMatrixCombine +from cus_ops.cus_cholesky_trsm import CusCholeskyTrsm +from cus_ops.cus_img2col import CusImg2Col +from cus_ops.cus_fused_abs_max1 import CusFusedAbsMax1 +from cus_ops.cus_batch_matmul import CusBatchMatMul +from cus_ops.cus_transpose02314 import CusTranspose02314 + +C0 = 16 +def caculate_device_shape(matrix_dim, channel, is_A): + if is_A: + if channel // C0 == 0: + matrix_dim = (matrix_dim / channel) * C0 + return (int(matrix_dim // C0), int(matrix_dim // C0), C0, C0), int(matrix_dim) + else: + return (int(matrix_dim // C0), int(matrix_dim // C0), C0, C0), int(matrix_dim) + +class _Conv(Cell): + r"""Applies a N-D convolution over an input signal composed of several input + planes. + """ + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride, + pad_mode, + padding, + dilation, + group, + data_format, + has_bias, + weight_init, + bias_init, + ): + super(_Conv, self).__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.kernel_size = kernel_size + self.stride = stride + self.pad_mode = pad_mode + self.padding = padding + self.dilation = dilation + self.group = group + self.data_format = data_format + self.has_bias = has_bias + if not (isinstance(in_channels, int) and in_channels > 0): + raise ValueError('Attr \'in_channels\' of \'Conv2D\' Op passed ' + +str(in_channels)+ ', should be a int and greater than 0.') + if (not isinstance(kernel_size, tuple)) or len(kernel_size) != 2 or \ + (not isinstance(kernel_size[0], int)) or (not isinstance(kernel_size[1], int)) or \ + kernel_size[0] < 1 or kernel_size[1] < 1: + raise ValueError('Attr \'kernel_size\' of \'Conv2D\' Op passed ' + +str(self.kernel_size)+', should be a int or tuple and equal to or greater than 1.') + if in_channels % group != 0: + raise ValueError('Attr \'in_channels\' of \'Conv2D\' Op must be divisible by ' + 'attr \'group\' of \'Conv2D\' Op.') + if out_channels % group != 0: + raise ValueError('Attr \'out_channels\' of \'Conv2D\' Op must be divisible by ' + 'attr \'group\' of \'Conv2D\' Op.') + + self.weight = Parameter(initializer( + weight_init, [out_channels, in_channels // group, *kernel_size]), + name='weight') + + if check_bool(has_bias): + self.bias = Parameter(_initializer( + bias_init, [out_channels]), name='bias') + else: + if bias_init != 'zeros': + logger.warning("Value of 'has_bias' is False, value of 'bias_init' will be ignored.") + self.bias = None + + def construct(self, *inputs): + raise NotImplementedError + +class Conv2d_Thor(_Conv): + def __init__(self, + in_channels, + out_channels, + kernel_size, + stride=1, + pad_mode='same', + padding=0, + dilation=1, + group=1, + data_format='NCHW', + has_bias=False, + weight_init='normal', + damping=0.03, + loss_scale=1, + frequency=278, + bias_init='zeros'): + self.thor = True + ksizes = (1, kernel_size, kernel_size, 1) + self.hw = kernel_size*kernel_size + strides = (1, stride, stride, 1) + kernel_size = twice(kernel_size) + super(Conv2d_Thor, self).__init__( + in_channels, + out_channels, + kernel_size, + stride, + pad_mode, + padding, + dilation, + group, + data_format, + has_bias, + weight_init, + bias_init, + ) + self.conv2d = P.Conv2D(out_channel=self.out_channels, + kernel_size=self.kernel_size, + mode=1, + pad_mode=self.pad_mode, + pad=self.padding, + stride=self.stride, + dilation=self.dilation, + group=self.group + ) + + self.img2col = CusImg2Col(ksizes = ksizes, strides = strides) + self.cube_matmul = CusMatMulCube(transpose_a=True) + self.matrix_combine = CusMatrixCombine() + self.cholesky = CusCholeskyTrsm() + self.transpose02314 = CusTranspose02314() + self.matrix_A_dim = self.in_channels*self.kernel_size[0]*self.kernel_size[1] + self.matrix_G_dim = self.out_channels + self.matrix_A_device_shape, self.matrix_A_device_dim = caculate_device_shape(self.matrix_A_dim, self.in_channels, True) + self.matrix_G_device_shape, self.matrix_G_device_dim = caculate_device_shape(self.matrix_G_dim, self.in_channels, False) + self.matrix_A_device_temp_shape = (self.matrix_A_device_shape[0], self.matrix_A_device_shape[2], self.matrix_A_device_shape[1], self.matrix_A_device_shape[3]) + self.matrix_G_device_temp_shape = (self.matrix_G_device_shape[0], self.matrix_G_device_shape[2], self.matrix_G_device_shape[1], self.matrix_G_device_shape[3]) + self.matrix_A_inv = Parameter(Tensor(np.reshape(np.identity(self.matrix_A_device_dim).astype(np.float16), self.matrix_A_device_shape)), name='matrix_A_inv', requires_grad=False) + self.A_inv_max = Parameter(initializer(0, [1], mstype.float32), name="A_inv_max", requires_grad=False) + self.matrix_G_inv = Parameter(Tensor(np.reshape(np.identity(self.matrix_G_device_dim).astype(np.float16), self.matrix_G_device_shape)), name="matrix_G_inv", requires_grad=False) + + self.G_inv_max = Parameter(initializer(0, [1], mstype.float32), name="G_inv_max", requires_grad=False) + self.fake_G = Tensor(np.reshape(np.identity(self.matrix_G_device_dim).astype(np.float16), self.matrix_G_device_shape)) + self.fake_G_inv_max = Tensor(np.zeros([1,]).astype(np.float32)) + + self.shape = P.Shape() + self.reshape = P.Reshape() + self.transpose = P.Transpose() + self.cov_step = Parameter(initializer(0, [1], mstype.int32), name="cov_step", requires_grad=False) + self.mul = P.Mul() + self.cast = P.Cast() + self.damping = Tensor(damping) + self.vector_matmul = CusBatchMatMul() + self.diag_block_dim = 128 + self.channels_slice_flag = False + if self.in_channels % C0 != 0: + self.channels_slice_flag = True + + self.padA_flag = False + if (self.matrix_A_dim // self.diag_block_dim) * self.diag_block_dim != self.matrix_A_dim and self.matrix_A_dim > self.diag_block_dim: + self.padA_flag = True + pad_dim = self.diag_block_dim - self.matrix_A_dim % self.diag_block_dim + self.padA = P.Pad(((0, pad_dim), (0, pad_dim))) + self.device_shape_pad_flag = False + if self.matrix_A_dim != self.matrix_A_device_dim: + self.device_shape_pad_flag = True + self.device_shape_pad = P.Pad(((0, 0), (0, C0 - self.in_channels), (0, 0), (0, C0 - self.in_channels))) + self.slice = P.Slice() + self.gather = P.GatherV2() + self.freq = Tensor(frequency, mstype.int32) + self.loss_scale = Tensor(1/loss_scale, mstype.float16) + self.axis = 0 + + dampingA_dim = self.matrix_A_dim + if (self.matrix_A_dim % self.diag_block_dim) != 0 and self.matrix_A_dim > self.diag_block_dim: + dampingA_dim = (self.matrix_A_dim // self.diag_block_dim + 1) * self.diag_block_dim + dampingG_dim = self.matrix_G_dim + if (self.matrix_G_dim % self.diag_block_dim) != 0 and self.matrix_G_dim > self.diag_block_dim: + dampingG_dim = (self.matrix_G_dim // self.diag_block_dim + 1) * self.diag_block_dim + + self.dampingA = Tensor(np.identity(dampingA_dim), mstype.float32) + self.dampingG = Tensor(np.identity(dampingG_dim), mstype.float32) + self.fused_abs_max1 = CusFusedAbsMax1([self.matrix_A_dim, self.matrix_A_dim]) + self.fused_abs_max2 = CusFusedAbsMax1() + self.log = P.Log() + self.exp = P.Exp() + self.sqrt = P.Sqrt() + self.getG = P.InsertGradientOf(self.save_gradient) + + def save_gradient(self, dout): + out = dout + dout = self.mul(dout, self.loss_scale) + dout = self.mul(dout, 32.0) + dout = self.transpose02314(dout) + dout_shape = self.shape(dout) + normalizer = dout_shape[0] + + matrix_G = self.cube_matmul(dout, dout) + normalizer = self.cast(normalizer, ms.float32) + matrix_G = self.mul(matrix_G, 1.0/normalizer) + damping_step = self.gather(self.damping, self.cov_step, 0) + self.cov_step = self.cov_step + self.freq + damping_step = self.cast(damping_step, mstype.float32) + damping = self.mul(damping_step, 32.0/normalizer) + damping = self.sqrt(damping) + dampingG = self.cast(self.dampingG, mstype.float32) + matrix_G = matrix_G + damping * dampingG + + matrix_G_inv = self.cholesky(matrix_G) + matrix_G_inv = self.vector_matmul(matrix_G_inv, matrix_G_inv) + matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv) + matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv_max) + self.G_inv_max = matrix_G_inv_max + matrix_G_inv = self.matrix_combine(matrix_G_inv) + matrix_G_inv_shape = self.shape(matrix_G_inv) + matrix_G_inv = self.reshape(matrix_G_inv, self.matrix_G_device_temp_shape) + matrix_G_inv = self.transpose(matrix_G_inv, (2, 0, 1, 3)) + matrix_G = self.cast(matrix_G_inv, mstype.float16) + self.matrix_G_inv = matrix_G + return out + + def construct(self, x): + if self.thor: + matrix_A = self.img2col(x) + matrix_A_shape = self.shape(matrix_A) + normalizer = matrix_A_shape[0] + matrix_A = self.cube_matmul(matrix_A, matrix_A) + + if self.channels_slice_flag: + matrix_A = self.reshape(matrix_A, (self.hw, C0, self.hw, C0)) + matrix_A = self.slice(matrix_A, (0, 0, 0, 0), (self.hw, self.in_channels, self.hw, self.in_channels)) + matrix_A = self.reshape(matrix_A, (self.matrix_A_dim, self.matrix_A_dim)) + normalizer = self.cast(normalizer, ms.float32) + matrix_A = self.mul(matrix_A, 1.0/normalizer) + if self.padA_flag: + matrix_A = self.padA(matrix_A) + damping_step = self.gather(self.damping, self.cov_step, self.axis) + damping_step = self.cast(damping_step, mstype.float32) + damping = self.mul(damping_step, 32.0 / normalizer) + damping = self.sqrt(damping) + damping_A = self.cast(self.dampingA, mstype.float32) + matrix_A = matrix_A + damping * damping_A + matrix_A_inv = self.cholesky(matrix_A) + matrix_A_inv = self.vector_matmul(matrix_A_inv, matrix_A_inv) + matrix_A_inv_max = self.fused_abs_max1(matrix_A_inv) + matrix_A_inv_max = self.fused_abs_max2(matrix_A_inv_max) + self.A_inv_max = matrix_A_inv_max + matrix_A_inv = self.matrix_combine(matrix_A_inv) + matrix_A_inv = self.cast(matrix_A_inv, mstype.float16) + in_channels = self.in_channels + if self.padA_flag: + matrix_A_inv = self.slice(matrix_A_inv, (0, 0), (self.matrix_A_dim, self.matrix_A_dim)) + + if self.device_shape_pad_flag: + matrix_A_inv = self.reshape(matrix_A_inv, (self.hw, self.in_channels, self.hw, self.in_channels)) + matrix_A_inv = self.device_shape_pad(matrix_A_inv) + matrix_A_inv = self.reshape(matrix_A_inv, self.matrix_A_device_temp_shape) + matrix_A_inv = self.transpose(matrix_A_inv, (2, 0, 1, 3)) + self.G_inv_max = self.fake_G_inv_max + self.matrix_A_inv = matrix_A_inv + self.matrix_G_inv = self.fake_G + out = self.conv2d(x, self.weight) + out = self.getG(out) + else: + out = self.conv2d(x, self.weight) + + return out + + def extra_repr(self): + s = 'input_channels={}, output_channels={}, kernel_size={},' \ + 'stride={}, pad_mode={}, padding={}, dilation={}, ' \ + 'group={}, data_format={}, has_bias={},' \ + 'weight_init={}, bias_init={}'.format( + self.in_channels, + self.out_channels, + self.kernel_size, + self.stride, + self.pad_mode, + self.padding, + self.dilation, + self.group, + self.data_format, + self.has_bias, + self.weight, + self.bias) + + if self.has_bias: + s += ', bias={}'.format(self.bias) + return s + +class Dense_Thor(Cell): + @cell_attr_register(attrs=['has_bias', 'activation']) + def __init__(self, + in_channels, + out_channels, + weight_init='normal', + bias_init='zeros', + damping=0.03, + loss_scale=1, + frequency=278, + has_bias=True, + activation=None): + super(Dense_Thor, self).__init__() + self.in_channels = check_int_positive(in_channels) + self.out_channels = check_int_positive(out_channels) + self.has_bias = check_bool(has_bias) + self.thor = True + if isinstance(weight_init, Tensor): + if weight_init.dim() != 2 or weight_init.shape()[0] != out_channels or \ + weight_init.shape()[1] != in_channels: + raise ValueError("weight_init shape error") + + self.weight = Parameter(initializer(weight_init, [out_channels, in_channels]), name="weight") + + if self.has_bias: + if isinstance(bias_init, Tensor): + if bias_init.dim() != 1 or bias_init.shape()[0] != out_channels: + raise ValueError("bias_init shape error") + + self.bias = Parameter(initializer(bias_init, [out_channels]), name="bias") + + self.matmul = P.MatMul(transpose_b=True) + self.bias_add = P.BiasAdd() + + self.activation = get_activation(activation) + self.activation_flag = self.activation is not None + + self.matrix_A_inv = Parameter(Tensor(np.zeros([128, 128, 16, 16]).astype(np.float16)), name='matrix_A_inv', + requires_grad=False) + self.matrix_G_inv = Parameter(Tensor(np.zeros([63, 63, 16, 16]).astype(np.float16)), name="matrix_G_inv", + requires_grad=False) + self.fake_G = Tensor(np.zeros([63, 63, 16, 16]).astype(np.float16)) + + self.matmul = P.MatMul(transpose_b=True) + self.cube_matmul = CusMatMulCube(transpose_a=True) + self.matrix_combine = CusMatrixCombine() + self.cholesky = CusCholeskyTrsm() + self.shape = P.Shape() + self.reshape = P.Reshape() + self.transpose = P.Transpose() + self.cov_step = Parameter(initializer(0, [1], mstype.int32), name="cov_step", requires_grad=False) + self.mul = P.Mul() + self.cast = P.Cast() + self.damping = Tensor(damping) + self.loss_scale = Tensor(1/loss_scale, mstype.float16) + self.vector_matmul = CusBatchMatMul() + self.pad = P.Pad(((0, 24), (0, 24))) + self.pad1 = P.Pad(((0, 8), (0, 8))) + self.slice = P.Slice() + self.gather = P.GatherV2() + self.assignadd = P.AssignAdd() + self.freq = Tensor(frequency, mstype.int32) + self.axis = 0 + self.A_inv_max = Parameter(initializer(0, [1], mstype.float32), name="A_inv_max", requires_grad=False) + self.G_inv_max = Parameter(initializer(0, [1], mstype.float32), name="G_inv_max", requires_grad=False) + self.fused_abs_max1 = CusFusedAbsMax1([1000, 1000]) + self.fused_abs_max2 = CusFusedAbsMax1() + self.log = P.Log() + self.exp = P.Exp() + self.dampingA = Tensor(np.identity(2048), mstype.float32) + self.dampingG = Tensor(np.identity(1024), mstype.float32) + self.add = P.TensorAdd() + self.sqrt = P.Sqrt() + self.getG = P.InsertGradientOf(self.save_gradient) + + def save_gradient(self, dout): + out = dout + dout = self.mul(dout, self.loss_scale) + dout = self.mul(dout, 32.0) + normalizer = 32 + matrix_G = self.cube_matmul(dout, dout) + normalizer = self.cast(normalizer, ms.float32) + matrix_G = self.mul(matrix_G, 1.0 / normalizer) + matrix_G = self.pad(matrix_G) + damping_step = self.gather(self.damping, self.cov_step, 0) + damping_step = self.cast(damping_step, mstype.float32) + self.cov_step = self.cov_step + self.freq + damping = self.sqrt(damping_step) + dampingG = self.cast(self.dampingG, mstype.float32) + matrix_G = matrix_G + damping * dampingG + matrix_G_inv = self.cholesky(matrix_G) + matrix_G_inv = self.vector_matmul(matrix_G_inv, matrix_G_inv) + matrix_G_inv_max = self.fused_abs_max1(matrix_G_inv) + matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv_max) + self.G_inv_max = matrix_G_inv_max + matrix_G_inv = self.matrix_combine(matrix_G_inv) + matrix_G_inv = self.slice(matrix_G_inv, (0, 0), (1000, 1000)) + matrix_G_inv = self.pad1(matrix_G_inv) + matrix_G_inv_shape = self.shape(matrix_G_inv) + matrix_G_inv = self.reshape(matrix_G_inv, (matrix_G_inv_shape[0] / 16, 16, matrix_G_inv_shape[0] / 16, 16)) + matrix_G_inv = self.transpose(matrix_G_inv, (2, 0, 1, 3)) + matrix_G_inv = self.cast(matrix_G_inv, mstype.float16) + self.matrix_G_inv = matrix_G_inv + return out + + def construct(self, x): + if self.thor: + inputs = self.cube_matmul(x, x) + normalizer = 32 + normalizer = self.cast(normalizer, ms.float32) + matrix_A = self.mul(inputs, 1.0 / normalizer) + + damping_step = self.gather(self.damping, self.cov_step, self.axis) + damping_step = self.cast(damping_step, mstype.float32) + damping = self.sqrt(damping_step) + dampingA = self.cast(self.dampingA, mstype.float32) + matrix_A = matrix_A + damping * dampingA + matrix_A_inv = self.cholesky(matrix_A) + matrix_A_inv = self.vector_matmul(matrix_A_inv, matrix_A_inv) + + matrix_A_inv_max = self.fused_abs_max2(matrix_A_inv) + matrix_A_inv_max = self.fused_abs_max2(matrix_A_inv_max) + self.A_inv_max = matrix_A_inv_max + + matrix_A_inv = self.matrix_combine(matrix_A_inv) + matrix_A_inv_shape = self.shape(matrix_A_inv) + matrix_A_inv = self.reshape(matrix_A_inv, (matrix_A_inv_shape[0] / 16, 16, matrix_A_inv_shape[0] / 16, 16)) + matrix_A_inv = self.transpose(matrix_A_inv, (2, 0, 1, 3)) + matrix_A_inv = self.cast(matrix_A_inv, mstype.float16) + self.matrix_A_inv = matrix_A_inv + self.matrix_G_inv = self.fake_G + output = self.matmul(x, self.weight) + output = self.getG(output) + else: + output = self.matmul(x, self.weight) + + if self.has_bias: + output = self.bias_add(output, self.bias) + if self.activation_flag: + return self.activation(output) + return output + + def extend_repr(self): + str_info = 'in_channels={}, out_channels={}, weight={}, has_bias={}' \ + .format(self.in_channels, self.out_channels, self.weight, self.has_bias) + if self.has_bias: + str_info = str_info + ', bias={}'.format(self.bias) + + if self.activation_flag: + str_info = str_info + ', activation={}'.format(self.activation) + + return str_info diff --git a/tests/st/networks/thor_test/run_distribute_train_new.sh b/tests/st/networks/thor_test/run_distribute_train_new.sh new file mode 100644 index 00000000000..3179a5b3a87 --- /dev/null +++ b/tests/st/networks/thor_test/run_distribute_train_new.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +if [ $# != 3 ] +then + echo "Usage: sh run_distribute_train.sh [MINDSPORE_HCCL_CONFIG_PATH] [DATASET_PATH] [DEVICE_NUM]" +exit 1 +fi + +if [ ! -f $1 ] +then + echo "error: DMINDSPORE_HCCL_CONFIG_PATH=$1 is not a file" +exit 1 +fi + +if [ ! -d $2 ] +then + echo "error: DATASET_PATH=$2 is not a directory" +exit 1 +fi + +ulimit -u unlimited +export DEVICE_NUM=$3 +export RANK_SIZE=$3 +export MINDSPORE_HCCL_CONFIG_PATH=$1 + +for((i=0; i<${DEVICE_NUM}; i++)) +do + export DEVICE_ID=$i + export RANK_ID=$i + rm -rf ./train_parallel$i + mkdir ./train_parallel$i + cp *.py ./train_parallel$i + cp *.sh ./train_parallel$i + cp -r second_order ./train_parallel$i/second_order + cp -r test_ops ./train_parallel$i/test_ops + cd ./train_parallel$i || exit + echo "start training for rank $RANK_ID, device $DEVICE_ID" + + env > env.log + python train_0517_1.py --do_train=True --run_distribute=True --device_num=$DEVICE_NUM --dataset_path=$2 > log 2>&1 & + cd .. +done diff --git a/tests/st/networks/thor_test/train.py b/tests/st/networks/thor_test/train.py new file mode 100644 index 00000000000..6e3febd144c --- /dev/null +++ b/tests/st/networks/thor_test/train.py @@ -0,0 +1,151 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""train_imagenet.""" +import os +import argparse +import random +import numpy as np +from dataset_imagenet import create_dataset +from lr_generator import get_lr, warmup_cosine_annealing_lr +from config_imagenet import config +from mindspore import context +from mindspore import Tensor +from mindspore.parallel._auto_parallel_context import auto_parallel_context +from mindspore.nn.optim.momentum import Momentum +from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits + +from mindspore.train.model import ParallelMode + +from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor +from mindspore.train.loss_scale_manager import FixedLossScaleManager +import mindspore.dataset.engine as de +from mindspore.communication.management import init + +import math +import mindspore.nn as nn +from crossentropy import CrossEntropy +from var_init import default_recurisive_init, KaimingNormal +from mindspore.common import initializer as weight_init + +from second_order.thor import THOR +from second_order.model_second_order import Model +from second_order.resnet import resnet50 +from mindspore.train.serialization import load_checkpoint, load_param_into_net + +random.seed(1) +np.random.seed(1) +de.config.set_seed(1) + +parser = argparse.ArgumentParser(description='Image classification') +parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') +parser.add_argument('--device_num', type=int, default=1, help='Device num.') +parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') +parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') +parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') + +args_opt = parser.parse_args() +device_id = int(os.getenv('DEVICE_ID')) + +context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=True, device_id=device_id) +context.set_context(enable_task_sink=True) +context.set_context(enable_loop_sink=True) +context.set_context(enable_mem_reuse=True) + +def get_second_order_lr(global_step, lr_init, decay, total_epochs, steps_per_epoch): + lr_each_step = [] + total_steps = steps_per_epoch * total_epochs + for i in range(total_steps): + epoch = (i+1)/steps_per_epoch + base = (1.0 - float(epoch)/total_epochs)**decay + lr = lr_init * base + lr_each_step.append(lr) + current_step = global_step + lr_each_step = np.array(lr_each_step).astype(np.float32) + print("learning_rate_is=====", lr_each_step) + learning_rate = lr_each_step[current_step:] + return learning_rate + +def get_second_order_damping(global_step, damping_init, decay_rate, total_epochs, steps_per_epoch): + damping_each_step = [] + total_steps = steps_per_epoch * total_epochs + for step in range(total_steps): + epoch = (step+1) / steps_per_epoch + damping = damping_init * (decay_rate ** (epoch / 10)) + damping_each_step.append(damping) + + current_step = global_step + damping_each_step = np.array(damping_each_step).astype(np.float32) + damping = damping_each_step[current_step:] + print("damping_is=========", damping) + return damping + +if __name__ == '__main__': + if args_opt.do_eval: + print("eval") + else: + if args_opt.run_distribute: + context.set_auto_parallel_context(device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, + mirror_mean=True, parameter_broadcast=True) + auto_parallel_context().set_all_reduce_fusion_split_indices([80], "hccl_world_groupsum1") + auto_parallel_context().set_all_reduce_fusion_split_indices([27], "hccl_world_groupsum3") + auto_parallel_context().set_all_reduce_fusion_split_indices([27], "hccl_world_groupsum4") + init() + else: + print(" ") + + epoch_size = config.epoch_size + damping = get_second_order_damping(0, 0.03, 0.87, 50, 5004) + net = resnet50(class_num=config.class_num, damping=damping, loss_scale=config.loss_scale, + frequency=config.frequency) + + if not config.label_smooth: + config.label_smooth_factor = 0.0 + loss = CrossEntropy(smooth_factor=config.label_smooth_factor, num_classes=config.class_num) + if args_opt.do_train: + dataset = create_dataset(dataset_path=args_opt.dataset_path, do_train=True, + repeat_num=epoch_size, batch_size=config.batch_size) + step_size = dataset.get_dataset_size() + + loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) + if config.lr_decay_mode == 'cosine': + lr = Tensor(warmup_cosine_annealing_lr(0.035, + step_size, + config.warmup_epochs, + 50, + config.T_max, + config.eta_min)) + else: + lr = Tensor(get_lr(global_step=0, lr_init=config.lr_init, lr_end=config.lr_end, lr_max=config.lr_max, + warmup_epochs=config.warmup_epochs, total_epochs=epoch_size, steps_per_epoch=step_size, + lr_decay_mode='poly')) + opt = THOR(filter(lambda x: x.requires_grad, net.get_parameters()), lr, + config.momentum, damping, config.frequency, + filter(lambda x: 'matrix_A' in x.name, net.get_parameters()), + filter(lambda x: 'matrix_G' in x.name, net.get_parameters()), + filter(lambda x: 'spatial_norm' in x.name, net.get_parameters()), + config.weight_decay, config.loss_scale) + + model = Model(net, loss_fn=loss, optimizer=opt, amp_level='O2', loss_scale_manager=loss_scale, keep_batchnorm_fp32=False, metrics={'acc'}, frequency=config.frequency) + + time_cb = TimeMonitor(data_size=step_size) + loss_cb = LossMonitor() + cb = [time_cb, loss_cb] + if config.save_checkpoint: + config_ck = CheckpointConfig(save_checkpoint_steps=config.save_checkpoint_steps, + keep_checkpoint_max=config.keep_checkpoint_max) + ckpt_cb = ModelCheckpoint(prefix="resnet", directory=config.save_checkpoint_path, config=config_ck) + cb += [ckpt_cb] + + model.train(epoch_size, dataset, callbacks=cb) From 27c13794d2569929d2fa5e6e20f63431bc58db8f Mon Sep 17 00:00:00 2001 From: z00478463 Date: Thu, 21 May 2020 14:28:36 +0800 Subject: [PATCH 02/11] add the copyright license header --- tests/st/networks/thor_test/crossentropy.py | 15 + .../thor_test/cus_ops/batch_matmul_impl.py | 14 + .../thor_test/cus_ops/cholesky_trsm.py | 14 + .../thor_test/cus_ops/cus_cholesky_trsm.py | 14 + .../thor_test/cus_ops/cus_fused_abs_max1.py | 14 + .../thor_test/cus_ops/fused_abs_max1.py | 15 + .../thor_test/cus_ops/img2col_impl.py | 15 + .../thor_test/cus_ops/matrix_combine_impl.py | 15 + .../thor_test/cus_ops/transpose02314_impl.py | 15 + tests/st/networks/thor_test/model/' | 342 ------------------ .../st/networks/thor_test/model/thor_layer.py | 15 + 11 files changed, 146 insertions(+), 342 deletions(-) delete mode 100644 tests/st/networks/thor_test/model/' diff --git a/tests/st/networks/thor_test/crossentropy.py b/tests/st/networks/thor_test/crossentropy.py index 5746051285d..a689e018a2d 100644 --- a/tests/st/networks/thor_test/crossentropy.py +++ b/tests/st/networks/thor_test/crossentropy.py @@ -1,3 +1,18 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + from mindspore.nn.loss.loss import _Loss from mindspore.ops import operations as P from mindspore.ops import functional as F diff --git a/tests/st/networks/thor_test/cus_ops/batch_matmul_impl.py b/tests/st/networks/thor_test/cus_ops/batch_matmul_impl.py index 5af96b712eb..78e46c846b9 100644 --- a/tests/st/networks/thor_test/cus_ops/batch_matmul_impl.py +++ b/tests/st/networks/thor_test/cus_ops/batch_matmul_impl.py @@ -1,3 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ from te import tik from topi.cce import util from mindspore.ops.op_info_register import op_info_register diff --git a/tests/st/networks/thor_test/cus_ops/cholesky_trsm.py b/tests/st/networks/thor_test/cus_ops/cholesky_trsm.py index 6ed324dd0e6..3e4c86c2971 100644 --- a/tests/st/networks/thor_test/cus_ops/cholesky_trsm.py +++ b/tests/st/networks/thor_test/cus_ops/cholesky_trsm.py @@ -1,3 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ from te import tik from topi.cce import util from mindspore.ops.op_info_register import op_info_register diff --git a/tests/st/networks/thor_test/cus_ops/cus_cholesky_trsm.py b/tests/st/networks/thor_test/cus_ops/cus_cholesky_trsm.py index ce5f052c9da..02ddf981978 100644 --- a/tests/st/networks/thor_test/cus_ops/cus_cholesky_trsm.py +++ b/tests/st/networks/thor_test/cus_ops/cus_cholesky_trsm.py @@ -1,3 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ import numpy as np from mindspore.ops import prim_attr_register, PrimitiveWithInfer from mindspore import Tensor diff --git a/tests/st/networks/thor_test/cus_ops/cus_fused_abs_max1.py b/tests/st/networks/thor_test/cus_ops/cus_fused_abs_max1.py index 41a1e993cea..810966795ed 100644 --- a/tests/st/networks/thor_test/cus_ops/cus_fused_abs_max1.py +++ b/tests/st/networks/thor_test/cus_ops/cus_fused_abs_max1.py @@ -1,3 +1,17 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ import numpy as np from mindspore.ops import prim_attr_register, PrimitiveWithInfer from mindspore import Tensor diff --git a/tests/st/networks/thor_test/cus_ops/fused_abs_max1.py b/tests/st/networks/thor_test/cus_ops/fused_abs_max1.py index e11347ed64f..d2ce2d43305 100644 --- a/tests/st/networks/thor_test/cus_ops/fused_abs_max1.py +++ b/tests/st/networks/thor_test/cus_ops/fused_abs_max1.py @@ -1,3 +1,18 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + from te import tik from topi.cce import util from mindspore.ops.op_info_register import op_info_register diff --git a/tests/st/networks/thor_test/cus_ops/img2col_impl.py b/tests/st/networks/thor_test/cus_ops/img2col_impl.py index 10e4a3acae2..c39355cedfa 100644 --- a/tests/st/networks/thor_test/cus_ops/img2col_impl.py +++ b/tests/st/networks/thor_test/cus_ops/img2col_impl.py @@ -1,3 +1,18 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + from te import tik from topi.cce import util from mindspore.ops.op_info_register import op_info_register diff --git a/tests/st/networks/thor_test/cus_ops/matrix_combine_impl.py b/tests/st/networks/thor_test/cus_ops/matrix_combine_impl.py index e9cead18fb9..791892a137e 100644 --- a/tests/st/networks/thor_test/cus_ops/matrix_combine_impl.py +++ b/tests/st/networks/thor_test/cus_ops/matrix_combine_impl.py @@ -1,3 +1,18 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + from te import tik from topi.cce import util from mindspore.ops.op_info_register import op_info_register diff --git a/tests/st/networks/thor_test/cus_ops/transpose02314_impl.py b/tests/st/networks/thor_test/cus_ops/transpose02314_impl.py index be4ee94d869..627eefe4e48 100644 --- a/tests/st/networks/thor_test/cus_ops/transpose02314_impl.py +++ b/tests/st/networks/thor_test/cus_ops/transpose02314_impl.py @@ -1,3 +1,18 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + from te import tik from topi.cce import util from mindspore.ops.op_info_register import op_info_register diff --git a/tests/st/networks/thor_test/model/' b/tests/st/networks/thor_test/model/' deleted file mode 100644 index d77239dbf29..00000000000 --- a/tests/st/networks/thor_test/model/' +++ /dev/null @@ -1,342 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""ResNet.""" -import numpy as np -import mindspore.nn as nn -from mindspore.ops import operations as P -from mindspore.common.tensor import Tensor -from second_order.thor_layer import Conv2d_Thor, Dense_Thor -import math - -def calculate_gain(nonlinearity, param=None): - linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d'] - if nonlinearity in linear_fns or nonlinearity == 'sigmoid': - return 1 - elif nonlinearity == 'tanh': - return 5.0 / 3 - elif nonlinearity == 'relu': - return math.sqrt(2.0) - elif nonlinearity == 'leaky_relu': - if param is None: - negative_slope = 0.01 - elif not isinstance(param, bool) and isinstance(param, int) or isinstance(param, float): - # True/False are instances of int, hence check above - negative_slope = param - else: - raise ValueError("negative_slope {} not a valid number".format(param)) - return math.sqrt(2.0 / (1 + negative_slope ** 2)) - else: - raise ValueError("Unsupported nonlinearity {}".format(nonlinearity)) - -def _calculate_fan_in_and_fan_out(tensor): - dimensions = len(tensor) - if dimensions < 2: - raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions") - - if dimensions == 2: # Linear - fan_in = tensor[1] - fan_out = tensor[0] - else: - num_input_fmaps = tensor[1] - num_output_fmaps = tensor[0] - receptive_field_size = 1 - if dimensions > 2: - receptive_field_size = tensor[2] * tensor[3] - fan_in = num_input_fmaps * receptive_field_size - fan_out = num_output_fmaps * receptive_field_size - return fan_in, fan_out - -def _calculate_correct_fan(tensor, mode): - mode = mode.lower() - valid_modes = ['fan_in', 'fan_out'] - if mode not in valid_modes: - raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes)) - - fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) - return fan_in if mode == 'fan_in' else fan_out - -def kaiming_normal(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu'): - fan = _calculate_correct_fan(inputs_shape, mode) - gain = calculate_gain(nonlinearity, a) - std = gain / math.sqrt(fan) - return np.random.normal(0, std, size=inputs_shape).astype(np.float32) - -def kaiming_uniform(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu'): - fan = _calculate_correct_fan(inputs_shape, mode) - gain = calculate_gain(nonlinearity, a) - std = gain / math.sqrt(fan) - bound = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation - return np.random.uniform(-bound, bound, size=inputs_shape).astype(np.float32) - -def _conv3x3(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): - weight_shape = (out_channel, in_channel, 3, 3) - weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) - return Conv2d_Thor(in_channel, out_channel, - kernel_size=3, stride=stride, padding=0, pad_mode='same', weight_init=weight, - damping=damping, loss_scale=loss_scale, frequency=frequency) - # return nn.Conv2d(in_channel, out_channel, - # kernel_size=3, stride=stride, padding=0, pad_mode='same', weight_init=weight) - -def _conv1x1(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): - weight_shape = (out_channel, in_channel, 1, 1) - weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) - return Conv2d_Thor(in_channel, out_channel, - kernel_size=1, stride=stride, padding=0, pad_mode='same', weight_init=weight, - damping=damping, loss_scale=loss_scale, frequency=frequency) - -def _conv7x7(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): - weight_shape = (out_channel, in_channel, 7, 7) - weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) - return Conv2d_Thor(in_channel, out_channel, - kernel_size=7, stride=stride, padding=0, pad_mode='same', weight_init=weight, - damping=damping, loss_scale=loss_scale, frequency=frequency) - -def _bn(channel): - return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, - gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) - -def _bn_last(channel): - return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, - gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) - -def _fc(in_channel, out_channel, damping, loss_scale, frequency): - weight_shape = (out_channel, in_channel) - weight = Tensor(kaiming_uniform(weight_shape, a = math.sqrt(5)) - return Dense_Thor(in_channel, out_channel, has_bias=False, weight_init=weight, bias_init=0, - damping=damping, loss_scale=loss_scale, frequency=frequency) - -class ResidualBlock(nn.Cell): - """ - ResNet V1 residual block definition. - - Args: - in_channel (int): Input channel. - out_channel (int): Output channel. - stride (int): Stride size for the first convolutional layer. Default: 1. - - Returns: - Tensor, output tensor. - - Examples: - >>> ResidualBlock(3, 256, stride=2) - """ - expansion = 4 - - def __init__(self, - in_channel, - out_channel, - stride=1, - damping=0.03, - loss_scale=1, - frequency=278): - super(ResidualBlock, self).__init__() - - channel = out_channel // self.expansion - self.conv1 = _conv1x1(in_channel, channel, stride=1, damping=damping, loss_scale=loss_scale, frequency=frequency) - self.bn1 = _bn(channel) - - self.conv2 = _conv3x3(channel, channel, stride=stride, damping=damping, loss_scale=loss_scale, frequency=frequency) - self.bn2 = _bn(channel) - - self.conv3 = _conv1x1(channel, out_channel, stride=1, damping=damping, loss_scale=loss_scale, frequency=frequency) - self.bn3 = _bn_last(out_channel) - - self.relu = nn.ReLU() - - self.down_sample = False - - if stride != 1 or in_channel != out_channel: - self.down_sample = True - self.down_sample_layer = None - - if self.down_sample: - self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channel, out_channel, stride, - damping=damping, loss_scale=loss_scale, frequency=frequency), - _bn(out_channel)]) - self.add = P.TensorAdd() - - def construct(self, x): - identity = x - - out = self.conv1(x) - out = self.bn1(out) - out = self.relu(out) - - out = self.conv2(out) - out = self.bn2(out) - out = self.relu(out) - - out = self.conv3(out) - out = self.bn3(out) - - if self.down_sample: - identity = self.down_sample_layer(identity) - - out = self.add(out, identity) - out = self.relu(out) - - return out - - -class ResNet(nn.Cell): - """ - ResNet architecture. - - Args: - block (Cell): Block for network. - layer_nums (list): Numbers of block in different layers. - in_channels (list): Input channel in each layer. - out_channels (list): Output channel in each layer. - strides (list): Stride size in each layer. - num_classes (int): The number of classes that the training images are belonging to. - Returns: - Tensor, output tensor. - - Examples: - >>> ResNet(ResidualBlock, - >>> [3, 4, 6, 3], - >>> [64, 256, 512, 1024], - >>> [256, 512, 1024, 2048], - >>> [1, 2, 2, 2], - >>> 10) - """ - - def __init__(self, - block, - layer_nums, - in_channels, - out_channels, - strides, - num_classes, - damping, - loss_scale, - frequency): - super(ResNet, self).__init__() - - if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: - raise ValueError("the length of layer_num, in_channels, out_channels list must be 4!") - - self.conv1 = _conv7x7(3, 64, stride=2, damping=damping, loss_scale=loss_scale, frequency=frequency) - self.bn1 = _bn(64) - self.relu = P.ReLU() - self.maxpool = P.MaxPoolWithArgmax(padding="same", ksize=3, strides=2) - - self.layer1 = self._make_layer(block, - layer_nums[0], - in_channel=in_channels[0], - out_channel=out_channels[0], - stride=strides[0], - damping=damping, - loss_scale=loss_scale, - frequency=frequency) - self.layer2 = self._make_layer(block, - layer_nums[1], - in_channel=in_channels[1], - out_channel=out_channels[1], - stride=strides[1], - damping=damping, - loss_scale=loss_scale, - frequency=frequency) - self.layer3 = self._make_layer(block, - layer_nums[2], - in_channel=in_channels[2], - out_channel=out_channels[2], - stride=strides[2],damping=damping, - loss_scale=loss_scale, - frequency=frequency) - self.layer4 = self._make_layer(block, - layer_nums[3], - in_channel=in_channels[3], - out_channel=out_channels[3], - stride=strides[3], - damping=damping, - loss_scale=loss_scale, - frequency=frequency) - - self.mean = P.ReduceMean(keep_dims=True) - self.flatten = nn.Flatten() - self.end_point = _fc(out_channels[3], num_classes, damping=damping, loss_scale=loss_scale, frequency=frequency) - - def _make_layer(self, block, layer_num, in_channel, out_channel, stride, - damping, loss_scale, frequency): - """ - Make stage network of ResNet. - - Args: - block (Cell): Resnet block. - layer_num (int): Layer number. - in_channel (int): Input channel. - out_channel (int): Output channel. - stride (int): Stride size for the first convolutional layer. - - Returns: - SequentialCell, the output layer. - - Examples: - >>> _make_layer(ResidualBlock, 3, 128, 256, 2) - """ - layers = [] - - resnet_block = block(in_channel, out_channel, stride=stride, - damping=damping, loss_scale=loss_scale, frequency=frequency) - layers.append(resnet_block) - - for _ in range(1, layer_num): - resnet_block = block(out_channel, out_channel, stride=1, - damping=damping, loss_scale=loss_scale, frequency=frequency) - layers.append(resnet_block) - - return nn.SequentialCell(layers) - - def construct(self, x): - x = self.conv1(x) - x = self.bn1(x) - x = self.relu(x) - c1, argmax = self.maxpool(x) - - c2 = self.layer1(c1) - c3 = self.layer2(c2) - c4 = self.layer3(c3) - c5 = self.layer4(c4) - - out = self.mean(c5, (2, 3)) - out = self.flatten(out) - out = self.end_point(out) - - return out - - -def resnet50(class_num=10, damping=0.03, loss_scale=1, frequency=278): - """ - Get ResNet50 neural network. - - Args: - class_num (int): Class number. - - Returns: - Cell, cell instance of ResNet50 neural network. - - Examples: - >>> net = resnet50(10) - """ - return ResNet(ResidualBlock, - [3, 4, 6, 3], - [64, 256, 512, 1024], - [256, 512, 1024, 2048], - [1, 2, 2, 2], - class_num, - damping, - loss_scale, - frequency) diff --git a/tests/st/networks/thor_test/model/thor_layer.py b/tests/st/networks/thor_test/model/thor_layer.py index 70c207b7e2d..f91a7e2d169 100644 --- a/tests/st/networks/thor_test/model/thor_layer.py +++ b/tests/st/networks/thor_test/model/thor_layer.py @@ -1,3 +1,18 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + import numpy as np import mindspore as ms import mindspore.common.dtype as mstype From da62a44d85b85b5eea557cdf22bbe29403910ca4 Mon Sep 17 00:00:00 2001 From: z00478463 Date: Fri, 22 May 2020 09:22:11 +0800 Subject: [PATCH 03/11] switch positon of the codes for second_order --- .../config_imagenet.py | 1 + .../crossentropy.py | 16 +- .../cus_ops/batch_matmul_impl.py | 9 +- .../cus_ops/cholesky_trsm.py | 8 +- .../cus_ops/cus_batch_matmul.py | 33 +-- .../cus_ops/cus_cholesky_trsm.py | 19 +- .../cus_ops/cus_fused_abs_max1.py | 19 +- .../cus_ops/cus_img2col.py | 8 +- .../cus_ops/cus_matmul_cube.py | 23 +- .../cus_ops/cus_matmul_cube_dense_left.py | 17 +- .../cus_matmul_cube_fracz_right_mul.py | 16 +- .../cus_ops/cus_matrix_combine.py | 18 +- .../cus_ops/cus_transpose02314.py | 20 +- .../cus_ops/fused_abs_max1.py | 6 +- .../cus_ops/img2col_impl.py | 5 +- .../cus_ops/matmul_cube_dense_left.py | 24 +- .../matmul_cube_fracz_left_cast_impl.py | 17 +- .../matmul_cube_fracz_right_mul_impl.py | 23 +- .../cus_ops/matmul_cube_impl.py | 20 +- .../cus_ops/matrix_combine_impl.py | 8 +- .../cus_ops/transpose02314_impl.py | 5 +- .../dataset_imagenet.py | 11 +- .../lr_generator.py | 7 +- .../model/dataset_helper.py | 74 +++--- .../model/grad_reducer_thor.py | 12 +- .../model/model_thor.py | 184 +++++++------- .../model/resnet.py | 131 +++++----- .../model/thor.py | 60 +++-- .../model/thor_layer.py | 240 ++++++++++-------- .../run_distribute_train_new.sh | 0 .../resnet101_imagenet2012_THOR}/train.py | 61 ++--- 31 files changed, 546 insertions(+), 549 deletions(-) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/config_imagenet.py (99%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/crossentropy.py (92%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/batch_matmul_impl.py (97%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/cholesky_trsm.py (94%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/cus_batch_matmul.py (62%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/cus_cholesky_trsm.py (82%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/cus_fused_abs_max1.py (86%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/cus_img2col.py (90%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/cus_matmul_cube.py (86%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/cus_matmul_cube_dense_left.py (88%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/cus_matmul_cube_fracz_right_mul.py (86%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/cus_matrix_combine.py (88%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/cus_transpose02314.py (85%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/fused_abs_max1.py (93%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/img2col_impl.py (98%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/matmul_cube_dense_left.py (88%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/matmul_cube_fracz_left_cast_impl.py (95%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/matmul_cube_fracz_right_mul_impl.py (87%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/matmul_cube_impl.py (93%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/matrix_combine_impl.py (93%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/cus_ops/transpose02314_impl.py (97%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/dataset_imagenet.py (94%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/lr_generator.py (99%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/model/dataset_helper.py (97%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/model/grad_reducer_thor.py (98%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/model/model_thor.py (96%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/model/resnet.py (92%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/model/thor.py (94%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/model/thor_layer.py (80%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/run_distribute_train_new.sh (100%) rename {tests/st/networks/thor_test => example/resnet101_imagenet2012_THOR}/train.py (89%) diff --git a/tests/st/networks/thor_test/config_imagenet.py b/example/resnet101_imagenet2012_THOR/config_imagenet.py similarity index 99% rename from tests/st/networks/thor_test/config_imagenet.py rename to example/resnet101_imagenet2012_THOR/config_imagenet.py index 3158dfaddd4..6c664891f76 100644 --- a/tests/st/networks/thor_test/config_imagenet.py +++ b/example/resnet101_imagenet2012_THOR/config_imagenet.py @@ -16,6 +16,7 @@ network config setting, will be used in train.py and eval.py """ from easydict import EasyDict as ed + config = ed({ "class_num": 1000, "batch_size": 32, diff --git a/tests/st/networks/thor_test/crossentropy.py b/example/resnet101_imagenet2012_THOR/crossentropy.py similarity index 92% rename from tests/st/networks/thor_test/crossentropy.py rename to example/resnet101_imagenet2012_THOR/crossentropy.py index a689e018a2d..24ae022b58f 100644 --- a/tests/st/networks/thor_test/crossentropy.py +++ b/example/resnet101_imagenet2012_THOR/crossentropy.py @@ -13,24 +13,26 @@ # limitations under the License. # ============================================================================ -from mindspore.nn.loss.loss import _Loss -from mindspore.ops import operations as P -from mindspore.ops import functional as F +import mindspore.nn as nn from mindspore import Tensor from mindspore.common import dtype as mstype -import mindspore.nn as nn +from mindspore.nn.loss.loss import _Loss +from mindspore.ops import functional as F +from mindspore.ops import operations as P + class CrossEntropy(_Loss): def __init__(self, smooth_factor=0., num_classes=1000): super(CrossEntropy, self).__init__() self.onehot = P.OneHot() self.on_value = Tensor(1.0 - smooth_factor, mstype.float32) - self.off_value = Tensor(1.0 * smooth_factor / (num_classes -1), mstype.float32) - #self.cast = P.Cast() + self.off_value = Tensor(1.0 * smooth_factor / (num_classes - 1), mstype.float32) + # self.cast = P.Cast() self.ce = nn.SoftmaxCrossEntropyWithLogits() self.mean = P.ReduceMean(False) + def construct(self, logit, label): - #one_hot_label = self.onehot(self.cast(label, mstype.int32), + # one_hot_label = self.onehot(self.cast(label, mstype.int32), # F.shape(logit)[1], self.on_value, self.off_value)、 one_hot_label = self.onehot(label, F.shape(logit)[1], self.on_value, self.off_value) loss = self.ce(logit, one_hot_label) diff --git a/tests/st/networks/thor_test/cus_ops/batch_matmul_impl.py b/example/resnet101_imagenet2012_THOR/cus_ops/batch_matmul_impl.py similarity index 97% rename from tests/st/networks/thor_test/cus_ops/batch_matmul_impl.py rename to example/resnet101_imagenet2012_THOR/cus_ops/batch_matmul_impl.py index 78e46c846b9..b75f3b49cab 100644 --- a/tests/st/networks/thor_test/cus_ops/batch_matmul_impl.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/batch_matmul_impl.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -from te import tik -from topi.cce import util from mindspore.ops.op_info_register import op_info_register + @op_info_register("""{ "op_name": "CusBatchMatMul", "imply_type": "TBE", @@ -71,11 +70,5 @@ from mindspore.ops.op_info_register import op_info_register } ] }""") - - - - - def CusBatchMatMul(input_x1, input_x2, output, transpose_a=False, transpose_b=True, kernel_name="batchmatmul"): - return diff --git a/tests/st/networks/thor_test/cus_ops/cholesky_trsm.py b/example/resnet101_imagenet2012_THOR/cus_ops/cholesky_trsm.py similarity index 94% rename from tests/st/networks/thor_test/cus_ops/cholesky_trsm.py rename to example/resnet101_imagenet2012_THOR/cus_ops/cholesky_trsm.py index 3e4c86c2971..f18d513f17b 100644 --- a/tests/st/networks/thor_test/cus_ops/cholesky_trsm.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/cholesky_trsm.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -from te import tik -from topi.cce import util from mindspore.ops.op_info_register import op_info_register + + @op_info_register("""{ "op_name": "CusCholeskyTrsm", "imply_type": "TBE", @@ -58,7 +58,5 @@ from mindspore.ops.op_info_register import op_info_register } ] }""") - - -def CusCholeskyTrsm(input_x,output, kernel_name): +def CusCholeskyTrsm(input_x, output, kernel_name): return diff --git a/tests/st/networks/thor_test/cus_ops/cus_batch_matmul.py b/example/resnet101_imagenet2012_THOR/cus_ops/cus_batch_matmul.py similarity index 62% rename from tests/st/networks/thor_test/cus_ops/cus_batch_matmul.py rename to example/resnet101_imagenet2012_THOR/cus_ops/cus_batch_matmul.py index c58a15c88e1..2379181ac38 100644 --- a/tests/st/networks/thor_test/cus_ops/cus_batch_matmul.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/cus_batch_matmul.py @@ -12,42 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -import numpy as np from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore import Tensor -import mindspore as ms from mindspore.ops.composite import multitype_ops as C + + # y = x^2 class CusBatchMatMul(PrimitiveWithInfer): """CusMatMulCube definition""" + @prim_attr_register def __init__(self): """init CusMatMulCube""" self.init_prim_io_names(inputs=['x1', 'x2'], outputs=['y']) - # self.transpose_a = transpose_a - # self.transpose_b = transpose_b - from .batch_matmul_impl import CusBatchMatMul - + def get_bprop(self): def bprop(x1, x2, out, dout): - return (C.zeros_like(x1),C.zeros_like(x2)) + return (C.zeros_like(x1), C.zeros_like(x2)) + return bprop - + def infer_shape(self, data1_shape, data2_shape): - #shape = [1, data1_shape[1], data2_shape[2], 16, 16] - #return shape - # if self.transpose_a == True: - # k1, m = data1_shape - # else: - # m, k1 = data1_shape - # if self.transpose_b == True: - # n, k2 = data2_shape - # else: - # k2, n = data2_shape - # assert k1==k2 - # shape = [m, n] return data1_shape - + def infer_dtype(self, data1_dtype, data2_dtype): - return data1_dtype - # return ms.common.dtype.tensor_type(getattr(ms, "float32")) + return data1_dtype \ No newline at end of file diff --git a/tests/st/networks/thor_test/cus_ops/cus_cholesky_trsm.py b/example/resnet101_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py similarity index 82% rename from tests/st/networks/thor_test/cus_ops/cus_cholesky_trsm.py rename to example/resnet101_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py index 02ddf981978..4328f0f7479 100644 --- a/tests/st/networks/thor_test/cus_ops/cus_cholesky_trsm.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py @@ -12,24 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -import numpy as np from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore import Tensor - + + class CusCholeskyTrsm(PrimitiveWithInfer): """CusCholeskyTrsm definition""" + @prim_attr_register def __init__(self): """init CusCholeskyTrsm""" self.init_prim_io_names(inputs=['x1'], outputs=['y']) - from .cholesky_trsm import CusCholeskyTrsm - + def infer_shape(self, data1_shape): - m,n = data1_shape + m, n = data1_shape if m >= 128: - return [m//128,128,128] + return [m // 128, 128, 128] else: - return [1,64,64] - + return [1, 64, 64] + def infer_dtype(self, data1_dtype): - return data1_dtype + return data1_dtype \ No newline at end of file diff --git a/tests/st/networks/thor_test/cus_ops/cus_fused_abs_max1.py b/example/resnet101_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py similarity index 86% rename from tests/st/networks/thor_test/cus_ops/cus_fused_abs_max1.py rename to example/resnet101_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py index 810966795ed..1bfebedde8f 100644 --- a/tests/st/networks/thor_test/cus_ops/cus_fused_abs_max1.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py @@ -12,31 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -import numpy as np from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore import Tensor from mindspore.ops.composite import multitype_ops as C - + + class CusFusedAbsMax1(PrimitiveWithInfer): """CusCholeskyTrsm definition""" + @prim_attr_register - def __init__(self, origin_shape = [-1,-1]): + def __init__(self, origin_shape=[-1, -1]): """init CusCholeskyTrsm""" self.init_prim_io_names(inputs=['x1'], outputs=['y']) - from .fused_abs_max1 import CusFusedAbsMax1 self.origin_shape = origin_shape - + def get_bprop(self): def bprop(x, out, dout): return (C.zeros_like(x),) + return bprop - + def infer_shape(self, data1_shape): if len(data1_shape) == 2: - return [1,] + return [1, ] else: return [32, 64] - # return [128,128] - + def infer_dtype(self, data1_dtype): return data1_dtype diff --git a/tests/st/networks/thor_test/cus_ops/cus_img2col.py b/example/resnet101_imagenet2012_THOR/cus_ops/cus_img2col.py similarity index 90% rename from tests/st/networks/thor_test/cus_ops/cus_img2col.py rename to example/resnet101_imagenet2012_THOR/cus_ops/cus_img2col.py index 5a1249938ff..ce363400ba3 100644 --- a/tests/st/networks/thor_test/cus_ops/cus_img2col.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/cus_img2col.py @@ -13,26 +13,26 @@ # limitations under the License. # ============================================================================ -import numpy as np from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore import Tensor from mindspore.ops.composite import multitype_ops as C + class CusImg2Col(PrimitiveWithInfer): """CusImg2Col definition""" + @prim_attr_register - def __init__(self, ksizes, strides, dilates = (1, 1, 1, 1), mode="NC1HWC0"): + def __init__(self, ksizes, strides, dilates=(1, 1, 1, 1), mode="NC1HWC0"): """init CusImg2Col""" self.init_prim_io_names(inputs=['x1'], outputs=['y']) self.ksizes = ksizes self.strides = strides self.dilates = dilates self.mode = mode - from .img2col_impl import CusImg2Col def get_bprop(self): def bprop(x, out, dout): return (C.zeros_like(x),) + return bprop def infer_shape(self, data1_shape): diff --git a/tests/st/networks/thor_test/cus_ops/cus_matmul_cube.py b/example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube.py similarity index 86% rename from tests/st/networks/thor_test/cus_ops/cus_matmul_cube.py rename to example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube.py index 1b9d295b1c7..71933737244 100644 --- a/tests/st/networks/thor_test/cus_ops/cus_matmul_cube.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube.py @@ -12,30 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -import numpy as np -from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore import Tensor import mindspore as ms +from mindspore.ops import prim_attr_register, PrimitiveWithInfer from mindspore.ops.composite import multitype_ops as C + + # y = x^2 class CusMatMulCube(PrimitiveWithInfer): """CusMatMulCube definition""" + @prim_attr_register def __init__(self, transpose_a=False, transpose_b=False): """init CusMatMulCube""" self.init_prim_io_names(inputs=['x1', 'x2'], outputs=['y']) self.transpose_a = transpose_a self.transpose_b = transpose_b - from .matmul_cube_impl import CusMatMulCube - + def get_bprop(self): def bprop(x1, x2, out, dout): - return (C.zeros_like(x1),C.zeros_like(x2)) + return (C.zeros_like(x1), C.zeros_like(x2)) + return bprop - + def infer_shape(self, data1_shape, data2_shape): - #shape = [1, data1_shape[1], data2_shape[2], 16, 16] - #return shape + # shape = [1, data1_shape[1], data2_shape[2], 16, 16] + # return shape if self.transpose_a == True: k1, m = data1_shape else: @@ -44,9 +45,9 @@ class CusMatMulCube(PrimitiveWithInfer): n, k2 = data2_shape else: k2, n = data2_shape - assert k1==k2 + assert k1 == k2 shape = [m, n] return shape - + def infer_dtype(self, data1_dtype, data2_dtype): return ms.common.dtype.tensor_type(getattr(ms, "float32")) diff --git a/tests/st/networks/thor_test/cus_ops/cus_matmul_cube_dense_left.py b/example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube_dense_left.py similarity index 88% rename from tests/st/networks/thor_test/cus_ops/cus_matmul_cube_dense_left.py rename to example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube_dense_left.py index 71b77910ecc..603d8487cef 100644 --- a/tests/st/networks/thor_test/cus_ops/cus_matmul_cube_dense_left.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube_dense_left.py @@ -12,27 +12,28 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -import numpy as np -from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore import Tensor import mindspore as ms +from mindspore.ops import prim_attr_register, PrimitiveWithInfer from mindspore.ops.composite import multitype_ops as C + + # y = x^2 class CusMatMulCubeDenseLeft(PrimitiveWithInfer): """CusMatMulCube definition""" + @prim_attr_register def __init__(self): """init CusMatMulCube""" self.init_prim_io_names(inputs=['x1', 'x2'], outputs=['y']) - from .matmul_cube_dense_left import CusMatMulCubeDenseLeft - + def get_bprop(self): def bprop(x1, x2, out, dout): - return (C.zeros_like(x1),C.zeros_like(x2)) + return (C.zeros_like(x1), C.zeros_like(x2)) + return bprop - + def infer_shape(self, data1_shape, data2_shape): return data2_shape - + def infer_dtype(self, data1_dtype, data2_dtype): return ms.common.dtype.tensor_type(getattr(ms, "float16")) diff --git a/tests/st/networks/thor_test/cus_ops/cus_matmul_cube_fracz_right_mul.py b/example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube_fracz_right_mul.py similarity index 86% rename from tests/st/networks/thor_test/cus_ops/cus_matmul_cube_fracz_right_mul.py rename to example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube_fracz_right_mul.py index 20b278e1fc2..f9b68a76551 100644 --- a/tests/st/networks/thor_test/cus_ops/cus_matmul_cube_fracz_right_mul.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube_fracz_right_mul.py @@ -12,27 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -import numpy as np -from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore import Tensor import mindspore as ms +from mindspore.ops import prim_attr_register, PrimitiveWithInfer from mindspore.ops.composite import multitype_ops as C -# y = x^2 + + class CusMatMulCubeFraczRightMul(PrimitiveWithInfer): """CusMatMulCubeFraczRightMul definition""" + @prim_attr_register def __init__(self): """init CusMatMulCubeFraczRightMul""" self.init_prim_io_names(inputs=['x1', 'x2', 'x3'], outputs=['y']) - from .matmul_cube_fracz_right_mul_impl import CusMatMulCubeFraczRightMul def get_bprop(self): def bprop(x1, x2, x3, out, dout): - return (C.zeros_like(x1),C.zeros_like(x2),C.zeros_like(x3)) + return (C.zeros_like(x1), C.zeros_like(x2), C.zeros_like(x3)) + return bprop - + def infer_shape(self, data1_shape, data2_shape, data3_shape): return data1_shape - + def infer_dtype(self, data1_dtype, data2_dtype, data3_dtype): return ms.common.dtype.tensor_type(getattr(ms, "float32")) diff --git a/tests/st/networks/thor_test/cus_ops/cus_matrix_combine.py b/example/resnet101_imagenet2012_THOR/cus_ops/cus_matrix_combine.py similarity index 88% rename from tests/st/networks/thor_test/cus_ops/cus_matrix_combine.py rename to example/resnet101_imagenet2012_THOR/cus_ops/cus_matrix_combine.py index 66230e0a176..98add9f542d 100644 --- a/tests/st/networks/thor_test/cus_ops/cus_matrix_combine.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/cus_matrix_combine.py @@ -12,29 +12,29 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -import numpy as np from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore import Tensor -import mindspore as ms from mindspore.ops.composite import multitype_ops as C -# y = x^2 + + class CusMatrixCombine(PrimitiveWithInfer): """CusMatMulCube definition""" + @prim_attr_register def __init__(self): """init CusMatMulCube""" self.init_prim_io_names(inputs=['x'], outputs=['y']) - from .matrix_combine_impl import CusMatrixCombine + def get_bprop(self): def bprop(x, out, dout): return (C.zeros_like(x),) + return bprop - + def infer_shape(self, data_shape): a, b, c = data_shape - shape = [a*b, a*c] - + shape = [a * b, a * c] + return shape - + def infer_dtype(self, data_dtype): return data_dtype diff --git a/tests/st/networks/thor_test/cus_ops/cus_transpose02314.py b/example/resnet101_imagenet2012_THOR/cus_ops/cus_transpose02314.py similarity index 85% rename from tests/st/networks/thor_test/cus_ops/cus_transpose02314.py rename to example/resnet101_imagenet2012_THOR/cus_ops/cus_transpose02314.py index 90051022f8e..944f97a74e2 100644 --- a/tests/st/networks/thor_test/cus_ops/cus_transpose02314.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/cus_transpose02314.py @@ -12,35 +12,33 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - - -import numpy as np + + from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore import Tensor from mindspore.ops.composite import multitype_ops as C - + + class CusTranspose02314(PrimitiveWithInfer): """CusTranspose02314 definition""" + @prim_attr_register def __init__(self): """init CusTranspose02314""" self.init_prim_io_names(inputs=['x1'], outputs=['y']) - from .transpose02314_impl import CusTranspose02314 - + def get_bprop(self): def bprop(x, out, dout): return (C.zeros_like(x),) + return bprop - + def infer_shape(self, data1_shape): assert len(data1_shape) == 4 n, c, h, w = data1_shape c0 = 16 c1 = c // 16 shape = (n * h * w, c1 * c0) - # axis_0, axis_1, axis_2, axis_3, axis_4 = data1_shape - # shape = (axis_0, axis_2, axis_3, axis_1, axis_4) return shape - + def infer_dtype(self, data1_dtype): return data1_dtype diff --git a/tests/st/networks/thor_test/cus_ops/fused_abs_max1.py b/example/resnet101_imagenet2012_THOR/cus_ops/fused_abs_max1.py similarity index 93% rename from tests/st/networks/thor_test/cus_ops/fused_abs_max1.py rename to example/resnet101_imagenet2012_THOR/cus_ops/fused_abs_max1.py index d2ce2d43305..146533131ea 100644 --- a/tests/st/networks/thor_test/cus_ops/fused_abs_max1.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/fused_abs_max1.py @@ -13,9 +13,9 @@ # limitations under the License. # ============================================================================ -from te import tik -from topi.cce import util from mindspore.ops.op_info_register import op_info_register + + @op_info_register("""{ "op_name": "CusFusedAbsMax1", "imply_type": "TBE", @@ -64,5 +64,5 @@ from mindspore.ops.op_info_register import op_info_register } ] }""") -def CusFusedAbsMax1(input_x, output, origin_shape = None, kernel_name="fused_abs_max1"): +def CusFusedAbsMax1(input_x, output, origin_shape=None, kernel_name="fused_abs_max1"): return diff --git a/tests/st/networks/thor_test/cus_ops/img2col_impl.py b/example/resnet101_imagenet2012_THOR/cus_ops/img2col_impl.py similarity index 98% rename from tests/st/networks/thor_test/cus_ops/img2col_impl.py rename to example/resnet101_imagenet2012_THOR/cus_ops/img2col_impl.py index c39355cedfa..67081823e80 100644 --- a/tests/st/networks/thor_test/cus_ops/img2col_impl.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/img2col_impl.py @@ -13,9 +13,9 @@ # limitations under the License. # ============================================================================ -from te import tik -from topi.cce import util from mindspore.ops.op_info_register import op_info_register + + @op_info_register("""{ "op_name": "CusImg2ColNC1HWC0", "imply_type": "TBE", @@ -82,6 +82,5 @@ from mindspore.ops.op_info_register import op_info_register } ] }""") - def CusImg2ColNC1HWC0(input_x, output, ksizes, strides, dilates, padding, kernel_name="img2col"): return diff --git a/tests/st/networks/thor_test/cus_ops/matmul_cube_dense_left.py b/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_dense_left.py similarity index 88% rename from tests/st/networks/thor_test/cus_ops/matmul_cube_dense_left.py rename to example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_dense_left.py index eaa7f9b1ec7..7c4b8c47f3c 100644 --- a/tests/st/networks/thor_test/cus_ops/matmul_cube_dense_left.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_dense_left.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding:utf-8 -*- """ -copyright 2019 Huawei Technologies Co., Ltd +copyright 2020 Huawei Technologies Co., Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,22 +18,15 @@ limitations under the License. matmul """ from __future__ import absolute_import - -import te.lang.cce -import te.platform.cce_params as cce -from te.platform.fusion_manager import fusion_manager -from te import tvm -from topi import generic -from topi.cce import util - -from impl.matmul_vector import matmul_vector_cce - -from te import tik + from mindspore.ops.op_info_register import op_info_register +from topi.cce import util + # General limitation of the size for input shape: 2**31 SHAPE_SIZE_LIMIT = 2147483648 NoneType = type(None) - + + @op_info_register("""{ "op_name": "CusMatMulCubeDenseLeft", "imply_type": "TBE", @@ -102,8 +95,7 @@ NoneType = type(None) } ] }""") - @util.check_input_type(dict, dict, (dict, NoneType), dict, bool, bool, str) -def CusMatMulCubeDenseLeft(input_x1, input_x2, bias=None, output_y={}, trans_a=False, trans_b=False, kernel_name="matmulcube"): +def CusMatMulCubeDenseLeft(input_x1, input_x2, bias=None, output_y={}, trans_a=False, trans_b=False, + kernel_name="matmulcube"): return - diff --git a/tests/st/networks/thor_test/cus_ops/matmul_cube_fracz_left_cast_impl.py b/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_fracz_left_cast_impl.py similarity index 95% rename from tests/st/networks/thor_test/cus_ops/matmul_cube_fracz_left_cast_impl.py rename to example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_fracz_left_cast_impl.py index e0126a02dcc..af0f6dac706 100644 --- a/tests/st/networks/thor_test/cus_ops/matmul_cube_fracz_left_cast_impl.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_fracz_left_cast_impl.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding:utf-8 -*- """ -copyright 2019 Huawei Technologies Co., Ltd +copyright 2020 Huawei Technologies Co., Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,19 +18,15 @@ limitations under the License. matmul """ from __future__ import absolute_import - -import te.platform.cce_params as cce -from te import tvm -from topi.cce import util - -from te import tik + from mindspore.ops.op_info_register import op_info_register - +from topi.cce import util + # General limitation of the size for input shape: 2**31 SHAPE_SIZE_LIMIT = 2147483648 NoneType = type(None) - - + + @op_info_register("""{ "op_name": "CusMatMulCubeFraczLeftCast", "imply_type": "TBE", @@ -99,7 +95,6 @@ NoneType = type(None) } ] }""") - # pylint: disable=locally-disabled,too-many-arguments, too-many-locals, too-many-statements @util.check_input_type(dict, dict, (dict, NoneType), dict, bool, bool, str) def CusMatMulCubeFraczLeftCast(input_x1, input_x2, bias=None, output_y={}, trans_a=False, trans_b=False, diff --git a/tests/st/networks/thor_test/cus_ops/matmul_cube_fracz_right_mul_impl.py b/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py similarity index 87% rename from tests/st/networks/thor_test/cus_ops/matmul_cube_fracz_right_mul_impl.py rename to example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py index c3589bb4d89..a6afa80b24d 100644 --- a/tests/st/networks/thor_test/cus_ops/matmul_cube_fracz_right_mul_impl.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding:utf-8 -*- """ -copyright 2019 Huawei Technologies Co., Ltd +copyright 2020 Huawei Technologies Co., Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,21 +18,14 @@ limitations under the License. matmul """ from __future__ import absolute_import - -import te.lang.cce -import te.platform.cce_params as cce -from te.platform.fusion_manager import fusion_manager -from te import tvm -from topi import generic -from topi.cce import util -from te import tik -from impl.matmul_vector import matmul_vector_cce + from mindspore.ops.op_info_register import op_info_register + # General limitation of the size for input shape: 2**31 SHAPE_SIZE_LIMIT = 2147483648 NoneType = type(None) - - + + @op_info_register("""{ "op_name": "CusMatMulCubeFraczRightMul", "imply_type": "TBE", @@ -114,8 +107,6 @@ NoneType = type(None) } ] }""") - -def CusMatMulCubeFraczRightMul(input_x1, input_x2, input_x3, bias=None, output_y={}, trans_a=False, trans_b=False, kernel_name="matmulcube"): +def CusMatMulCubeFraczRightMul(input_x1, input_x2, input_x3, bias=None, output_y={}, trans_a=False, trans_b=False, + kernel_name="matmulcube"): return - - diff --git a/tests/st/networks/thor_test/cus_ops/matmul_cube_impl.py b/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_impl.py similarity index 93% rename from tests/st/networks/thor_test/cus_ops/matmul_cube_impl.py rename to example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_impl.py index 4cb12751d71..0b6d91ef79e 100644 --- a/tests/st/networks/thor_test/cus_ops/matmul_cube_impl.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_impl.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding:utf-8 -*- """ -copyright 2019 Huawei Technologies Co., Ltd +copyright 2020 Huawei Technologies Co., Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -18,20 +18,15 @@ limitations under the License. matmul """ from __future__ import absolute_import - -import te.lang.cce -import te.platform.cce_params as cce -from te import tvm -from topi import generic -from topi.cce import util - -from impl.matmul_vector import matmul_vector_cce + from mindspore.ops.op_info_register import op_info_register - +from topi.cce import util + # General limitation of the size for input shape: 2**31 SHAPE_SIZE_LIMIT = 2147483648 NoneType = type(None) - + + @op_info_register("""{ "op_name": "CusMatMulCube", "imply_type": "TBE", @@ -112,8 +107,7 @@ NoneType = type(None) } ] }""") - # pylint: disable=locally-disabled,too-many-arguments, too-many-locals, too-many-statements @util.check_input_type(dict, dict, (dict, NoneType), dict, bool, bool, str) def CusMatMulCube(input_x1, input_x2, bias=None, output_y={}, trans_a=False, trans_b=False, kernel_name="matmulcube"): - return + return diff --git a/tests/st/networks/thor_test/cus_ops/matrix_combine_impl.py b/example/resnet101_imagenet2012_THOR/cus_ops/matrix_combine_impl.py similarity index 93% rename from tests/st/networks/thor_test/cus_ops/matrix_combine_impl.py rename to example/resnet101_imagenet2012_THOR/cus_ops/matrix_combine_impl.py index 791892a137e..f300f0b1a96 100644 --- a/tests/st/networks/thor_test/cus_ops/matrix_combine_impl.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/matrix_combine_impl.py @@ -13,9 +13,9 @@ # limitations under the License. # ============================================================================ -from te import tik -from topi.cce import util from mindspore.ops.op_info_register import op_info_register + + @op_info_register("""{ "op_name": "CusMatrixCombine", "imply_type": "TBE", @@ -58,7 +58,5 @@ from mindspore.ops.op_info_register import op_info_register } ] }""") - - -def CusMatrixCombine(input_x, output,kernel_name="matrix_combine"): +def CusMatrixCombine(input_x, output, kernel_name="matrix_combine"): return diff --git a/tests/st/networks/thor_test/cus_ops/transpose02314_impl.py b/example/resnet101_imagenet2012_THOR/cus_ops/transpose02314_impl.py similarity index 97% rename from tests/st/networks/thor_test/cus_ops/transpose02314_impl.py rename to example/resnet101_imagenet2012_THOR/cus_ops/transpose02314_impl.py index 627eefe4e48..fba24f082c8 100644 --- a/tests/st/networks/thor_test/cus_ops/transpose02314_impl.py +++ b/example/resnet101_imagenet2012_THOR/cus_ops/transpose02314_impl.py @@ -13,9 +13,9 @@ # limitations under the License. # ============================================================================ -from te import tik -from topi.cce import util from mindspore.ops.op_info_register import op_info_register + + @op_info_register("""{ "op_name": "CusTranspose02314", "imply_type": "TBE", @@ -58,6 +58,5 @@ from mindspore.ops.op_info_register import op_info_register } ] }""") - def CusTranspose02314(input_x, output, kernel_name="transpose021354"): return diff --git a/tests/st/networks/thor_test/dataset_imagenet.py b/example/resnet101_imagenet2012_THOR/dataset_imagenet.py similarity index 94% rename from tests/st/networks/thor_test/dataset_imagenet.py rename to example/resnet101_imagenet2012_THOR/dataset_imagenet.py index 3febbd76ccb..296b675136d 100644 --- a/tests/st/networks/thor_test/dataset_imagenet.py +++ b/example/resnet101_imagenet2012_THOR/dataset_imagenet.py @@ -16,11 +16,12 @@ create train or eval dataset. """ import os + import mindspore.common.dtype as mstype import mindspore.dataset.engine as de -import mindspore.dataset.transforms.vision.c_transforms as V_C import mindspore.dataset.transforms.c_transforms as C2 -from config_imagenet import config +import mindspore.dataset.transforms.vision.c_transforms as V_C + def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): """ @@ -41,7 +42,7 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False) else: ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True, - num_shards=device_num, shard_id=rank_id) + num_shards=device_num, shard_id=rank_id) image_size = 224 mean = [0.485 * 255, 0.456 * 255, 0.406 * 255] @@ -61,9 +62,9 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32): V_C.Normalize(mean=mean, std=std), V_C.HWC2CHW() ] - #type_cast_op = C2.TypeCast(mstype.float16) + # type_cast_op = C2.TypeCast(mstype.float16) type_cast_op = C2.TypeCast(mstype.int32) - + ds = ds.map(input_columns="image", operations=transform_img, num_parallel_workers=8) ds = ds.map(input_columns="label", operations=type_cast_op, num_parallel_workers=8) diff --git a/tests/st/networks/thor_test/lr_generator.py b/example/resnet101_imagenet2012_THOR/lr_generator.py similarity index 99% rename from tests/st/networks/thor_test/lr_generator.py rename to example/resnet101_imagenet2012_THOR/lr_generator.py index 3536b0f89eb..ca290fd1585 100644 --- a/tests/st/networks/thor_test/lr_generator.py +++ b/example/resnet101_imagenet2012_THOR/lr_generator.py @@ -13,14 +13,17 @@ # limitations under the License. # ============================================================================ """learning rate generator""" -import numpy as np import math +import numpy as np + + def linear_warmup_lr(current_step, warmup_steps, base_lr, init_lr): lr_inc = (float(base_lr) - float(init_lr)) / float(warmup_steps) lr = float(init_lr) + lr_inc * current_step return lr + def cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0, num_periods=0.5): base_lr = lr warmup_init_lr = 0 @@ -39,6 +42,7 @@ def cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, et lr_each_step.append(lr) return np.array(lr_each_step).astype(np.float32) + def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0, num_periods=0.5): base_lr = lr warmup_init_lr = 0 @@ -57,6 +61,7 @@ def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_ lr_each_step.append(lr) return np.array(lr_each_step).astype(np.float32) + def get_lr(global_step, lr_init, lr_end, lr_max, warmup_epochs, total_epochs, steps_per_epoch, lr_decay_mode): """ generate learning rate array diff --git a/tests/st/networks/thor_test/model/dataset_helper.py b/example/resnet101_imagenet2012_THOR/model/dataset_helper.py similarity index 97% rename from tests/st/networks/thor_test/model/dataset_helper.py rename to example/resnet101_imagenet2012_THOR/model/dataset_helper.py index 36e4ce107a8..83e7f4e1a2c 100644 --- a/tests/st/networks/thor_test/model/dataset_helper.py +++ b/example/resnet101_imagenet2012_THOR/model/dataset_helper.py @@ -13,15 +13,15 @@ # limitations under the License. # ============================================================================ """Dataset help for minddata dataset""" -from mindspore._checkparam import check_bool from mindspore import context -from mindspore.train.parallel_utils import ParallelMode -from mindspore.train._utils import _exec_datagraph, _get_types_and_shapes, _to_tensor, \ - _construct_tensor_list, _to_full_shapes, _to_full_tensor +from mindspore._checkparam import check_bool from mindspore.nn.wrap import GetNextSingleOp from mindspore.parallel._utils import _get_device_num, _get_global_rank, _get_parallel_mode - - +from mindspore.train._utils import _exec_datagraph, _get_types_and_shapes, _to_tensor, \ + _construct_tensor_list, _to_full_shapes, _to_full_tensor +from mindspore.train.parallel_utils import ParallelMode + + class DatasetHelper: """ Help function to use the Minddata dataset. @@ -41,9 +41,10 @@ class DatasetHelper: >>> for inputs in dataset_helper: >>> outputs = network(*inputs) """ + def __init__(self, dataset, first_order_iter=0, dataset_sink_mode=True): check_bool(dataset_sink_mode) - + iterclass = _DatasetIterGE if not dataset_sink_mode: iterclass = _DatasetIterFeed @@ -52,24 +53,25 @@ class DatasetHelper: iterclass = _DatasetIterMSLoopSink else: iterclass = _DatasetIterMS - + self.iter = iterclass(dataset, first_order_iter) - + def __iter__(self): return self.iter.__iter__() - + # A temp solution for loop sink. Delete later def types_shapes(self): """Get the types and shapes from dataset on current config.""" return self.iter.types_shapes() - + def loop_size(self): """Get loop_size for every iteration.""" return self.iter.loop_size - - + + class _DatasetIter: """Base iter for dataset help""" + def __init__(self, dataset): self.loop_size = 1 if not hasattr(dataset, '__ME_INITED__'): @@ -78,7 +80,7 @@ class _DatasetIter: else: self.loop_size = dataset.__loop_size__ dataset.__ME_INITED__ = _exec_datagraph(dataset, self.loop_size).queue_name - + self.ind = 0 self.dataset = dataset dataset_types, dataset_shapes = _get_types_and_shapes(dataset) @@ -89,53 +91,57 @@ class _DatasetIter: if _get_parallel_mode() in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): device_num = _get_device_num() self.dataset_shapes = _to_full_shapes(dataset_shapes, device_num) - + def __iter__(self): self.ind = 0 return self - + def __next__(self): if self.ind >= self.loop_count: raise StopIteration() self.ind += 1 return self.op() - + def types_shapes(self): return self.dataset_types, self.dataset_shapes - + def get_loop_count(self, dataset): loop_count = 1 if hasattr(dataset, '__loop_size__'): loop_size = dataset.__loop_size__ - loop_count = int(dataset.get_dataset_size()/loop_size) + loop_count = int(dataset.get_dataset_size() / loop_size) return loop_count - - + + class _DatasetIterMSLoopSink(_DatasetIter): """Iter for context (enable_loop_sink=True)""" + def __init__(self, dataset, first_order_iter): super(_DatasetIterMSLoopSink, self).__init__(dataset) # self.loop_count = self.get_loop_count(dataset) loop_size = dataset.__loop_size__ + first_order_iter - self.loop_count = int(dataset.get_dataset_size()/loop_size) * 2 - + self.loop_count = int(dataset.get_dataset_size() / loop_size) * 2 + def op(): return tuple() + self.op = op - - + + class _DatasetIterMS(_DatasetIter): """Iter for context (enable_loop_sink=False)""" + def __init__(self, dataset, first_order_order): super(_DatasetIterMS, self).__init__(dataset) self.loop_count = dataset.get_dataset_size() self.loop_size = 1 queue_name = dataset.__ME_INITED__ self.op = GetNextSingleOp(self.dataset_types, self.dataset_shapes, queue_name) - + class _DatasetIterGE(_DatasetIter): """Iter for ge""" + def __init__(self, dataset): super(_DatasetIterGE, self).__init__(dataset) self.loop_count = self.get_loop_count(dataset) @@ -145,14 +151,16 @@ class _DatasetIterGE(_DatasetIter): if self.need_to_full: batch_expand_num = _get_device_num() tensor_list_run = _construct_tensor_list(self.dataset_types, self.dataset_shapes, batch_expand_num) - + def op(): return tensor_list_run + self.op = op - - + + class _DatasetIterFeed: """Iter for feed data""" + def __init__(self, dataset, first_order_order): self.dataset = dataset self.device_num = _get_device_num() @@ -161,18 +169,18 @@ class _DatasetIterFeed: self.repeat_ind = 0 self.loop_count = dataset.get_dataset_size() self.ind = 0 - + parallel_mode = context.get_auto_parallel_context("parallel_mode") self.need_to_full = parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL) - + def __iter__(self): if self.repeat_ind % self.repeat_count == 0: self.iter = self.dataset.__iter__() - + self.repeat_ind += 1 self.ind = 0 return self - + def __next__(self): if self.ind >= self.loop_count: raise StopIteration() diff --git a/tests/st/networks/thor_test/model/grad_reducer_thor.py b/example/resnet101_imagenet2012_THOR/model/grad_reducer_thor.py similarity index 98% rename from tests/st/networks/thor_test/model/grad_reducer_thor.py rename to example/resnet101_imagenet2012_THOR/model/grad_reducer_thor.py index 0540ebb287e..4714aad17f9 100644 --- a/tests/st/networks/thor_test/model/grad_reducer_thor.py +++ b/example/resnet101_imagenet2012_THOR/model/grad_reducer_thor.py @@ -12,28 +12,30 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -from mindspore.nn.cell import Cell +import mindspore.common.dtype as mstype from mindspore.communication.management import GlobalComm, get_group_size +from mindspore.nn.cell import Cell from mindspore.ops import functional as F, composite as C, operations as P from mindspore.ops.operations.comm_ops import AllReduce, ReduceOp -import mindspore.common.dtype as mstype -from mindspore.communication import create_group reduce_opt = C.MultitypeFuncGraph("reduce_opt") _all_reduce_A = AllReduce() + def _init_optimizer_allreduce(group): global _all_reduce_A _all_reduce_A = AllReduce(ReduceOp.SUM, GlobalComm.WORLD_COMM_GROUP) _all_reduce_A.add_prim_attr('fusion', group) + @reduce_opt.register("Function", "Number", "Tensor") def _tensors_allreduce_mean(mul, degree, grad): degree = F.scalar_cast(degree, F.dtype(grad)) grad = _all_reduce_A(grad) cast_op = P.Cast() - return mul(grad, cast_op(F.scalar_to_array(1.0/degree), F.dtype(grad))) + return mul(grad, cast_op(F.scalar_to_array(1.0 / degree), F.dtype(grad))) + @reduce_opt.register("Bool", "Tensor") def _tensors_allreduce(allreduce_filter, grad): @@ -41,8 +43,10 @@ def _tensors_allreduce(allreduce_filter, grad): return _all_reduce_A(grad) return grad + _get_datatype = C.MultitypeFuncGraph("_get_datatype") + @_get_datatype.register("Tensor") def _tensors_get_datatype(grad): """ diff --git a/tests/st/networks/thor_test/model/model_thor.py b/example/resnet101_imagenet2012_THOR/model/model_thor.py similarity index 96% rename from tests/st/networks/thor_test/model/model_thor.py rename to example/resnet101_imagenet2012_THOR/model/model_thor.py index 623a799e240..1527e28812d 100644 --- a/tests/st/networks/thor_test/model/model_thor.py +++ b/example/resnet101_imagenet2012_THOR/model/model_thor.py @@ -13,29 +13,26 @@ # limitations under the License. # ============================================================================ """Model.""" -import numpy as np import mindspore.nn as nn -from mindspore import log as logger -from mindspore.common.tensor import Tensor -from mindspore.nn.metrics import get_metrics -from mindspore._checkparam import check_input_data, check_output_data, check_int_positive, check_bool -from mindspore.train.callback import _InternalCallbackParam, RunContext, _build_callbacks +import numpy as np from mindspore import context +from mindspore import log as logger +from mindspore._c_expression import init_exec_dataset +from mindspore._checkparam import check_input_data, check_output_data, check_int_positive, check_bool +from mindspore.common import dtype as mstype +from mindspore.common.dtype import pytype_to_dtype +from mindspore.common.tensor import Tensor +from mindspore.nn.metrics import Loss +from mindspore.nn.metrics import get_metrics +from mindspore.nn.wrap.cell_wrapper import _VirtualDatasetCell from mindspore.parallel._utils import _get_parallel_mode, _get_device_num, _get_global_rank, \ _get_parameter_broadcast, _device_number_check, _parameter_broadcast_check -from mindspore.nn.metrics import Loss -from mindspore.nn.wrap import WithLossCell, WithEvalCell, \ - DataWrapper -from mindspore.nn.wrap.cell_wrapper import _VirtualDatasetCell -from mindspore.train.parallel_utils import ParallelMode -from mindspore.common import dtype as mstype -from second_order.dataset_helper import DatasetHelper from mindspore.train import amp - -from mindspore.common.dtype import pytype_to_dtype -from mindspore._c_expression import init_exec_dataset -from mindspore.common.parameter import Parameter - +from mindspore.train.callback import _InternalCallbackParam, RunContext, _build_callbacks +from mindspore.train.parallel_utils import ParallelMode +from second_order.dataset_helper import DatasetHelper + + def _convert_type(types): """ Convert from numpy type to tensor type. @@ -51,18 +48,20 @@ def _convert_type(types): ms_type = pytype_to_dtype(np_type) ms_types.append(ms_type) return ms_types - + + def _get_types_and_shapes(dataset): """Get dataset types and shapes.""" dataset_types = _convert_type(dataset.output_types()) dataset_shapes = dataset.output_shapes() return dataset_types, dataset_shapes - + + def _exec_datagraph(exec_dataset, dataset_size, phase='dataset'): """Initialize and execute the dataset graph.""" batch_size = exec_dataset.get_batch_size() input_indexs = exec_dataset.input_indexs - + # transform data format dataset_types, dataset_shapes = _get_types_and_shapes(exec_dataset) init_exec_dataset(exec_dataset.__ME_INITED__, @@ -72,8 +71,8 @@ def _exec_datagraph(exec_dataset, dataset_size, phase='dataset'): dataset_shapes, input_indexs, phase=phase) - - + + class Model: """ High-Level API for Training or Testing. @@ -131,7 +130,7 @@ class Model: >>> dataset = get_dataset() >>> model.train(2, dataset) """ - + def __init__(self, network, loss_fn=None, optimizer=None, metrics=None, eval_network=None, eval_indexes=None, amp_level="O0", frequency=278, **kwargs): self._network = network @@ -152,49 +151,49 @@ class Model: self._device_number = _get_device_num() self._global_rank = _get_global_rank() self._parameter_broadcast = _get_parameter_broadcast() - + self._train_network = self._build_train_network() self._build_eval_network(metrics, eval_network, eval_indexes) self._build_predict_network() - + def _check_kwargs(self, kwargs): for arg in kwargs: if arg not in ['loss_scale_manager', 'keep_batchnorm_fp32']: - raise ValueError(f"Unsupport arg '{arg}'") - + raise ValueError(f"Unsupport arg '{arg}'") + def _build_train_network(self): """Build train network""" network = self._network if self._optimizer: if self._loss_scale_manager_set: network = amp.build_train_network(network, - self._optimizer, - self._loss_fn, - level=self._amp_level, - loss_scale_manager=self._loss_scale_manager, - keep_batchnorm_fp32=self._keep_bn_fp32) + self._optimizer, + self._loss_fn, + level=self._amp_level, + loss_scale_manager=self._loss_scale_manager, + keep_batchnorm_fp32=self._keep_bn_fp32) else: network = amp.build_train_network(network, - self._optimizer, - self._loss_fn, - level=self._amp_level, - keep_batchnorm_fp32=self._keep_bn_fp32) + self._optimizer, + self._loss_fn, + level=self._amp_level, + keep_batchnorm_fp32=self._keep_bn_fp32) elif self._loss_fn: network = nn.WithLossCell(network, self._loss_fn) # If need to check if loss_fn is not None, but optimizer is None return network - + def _build_eval_network(self, metrics, eval_network, eval_indexes): """Build the network for evaluation.""" self._metric_fns = get_metrics(metrics) if not self._metric_fns: return - + if eval_network is not None: if eval_indexes is not None and not (isinstance(eval_indexes, list) and len(eval_indexes) == 3): raise ValueError("Eval_indexes must be a list or None. If eval_indexes is a list, length of it \ must be three. But got {}".format(eval_indexes)) - + self._eval_network = eval_network self._eval_indexes = eval_indexes else: @@ -202,27 +201,27 @@ class Model: raise ValueError("loss_fn can not be None.") self._eval_network = nn.WithEvalCell(self._network, self._loss_fn) self._eval_indexes = [0, 1, 2] - + def _build_predict_network(self): """Build the network for prediction.""" self._predict_network = self._network if self._parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL, ParallelMode.AUTO_PARALLEL): self._predict_network = _VirtualDatasetCell(self._network) - + def _clear_metrics(self): """Clear metrics local values.""" for metric in self._metric_fns.values(): metric.clear() - + def _update_metrics(self, outputs): """Update metrics local values.""" if not isinstance(outputs, tuple): raise ValueError("The `outputs` is not tuple.") - + if self._eval_indexes is not None and len(outputs) < 3: raise ValueError("The length of `outputs` must be greater than or equal to 3, \ but got {}".format(len(outputs))) - + for metric in self._metric_fns.values(): if self._eval_indexes is None: metric.update(*outputs) @@ -231,14 +230,14 @@ class Model: metric.update(outputs[self._eval_indexes[0]]) else: metric.update(outputs[self._eval_indexes[1]], outputs[self._eval_indexes[2]]) - + def _get_metrics(self): """Get metrics local values.""" metrics = dict() for key, value in self._metric_fns.items(): metrics[key] = value.eval() return metrics - + def _get_scaling_sens(self): """get the scaling sens""" scaling_sens = 1 @@ -247,7 +246,7 @@ class Model: if self._parallel_mode == ParallelMode.DATA_PARALLEL: scaling_sens /= self._device_number return scaling_sens - + def _train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=True): """ Training. @@ -266,10 +265,10 @@ class Model: """ epoch = check_int_positive(epoch) self._train_network.set_train() - + if self._parameter_broadcast: self._train_network.set_broadcast_flag() - + # build callback list list_callback = _build_callbacks(callbacks) cb_params = _InternalCallbackParam() @@ -283,7 +282,7 @@ class Model: cb_params.device_number = self._device_number cb_params.train_dataset = train_dataset cb_params.list_callback = list_callback - + if dataset_sink_mode: if context.get_context("mode") == context.PYNATIVE_MODE: logger.warning("The pynative mode cannot support dataset sink mode currently." @@ -293,7 +292,6 @@ class Model: self._train_dataset_sink_process(epoch, train_dataset, list_callback, cb_params) else: self._train_process(epoch, train_dataset, list_callback, cb_params) - def _train_dataset_sink_process(self, epoch, train_dataset, list_callback=None, cb_params=None): """ @@ -317,7 +315,7 @@ class Model: if not hasattr(train_dataset, '__ME_INITED__') and context.get_context("enable_loop_sink") \ and not context.get_context("enable_ge"): need_wrap = True - + dataset_helper = DatasetHelper(train_dataset, iter_first_order) # remove later to deal with loop sink if need_wrap: @@ -330,7 +328,7 @@ class Model: loop_size = dataset_helper.loop_size() run_context = RunContext(cb_params) list_callback.begin(run_context) - + # used to stop training for early stop, such as stopAtTIme or stopATStep should_stop = False has_do_train1_dataset = False @@ -338,7 +336,7 @@ class Model: for i in range(epoch): cb_params.cur_epoch_num = i + 1 list_callback.epoch_begin(run_context) - + # for data sink dataset_helper only iter once, other wise iter epoch_size times. for inputs in dataset_helper: list_callback.step_begin(run_context) @@ -357,14 +355,14 @@ class Model: outputs = self._train_network(*inputs) cb_params.net_outputs = outputs list_callback.step_end(run_context) - + list_callback.epoch_end(run_context) should_stop = should_stop or run_context.get_stop_requested() if should_stop: break - + list_callback.end(run_context) - + def _train_process(self, epoch, train_dataset, list_callback=None, cb_params=None): """ Training process. The data would be passed to network directly. @@ -385,12 +383,12 @@ class Model: _callback_wrapper(list_callback, run_context, "begin") # used to stop training for early stop, such as stopAtTIme or stopATStep should_stop = False - + for i in range(epoch): cb_params.cur_epoch_num = i + 1 - + _callback_wrapper(list_callback, run_context, "epoch_begin") - + for next_element in dataset_helper: len_element = len(next_element) if self._loss_fn and len_element != 2: @@ -398,33 +396,33 @@ class Model: "return two elements, but got {}".format(len_element)) cb_params.cur_step_num += 1 _callback_wrapper(list_callback, run_context, "step_begin") - + overflow = False if self._loss_scale_manager and self._loss_scale_manager.get_drop_overflow_update(): scaling_sens = self._get_scaling_sens() next_element = tuple(next_element) + (Tensor(scaling_sens, mstype.float32),) - + outputs = self._train_network(*next_element) cb_params.net_outputs = outputs if self._loss_scale_manager and self._loss_scale_manager.get_drop_overflow_update(): _, overflow, _ = outputs overflow = np.all(overflow.asnumpy()) self._loss_scale_manager.update_loss_scale(overflow) - + _callback_wrapper(list_callback, run_context, "step_end") should_stop = should_stop or run_context.get_stop_requested() if should_stop: break - + train_dataset.reset() - + _callback_wrapper(list_callback, run_context, "epoch_end") should_stop = should_stop or run_context.get_stop_requested() if should_stop: break - + _callback_wrapper(list_callback, run_context, "end") - + def train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=True): """ Training API where the iteration is controlled by python front-end. @@ -470,12 +468,12 @@ class Model: if context.get_context("device_target") in ["CPU", "GPU"] and context.get_context("enable_loop_sink"): raise ValueError("CPU and GPU can't support loop sink, please set enable_loop_sink=False.") - + self._train(epoch, train_dataset, callbacks=callbacks, dataset_sink_mode=dataset_sink_mode) - + def _eval_dataset_sink_process(self, valid_dataset, list_callback=None, cb_params=None): """ Evaluation. The data would be passed to network through dataset channel. @@ -489,42 +487,42 @@ class Model: Dict, returns the loss value & metrics values for the model in test mode. """ _device_number_check(self._parallel_mode, self._device_number) - + run_context = RunContext(cb_params) - + # remove later to deal with loop sink need_wrap = False if not hasattr(valid_dataset, '__ME_INITED__') and context.get_context("enable_loop_sink") \ - and not context.get_context("enable_ge"): + and not context.get_context("enable_ge"): need_wrap = True - + valid_dataset.__loop_size__ = 1 dataset_helper = DatasetHelper(valid_dataset) - + # remove later to deal with loop sink if need_wrap: self._eval_network = nn.DataWrapper(self._eval_network, *(dataset_helper.types_shapes()), - valid_dataset.__ME_INITED__) + valid_dataset.__ME_INITED__) self._eval_network.set_train(mode=False) self._eval_network.phase = 'eval' list_callback.begin(run_context) - + for inputs in dataset_helper: cb_params.cur_step_num += 1 list_callback.step_begin(run_context) - + outputs = self._eval_network(*inputs) - + cb_params.net_outputs = outputs list_callback.step_end(run_context) self._update_metrics(outputs) - + metrics = self._get_metrics() cb_params.metrics = metrics list_callback.end(run_context) - + return metrics - + def _eval_process(self, valid_dataset, list_callback=None, cb_params=None): """ Evaluation. The data would be passed to network directly. @@ -539,7 +537,7 @@ class Model: """ run_context = RunContext(cb_params) list_callback.begin(run_context) - + dataset_helper = DatasetHelper(valid_dataset, dataset_sink_mode=False) for next_element in dataset_helper: cb_params.cur_step_num += 1 @@ -548,12 +546,12 @@ class Model: cb_params.net_outputs = outputs list_callback.step_end(run_context) self._update_metrics(outputs) - + metrics = self._get_metrics() cb_params.metrics = metrics list_callback.end(run_context) return metrics - + def eval(self, valid_dataset, callbacks=None, dataset_sink_mode=True): """ Evaluation API where the iteration is controlled by python front-end. @@ -584,7 +582,7 @@ class Model: check_bool(dataset_sink_mode) if not self._metric_fns: raise ValueError("metric fn can not be None or empty.") - + list_callback = _build_callbacks(callbacks) cb_params = _InternalCallbackParam() cb_params.eval_network = self._eval_network @@ -592,16 +590,16 @@ class Model: cb_params.batch_num = valid_dataset.get_dataset_size() cb_params.mode = "eval" cb_params.cur_step_num = 0 - + self._eval_network.set_train(mode=False) self._eval_network.phase = 'eval' - + self._clear_metrics() - + if dataset_sink_mode: return self._eval_dataset_sink_process(valid_dataset, list_callback, cb_params) return self._eval_process(valid_dataset, list_callback, cb_params) - + def predict(self, *predict_data): """ Generates output predictions for the input samples. @@ -625,9 +623,9 @@ class Model: self._predict_network.set_train(False) check_input_data(*predict_data, data_class=Tensor) result = self._predict_network(*predict_data) - + check_output_data(result) return result - - + + __all__ = ["Model"] diff --git a/tests/st/networks/thor_test/model/resnet.py b/example/resnet101_imagenet2012_THOR/model/resnet.py similarity index 92% rename from tests/st/networks/thor_test/model/resnet.py rename to example/resnet101_imagenet2012_THOR/model/resnet.py index d77239dbf29..a6ba0b5c580 100644 --- a/tests/st/networks/thor_test/model/resnet.py +++ b/example/resnet101_imagenet2012_THOR/model/resnet.py @@ -13,13 +13,15 @@ # limitations under the License. # ============================================================================ """ResNet.""" -import numpy as np -import mindspore.nn as nn -from mindspore.ops import operations as P -from mindspore.common.tensor import Tensor -from second_order.thor_layer import Conv2d_Thor, Dense_Thor import math +import mindspore.nn as nn +import numpy as np +from mindspore.common.tensor import Tensor +from mindspore.ops import operations as P +from second_order.thor_layer import Conv2d_Thor, Dense_Thor + + def calculate_gain(nonlinearity, param=None): linear_fns = ['linear', 'conv1d', 'conv2d', 'conv3d', 'conv_transpose1d', 'conv_transpose2d', 'conv_transpose3d'] if nonlinearity in linear_fns or nonlinearity == 'sigmoid': @@ -39,12 +41,13 @@ def calculate_gain(nonlinearity, param=None): return math.sqrt(2.0 / (1 + negative_slope ** 2)) else: raise ValueError("Unsupported nonlinearity {}".format(nonlinearity)) - + + def _calculate_fan_in_and_fan_out(tensor): dimensions = len(tensor) if dimensions < 2: raise ValueError("Fan in and fan out can not be computed for tensor with fewer than 2 dimensions") - + if dimensions == 2: # Linear fan_in = tensor[1] fan_out = tensor[0] @@ -57,22 +60,25 @@ def _calculate_fan_in_and_fan_out(tensor): fan_in = num_input_fmaps * receptive_field_size fan_out = num_output_fmaps * receptive_field_size return fan_in, fan_out - + + def _calculate_correct_fan(tensor, mode): mode = mode.lower() valid_modes = ['fan_in', 'fan_out'] if mode not in valid_modes: raise ValueError("Mode {} not supported, please use one of {}".format(mode, valid_modes)) - + fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor) return fan_in if mode == 'fan_in' else fan_out + def kaiming_normal(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu'): fan = _calculate_correct_fan(inputs_shape, mode) gain = calculate_gain(nonlinearity, a) std = gain / math.sqrt(fan) return np.random.normal(0, std, size=inputs_shape).astype(np.float32) - + + def kaiming_uniform(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu'): fan = _calculate_correct_fan(inputs_shape, mode) gain = calculate_gain(nonlinearity, a) @@ -80,6 +86,7 @@ def kaiming_uniform(inputs_shape, a=0, mode='fan_in', nonlinearity='leaky_relu') bound = math.sqrt(3.0) * std # Calculate uniform bounds from standard deviation return np.random.uniform(-bound, bound, size=inputs_shape).astype(np.float32) + def _conv3x3(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): weight_shape = (out_channel, in_channel, 3, 3) weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) @@ -88,35 +95,41 @@ def _conv3x3(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, freq damping=damping, loss_scale=loss_scale, frequency=frequency) # return nn.Conv2d(in_channel, out_channel, # kernel_size=3, stride=stride, padding=0, pad_mode='same', weight_init=weight) - + + def _conv1x1(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): weight_shape = (out_channel, in_channel, 1, 1) weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) return Conv2d_Thor(in_channel, out_channel, - kernel_size=1, stride=stride, padding=0, pad_mode='same', weight_init=weight, - damping=damping, loss_scale=loss_scale, frequency=frequency) - + kernel_size=1, stride=stride, padding=0, pad_mode='same', weight_init=weight, + damping=damping, loss_scale=loss_scale, frequency=frequency) + + def _conv7x7(in_channel, out_channel, stride=1, damping=0.03, loss_scale=1, frequency=278): weight_shape = (out_channel, in_channel, 7, 7) weight = Tensor(kaiming_normal(weight_shape, mode="fan_out", nonlinearity='relu')) return Conv2d_Thor(in_channel, out_channel, - kernel_size=7, stride=stride, padding=0, pad_mode='same', weight_init=weight, - damping=damping, loss_scale=loss_scale, frequency=frequency) - + kernel_size=7, stride=stride, padding=0, pad_mode='same', weight_init=weight, + damping=damping, loss_scale=loss_scale, frequency=frequency) + + def _bn(channel): return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) - + + def _bn_last(channel): return nn.BatchNorm2d(channel, eps=1e-4, momentum=0.9, gamma_init=1, beta_init=0, moving_mean_init=0, moving_var_init=1) + def _fc(in_channel, out_channel, damping, loss_scale, frequency): weight_shape = (out_channel, in_channel) - weight = Tensor(kaiming_uniform(weight_shape, a = math.sqrt(5)) + weight = Tensor(kaiming_uniform(weight_shape, a=math.sqrt(5)) return Dense_Thor(in_channel, out_channel, has_bias=False, weight_init=weight, bias_init=0, damping=damping, loss_scale=loss_scale, frequency=frequency) - + + class ResidualBlock(nn.Cell): """ ResNet V1 residual block definition. @@ -133,7 +146,7 @@ class ResidualBlock(nn.Cell): >>> ResidualBlock(3, 256, stride=2) """ expansion = 4 - + def __init__(self, in_channel, out_channel, @@ -142,54 +155,58 @@ class ResidualBlock(nn.Cell): loss_scale=1, frequency=278): super(ResidualBlock, self).__init__() - + channel = out_channel // self.expansion - self.conv1 = _conv1x1(in_channel, channel, stride=1, damping=damping, loss_scale=loss_scale, frequency=frequency) + self.conv1 = _conv1x1(in_channel, channel, stride=1, damping=damping, loss_scale=loss_scale, + frequency=frequency) self.bn1 = _bn(channel) - - self.conv2 = _conv3x3(channel, channel, stride=stride, damping=damping, loss_scale=loss_scale, frequency=frequency) + + self.conv2 = _conv3x3(channel, channel, stride=stride, damping=damping, loss_scale=loss_scale, + frequency=frequency) self.bn2 = _bn(channel) - - self.conv3 = _conv1x1(channel, out_channel, stride=1, damping=damping, loss_scale=loss_scale, frequency=frequency) + + self.conv3 = _conv1x1(channel, out_channel, stride=1, damping=damping, loss_scale=loss_scale, + frequency=frequency) self.bn3 = _bn_last(out_channel) - + self.relu = nn.ReLU() - + self.down_sample = False - + if stride != 1 or in_channel != out_channel: self.down_sample = True self.down_sample_layer = None - + if self.down_sample: self.down_sample_layer = nn.SequentialCell([_conv1x1(in_channel, out_channel, stride, - damping=damping, loss_scale=loss_scale, frequency=frequency), + damping=damping, loss_scale=loss_scale, + frequency=frequency), _bn(out_channel)]) self.add = P.TensorAdd() - + def construct(self, x): identity = x - + out = self.conv1(x) out = self.bn1(out) out = self.relu(out) - + out = self.conv2(out) out = self.bn2(out) out = self.relu(out) - + out = self.conv3(out) out = self.bn3(out) - + if self.down_sample: identity = self.down_sample_layer(identity) - + out = self.add(out, identity) out = self.relu(out) - + return out - - + + class ResNet(nn.Cell): """ ResNet architecture. @@ -212,7 +229,7 @@ class ResNet(nn.Cell): >>> [1, 2, 2, 2], >>> 10) """ - + def __init__(self, block, layer_nums, @@ -224,15 +241,15 @@ class ResNet(nn.Cell): loss_scale, frequency): super(ResNet, self).__init__() - + if not len(layer_nums) == len(in_channels) == len(out_channels) == 4: raise ValueError("the length of layer_num, in_channels, out_channels list must be 4!") - + self.conv1 = _conv7x7(3, 64, stride=2, damping=damping, loss_scale=loss_scale, frequency=frequency) self.bn1 = _bn(64) self.relu = P.ReLU() self.maxpool = P.MaxPoolWithArgmax(padding="same", ksize=3, strides=2) - + self.layer1 = self._make_layer(block, layer_nums[0], in_channel=in_channels[0], @@ -253,7 +270,7 @@ class ResNet(nn.Cell): layer_nums[2], in_channel=in_channels[2], out_channel=out_channels[2], - stride=strides[2],damping=damping, + stride=strides[2], damping=damping, loss_scale=loss_scale, frequency=frequency) self.layer4 = self._make_layer(block, @@ -264,11 +281,11 @@ class ResNet(nn.Cell): damping=damping, loss_scale=loss_scale, frequency=frequency) - + self.mean = P.ReduceMean(keep_dims=True) self.flatten = nn.Flatten() self.end_point = _fc(out_channels[3], num_classes, damping=damping, loss_scale=loss_scale, frequency=frequency) - + def _make_layer(self, block, layer_num, in_channel, out_channel, stride, damping, loss_scale, frequency): """ @@ -288,36 +305,36 @@ class ResNet(nn.Cell): >>> _make_layer(ResidualBlock, 3, 128, 256, 2) """ layers = [] - + resnet_block = block(in_channel, out_channel, stride=stride, damping=damping, loss_scale=loss_scale, frequency=frequency) layers.append(resnet_block) - + for _ in range(1, layer_num): resnet_block = block(out_channel, out_channel, stride=1, damping=damping, loss_scale=loss_scale, frequency=frequency) layers.append(resnet_block) - + return nn.SequentialCell(layers) - + def construct(self, x): x = self.conv1(x) x = self.bn1(x) x = self.relu(x) c1, argmax = self.maxpool(x) - + c2 = self.layer1(c1) c3 = self.layer2(c2) c4 = self.layer3(c3) c5 = self.layer4(c4) - + out = self.mean(c5, (2, 3)) out = self.flatten(out) out = self.end_point(out) - + return out - - + + def resnet50(class_num=10, damping=0.03, loss_scale=1, frequency=278): """ Get ResNet50 neural network. diff --git a/tests/st/networks/thor_test/model/thor.py b/example/resnet101_imagenet2012_THOR/model/thor.py similarity index 94% rename from tests/st/networks/thor_test/model/thor.py rename to example/resnet101_imagenet2012_THOR/model/thor.py index 33d11c018fd..82e4e3b1108 100644 --- a/tests/st/networks/thor_test/model/thor.py +++ b/example/resnet101_imagenet2012_THOR/model/thor.py @@ -13,42 +13,47 @@ # limitations under the License. # ============================================================================ """momentum""" -import numpy as np -from mindspore.ops import functional as F, composite as C, operations as P -from mindspore.common.parameter import Parameter -from mindspore.common.tensor import Tensor import mindspore.common.dtype as mstype -from mindspore.nn.optim.optimizer import Optimizer -from mindspore.common.parameter import ParameterTuple -from mindspore.parallel._utils import _get_device_num, _get_parallel_mode, _get_mirror_mean -from mindspore.common.initializer import initializer -from model.grad_reducer_thor import DistributedGradReducerThor -from cus_ops.cus_matmul_cube_fracz_right_mul import CusMatMulCubeFraczRightMul -from cus_ops.cus_fused_abs_max1 import CusFusedAbsMax1 -from cus_ops.cus_matmul_cube_fracz_left_cast import CusMatMulCubeFraczLeftCast -from cus_ops.cus_matmul_cube_dense_left import CusMatMulCubeDenseLeft from cus_ops.cus_matmul_cube_dense_right import CusMatMulCubeDenseRight - +from cus_ops.cus_matmul_cube_fracz_left_cast import CusMatMulCubeFraczLeftCast +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.common.parameter import ParameterTuple +from mindspore.common.tensor import Tensor +from mindspore.nn.optim.optimizer import Optimizer +from mindspore.ops import functional as F, composite as C, operations as P +from mindspore.parallel._utils import _get_device_num, _get_mirror_mean + +from cus_ops.cus_matmul_cube_dense_left import CusMatMulCubeDenseLeft +from cus_ops.cus_matmul_cube_fracz_right_mul import CusMatMulCubeFraczRightMul +from model.grad_reducer_thor import DistributedGradReducerThor + momentum_opt = C.MultitypeFuncGraph("momentum_opt") + + @momentum_opt.register("Function", "Tensor", "Tensor", "Tensor", "Tensor", "Tensor") def _tensor_run_opt_ext(opt, learning_rate, momentum, gradient, weight, moment): """Apply momentum optimizer to the weight parameter using Tensor.""" success = True success = F.depend(success, opt(weight, moment, learning_rate, gradient, momentum)) return success - + + op_add = P.AddN() apply_decay = C.MultitypeFuncGraph("apply_decay") - + + @apply_decay.register("Number", "Bool", "Tensor", "Tensor") def _tensor_apply_decay(weight_decay, if_apply, weight, gradient): """Get grad with weight_decay.""" if if_apply: return op_add((weight * weight_decay, gradient)) return gradient - + + class THOR(Optimizer): - def __init__(self, params, learning_rate, momentum, matrix_A, matrix_G, A_inv_max, G_inv_max, weight_decay=0.0, loss_scale=1.0, + def __init__(self, params, learning_rate, momentum, matrix_A, matrix_G, A_inv_max, G_inv_max, weight_decay=0.0, + loss_scale=1.0, decay_filter=lambda x: x.name not in []): super(THOR, self).__init__(learning_rate, params, weight_decay, loss_scale) if isinstance(momentum, float) and momentum < 0.0: @@ -93,9 +98,10 @@ class THOR(Optimizer): self.matrix_A_inv = () self.matrix_G_inv = () self.matrix_max_inv = () - + for i in range(54): - self.matrix_max_inv = self.matrix_max_inv + (Parameter(initializer(1, [1], mstype.float32), name="matrix_max"+str(i), requires_grad=False), ) + self.matrix_max_inv = self.matrix_max_inv + ( + Parameter(initializer(1, [1], mstype.float32), name="matrix_max" + str(i), requires_grad=False),) self.log = P.Log() self.exp = P.Exp() self.sqrt = P.Sqrt() @@ -105,7 +111,7 @@ class THOR(Optimizer): self.thor = True self.weight_decay = weight_decay * loss_scale self.decay_flags = tuple(decay_filter(x) for x in self.parameters) - + def construct(self, gradients): params = self.params moments = self.moments @@ -124,9 +130,9 @@ class THOR(Optimizer): matrix_G = F.depend(matrix_G, g) A_max = F.depend(A_max, g) G_max = F.depend(G_max, g) - matrix_A_allreduce = matrix_A_allreduce + (matrix_A, ) - matrix_G_allreduce = matrix_G_allreduce + (matrix_G, ) - matrix_A_max_allreduce = matrix_A_max_allreduce + (A_max, ) + matrix_A_allreduce = matrix_A_allreduce + (matrix_A,) + matrix_G_allreduce = matrix_G_allreduce + (matrix_G,) + matrix_A_max_allreduce = matrix_A_max_allreduce + (A_max,) matrix_G_max_allreduce = matrix_G_max_allreduce + (G_max,) matrix_A_allreduce = self.grad_reducer_A(matrix_A_allreduce) matrix_G_allreduce = self.grad_reducer_G(matrix_G_allreduce) @@ -182,13 +188,13 @@ class THOR(Optimizer): new_grads = new_grads + (g,) else: g = self.cube_matmul_left(matrix_G, g) - g =self.cube_matmul_right_mul(g, matrix_A, matrix_max) + g = self.cube_matmul_right_mul(g, matrix_A, matrix_max) new_grads = new_grads + (g, gradients[i * 3 + 1], gradients[i * 3 + 2]) gradients = new_grads - + if self.weight_decay > 0: gradients = self.hyper_map(F.partial(apply_decay, self.weight_decay), self.decay_flags, - params, gradients) + params, gradients) gradients = self.scale_grad(gradients) lr = self.get_lr() success = self.hyper_map(F.partial(momentum_opt, self.opt, lr, self.momentum), gradients, params, moments) diff --git a/tests/st/networks/thor_test/model/thor_layer.py b/example/resnet101_imagenet2012_THOR/model/thor_layer.py similarity index 80% rename from tests/st/networks/thor_test/model/thor_layer.py rename to example/resnet101_imagenet2012_THOR/model/thor_layer.py index f91a7e2d169..7f777aa3360 100644 --- a/tests/st/networks/thor_test/model/thor_layer.py +++ b/example/resnet101_imagenet2012_THOR/model/thor_layer.py @@ -13,27 +13,29 @@ # limitations under the License. # ============================================================================ -import numpy as np import mindspore as ms import mindspore.common.dtype as mstype -from mindspore.ops import operations as P -from mindspore.common.parameter import Parameter -from mindspore.common.initializer import initializer +import numpy as np from mindspore._checkparam import check_bool, twice, check_int_positive -from mindspore.nn.cell import Cell -from mindspore.ops import functional as F -from mindspore.common.tensor import Tensor -from mindspore.nn.layer.activation import get_activation from mindspore._extends import cell_attr_register +from mindspore.common.initializer import initializer +from mindspore.common.parameter import Parameter +from mindspore.common.tensor import Tensor +from mindspore.nn.cell import Cell +from mindspore.nn.layer.activation import get_activation +from mindspore.ops import operations as P + +from cus_ops.cus_batch_matmul import CusBatchMatMul +from cus_ops.cus_cholesky_trsm import CusCholeskyTrsm +from cus_ops.cus_fused_abs_max1 import CusFusedAbsMax1 +from cus_ops.cus_img2col import CusImg2Col from cus_ops.cus_matmul_cube import CusMatMulCube from cus_ops.cus_matrix_combine import CusMatrixCombine -from cus_ops.cus_cholesky_trsm import CusCholeskyTrsm -from cus_ops.cus_img2col import CusImg2Col -from cus_ops.cus_fused_abs_max1 import CusFusedAbsMax1 -from cus_ops.cus_batch_matmul import CusBatchMatMul from cus_ops.cus_transpose02314 import CusTranspose02314 - + C0 = 16 + + def caculate_device_shape(matrix_dim, channel, is_A): if is_A: if channel // C0 == 0: @@ -41,11 +43,13 @@ def caculate_device_shape(matrix_dim, channel, is_A): return (int(matrix_dim // C0), int(matrix_dim // C0), C0, C0), int(matrix_dim) else: return (int(matrix_dim // C0), int(matrix_dim // C0), C0, C0), int(matrix_dim) - + + class _Conv(Cell): r"""Applies a N-D convolution over an input signal composed of several input planes. """ + def __init__(self, in_channels, out_channels, @@ -73,23 +77,23 @@ class _Conv(Cell): self.has_bias = has_bias if not (isinstance(in_channels, int) and in_channels > 0): raise ValueError('Attr \'in_channels\' of \'Conv2D\' Op passed ' - +str(in_channels)+ ', should be a int and greater than 0.') + + str(in_channels) + ', should be a int and greater than 0.') if (not isinstance(kernel_size, tuple)) or len(kernel_size) != 2 or \ - (not isinstance(kernel_size[0], int)) or (not isinstance(kernel_size[1], int)) or \ + (not isinstance(kernel_size[0], int)) or (not isinstance(kernel_size[1], int)) or \ kernel_size[0] < 1 or kernel_size[1] < 1: raise ValueError('Attr \'kernel_size\' of \'Conv2D\' Op passed ' - +str(self.kernel_size)+', should be a int or tuple and equal to or greater than 1.') + + str(self.kernel_size) + ', should be a int or tuple and equal to or greater than 1.') if in_channels % group != 0: raise ValueError('Attr \'in_channels\' of \'Conv2D\' Op must be divisible by ' 'attr \'group\' of \'Conv2D\' Op.') if out_channels % group != 0: raise ValueError('Attr \'out_channels\' of \'Conv2D\' Op must be divisible by ' 'attr \'group\' of \'Conv2D\' Op.') - + self.weight = Parameter(initializer( weight_init, [out_channels, in_channels // group, *kernel_size]), - name='weight') - + name='weight') + if check_bool(has_bias): self.bias = Parameter(_initializer( bias_init, [out_channels]), name='bias') @@ -97,10 +101,11 @@ class _Conv(Cell): if bias_init != 'zeros': logger.warning("Value of 'has_bias' is False, value of 'bias_init' will be ignored.") self.bias = None - + def construct(self, *inputs): raise NotImplementedError - + + class Conv2d_Thor(_Conv): def __init__(self, in_channels, @@ -120,7 +125,7 @@ class Conv2d_Thor(_Conv): bias_init='zeros'): self.thor = True ksizes = (1, kernel_size, kernel_size, 1) - self.hw = kernel_size*kernel_size + self.hw = kernel_size * kernel_size strides = (1, stride, stride, 1) kernel_size = twice(kernel_size) super(Conv2d_Thor, self).__init__( @@ -146,26 +151,37 @@ class Conv2d_Thor(_Conv): dilation=self.dilation, group=self.group ) - - self.img2col = CusImg2Col(ksizes = ksizes, strides = strides) + + self.img2col = CusImg2Col(ksizes=ksizes, strides=strides) self.cube_matmul = CusMatMulCube(transpose_a=True) self.matrix_combine = CusMatrixCombine() self.cholesky = CusCholeskyTrsm() self.transpose02314 = CusTranspose02314() - self.matrix_A_dim = self.in_channels*self.kernel_size[0]*self.kernel_size[1] + self.matrix_A_dim = self.in_channels * self.kernel_size[0] * self.kernel_size[1] self.matrix_G_dim = self.out_channels - self.matrix_A_device_shape, self.matrix_A_device_dim = caculate_device_shape(self.matrix_A_dim, self.in_channels, True) - self.matrix_G_device_shape, self.matrix_G_device_dim = caculate_device_shape(self.matrix_G_dim, self.in_channels, False) - self.matrix_A_device_temp_shape = (self.matrix_A_device_shape[0], self.matrix_A_device_shape[2], self.matrix_A_device_shape[1], self.matrix_A_device_shape[3]) - self.matrix_G_device_temp_shape = (self.matrix_G_device_shape[0], self.matrix_G_device_shape[2], self.matrix_G_device_shape[1], self.matrix_G_device_shape[3]) - self.matrix_A_inv = Parameter(Tensor(np.reshape(np.identity(self.matrix_A_device_dim).astype(np.float16), self.matrix_A_device_shape)), name='matrix_A_inv', requires_grad=False) + self.matrix_A_device_shape, self.matrix_A_device_dim = caculate_device_shape(self.matrix_A_dim, + self.in_channels, True) + self.matrix_G_device_shape, self.matrix_G_device_dim = caculate_device_shape(self.matrix_G_dim, + self.in_channels, False) + self.matrix_A_device_temp_shape = ( + self.matrix_A_device_shape[0], self.matrix_A_device_shape[2], self.matrix_A_device_shape[1], + self.matrix_A_device_shape[3]) + self.matrix_G_device_temp_shape = ( + self.matrix_G_device_shape[0], self.matrix_G_device_shape[2], self.matrix_G_device_shape[1], + self.matrix_G_device_shape[3]) + self.matrix_A_inv = Parameter( + Tensor(np.reshape(np.identity(self.matrix_A_device_dim).astype(np.float16), self.matrix_A_device_shape)), + name='matrix_A_inv', requires_grad=False) self.A_inv_max = Parameter(initializer(0, [1], mstype.float32), name="A_inv_max", requires_grad=False) - self.matrix_G_inv = Parameter(Tensor(np.reshape(np.identity(self.matrix_G_device_dim).astype(np.float16), self.matrix_G_device_shape)), name="matrix_G_inv", requires_grad=False) - + self.matrix_G_inv = Parameter( + Tensor(np.reshape(np.identity(self.matrix_G_device_dim).astype(np.float16), self.matrix_G_device_shape)), + name="matrix_G_inv", requires_grad=False) + self.G_inv_max = Parameter(initializer(0, [1], mstype.float32), name="G_inv_max", requires_grad=False) - self.fake_G = Tensor(np.reshape(np.identity(self.matrix_G_device_dim).astype(np.float16), self.matrix_G_device_shape)) - self.fake_G_inv_max = Tensor(np.zeros([1,]).astype(np.float32)) - + self.fake_G = Tensor( + np.reshape(np.identity(self.matrix_G_device_dim).astype(np.float16), self.matrix_G_device_shape)) + self.fake_G_inv_max = Tensor(np.zeros([1, ]).astype(np.float32)) + self.shape = P.Shape() self.reshape = P.Reshape() self.transpose = P.Transpose() @@ -178,9 +194,10 @@ class Conv2d_Thor(_Conv): self.channels_slice_flag = False if self.in_channels % C0 != 0: self.channels_slice_flag = True - + self.padA_flag = False - if (self.matrix_A_dim // self.diag_block_dim) * self.diag_block_dim != self.matrix_A_dim and self.matrix_A_dim > self.diag_block_dim: + if ( + self.matrix_A_dim // self.diag_block_dim) * self.diag_block_dim != self.matrix_A_dim and self.matrix_A_dim > self.diag_block_dim: self.padA_flag = True pad_dim = self.diag_block_dim - self.matrix_A_dim % self.diag_block_dim self.padA = P.Pad(((0, pad_dim), (0, pad_dim))) @@ -191,16 +208,16 @@ class Conv2d_Thor(_Conv): self.slice = P.Slice() self.gather = P.GatherV2() self.freq = Tensor(frequency, mstype.int32) - self.loss_scale = Tensor(1/loss_scale, mstype.float16) + self.loss_scale = Tensor(1 / loss_scale, mstype.float16) self.axis = 0 - + dampingA_dim = self.matrix_A_dim if (self.matrix_A_dim % self.diag_block_dim) != 0 and self.matrix_A_dim > self.diag_block_dim: dampingA_dim = (self.matrix_A_dim // self.diag_block_dim + 1) * self.diag_block_dim dampingG_dim = self.matrix_G_dim if (self.matrix_G_dim % self.diag_block_dim) != 0 and self.matrix_G_dim > self.diag_block_dim: dampingG_dim = (self.matrix_G_dim // self.diag_block_dim + 1) * self.diag_block_dim - + self.dampingA = Tensor(np.identity(dampingA_dim), mstype.float32) self.dampingG = Tensor(np.identity(dampingG_dim), mstype.float32) self.fused_abs_max1 = CusFusedAbsMax1([self.matrix_A_dim, self.matrix_A_dim]) @@ -211,50 +228,50 @@ class Conv2d_Thor(_Conv): self.getG = P.InsertGradientOf(self.save_gradient) def save_gradient(self, dout): - out = dout - dout = self.mul(dout, self.loss_scale) - dout = self.mul(dout, 32.0) - dout = self.transpose02314(dout) - dout_shape = self.shape(dout) - normalizer = dout_shape[0] - - matrix_G = self.cube_matmul(dout, dout) - normalizer = self.cast(normalizer, ms.float32) - matrix_G = self.mul(matrix_G, 1.0/normalizer) - damping_step = self.gather(self.damping, self.cov_step, 0) - self.cov_step = self.cov_step + self.freq - damping_step = self.cast(damping_step, mstype.float32) - damping = self.mul(damping_step, 32.0/normalizer) - damping = self.sqrt(damping) - dampingG = self.cast(self.dampingG, mstype.float32) - matrix_G = matrix_G + damping * dampingG - - matrix_G_inv = self.cholesky(matrix_G) - matrix_G_inv = self.vector_matmul(matrix_G_inv, matrix_G_inv) - matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv) - matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv_max) - self.G_inv_max = matrix_G_inv_max - matrix_G_inv = self.matrix_combine(matrix_G_inv) - matrix_G_inv_shape = self.shape(matrix_G_inv) - matrix_G_inv = self.reshape(matrix_G_inv, self.matrix_G_device_temp_shape) - matrix_G_inv = self.transpose(matrix_G_inv, (2, 0, 1, 3)) - matrix_G = self.cast(matrix_G_inv, mstype.float16) - self.matrix_G_inv = matrix_G - return out - + out = dout + dout = self.mul(dout, self.loss_scale) + dout = self.mul(dout, 32.0) + dout = self.transpose02314(dout) + dout_shape = self.shape(dout) + normalizer = dout_shape[0] + + matrix_G = self.cube_matmul(dout, dout) + normalizer = self.cast(normalizer, ms.float32) + matrix_G = self.mul(matrix_G, 1.0 / normalizer) + damping_step = self.gather(self.damping, self.cov_step, 0) + self.cov_step = self.cov_step + self.freq + damping_step = self.cast(damping_step, mstype.float32) + damping = self.mul(damping_step, 32.0 / normalizer) + damping = self.sqrt(damping) + dampingG = self.cast(self.dampingG, mstype.float32) + matrix_G = matrix_G + damping * dampingG + + matrix_G_inv = self.cholesky(matrix_G) + matrix_G_inv = self.vector_matmul(matrix_G_inv, matrix_G_inv) + matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv) + matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv_max) + self.G_inv_max = matrix_G_inv_max + matrix_G_inv = self.matrix_combine(matrix_G_inv) + matrix_G_inv_shape = self.shape(matrix_G_inv) + matrix_G_inv = self.reshape(matrix_G_inv, self.matrix_G_device_temp_shape) + matrix_G_inv = self.transpose(matrix_G_inv, (2, 0, 1, 3)) + matrix_G = self.cast(matrix_G_inv, mstype.float16) + self.matrix_G_inv = matrix_G + return out + def construct(self, x): if self.thor: matrix_A = self.img2col(x) matrix_A_shape = self.shape(matrix_A) normalizer = matrix_A_shape[0] matrix_A = self.cube_matmul(matrix_A, matrix_A) - + if self.channels_slice_flag: matrix_A = self.reshape(matrix_A, (self.hw, C0, self.hw, C0)) matrix_A = self.slice(matrix_A, (0, 0, 0, 0), (self.hw, self.in_channels, self.hw, self.in_channels)) matrix_A = self.reshape(matrix_A, (self.matrix_A_dim, self.matrix_A_dim)) normalizer = self.cast(normalizer, ms.float32) - matrix_A = self.mul(matrix_A, 1.0/normalizer) + matrix_A = self.mul(matrix_A, 1.0 / normalizer) if self.padA_flag: matrix_A = self.padA(matrix_A) damping_step = self.gather(self.damping, self.cov_step, self.axis) @@ -273,7 +290,7 @@ class Conv2d_Thor(_Conv): in_channels = self.in_channels if self.padA_flag: matrix_A_inv = self.slice(matrix_A_inv, (0, 0), (self.matrix_A_dim, self.matrix_A_dim)) - + if self.device_shape_pad_flag: matrix_A_inv = self.reshape(matrix_A_inv, (self.hw, self.in_channels, self.hw, self.in_channels)) matrix_A_inv = self.device_shape_pad(matrix_A_inv) @@ -286,31 +303,32 @@ class Conv2d_Thor(_Conv): out = self.getG(out) else: out = self.conv2d(x, self.weight) - + return out - + def extra_repr(self): s = 'input_channels={}, output_channels={}, kernel_size={},' \ - 'stride={}, pad_mode={}, padding={}, dilation={}, ' \ + 'stride={}, pad_mode={}, padding={}, dilation={}, ' \ 'group={}, data_format={}, has_bias={},' \ - 'weight_init={}, bias_init={}'.format( - self.in_channels, - self.out_channels, - self.kernel_size, - self.stride, - self.pad_mode, - self.padding, - self.dilation, - self.group, - self.data_format, - self.has_bias, - self.weight, - self.bias) - + 'weight_init={}, bias_init={}'.format( + self.in_channels, + self.out_channels, + self.kernel_size, + self.stride, + self.pad_mode, + self.padding, + self.dilation, + self.group, + self.data_format, + self.has_bias, + self.weight, + self.bias) + if self.has_bias: s += ', bias={}'.format(self.bias) return s - + + class Dense_Thor(Cell): @cell_attr_register(attrs=['has_bias', 'activation']) def __init__(self, @@ -330,30 +348,30 @@ class Dense_Thor(Cell): self.thor = True if isinstance(weight_init, Tensor): if weight_init.dim() != 2 or weight_init.shape()[0] != out_channels or \ - weight_init.shape()[1] != in_channels: + weight_init.shape()[1] != in_channels: raise ValueError("weight_init shape error") - + self.weight = Parameter(initializer(weight_init, [out_channels, in_channels]), name="weight") - + if self.has_bias: if isinstance(bias_init, Tensor): if bias_init.dim() != 1 or bias_init.shape()[0] != out_channels: raise ValueError("bias_init shape error") - + self.bias = Parameter(initializer(bias_init, [out_channels]), name="bias") - + self.matmul = P.MatMul(transpose_b=True) self.bias_add = P.BiasAdd() - + self.activation = get_activation(activation) self.activation_flag = self.activation is not None - + self.matrix_A_inv = Parameter(Tensor(np.zeros([128, 128, 16, 16]).astype(np.float16)), name='matrix_A_inv', requires_grad=False) self.matrix_G_inv = Parameter(Tensor(np.zeros([63, 63, 16, 16]).astype(np.float16)), name="matrix_G_inv", requires_grad=False) self.fake_G = Tensor(np.zeros([63, 63, 16, 16]).astype(np.float16)) - + self.matmul = P.MatMul(transpose_b=True) self.cube_matmul = CusMatMulCube(transpose_a=True) self.matrix_combine = CusMatrixCombine() @@ -365,7 +383,7 @@ class Dense_Thor(Cell): self.mul = P.Mul() self.cast = P.Cast() self.damping = Tensor(damping) - self.loss_scale = Tensor(1/loss_scale, mstype.float16) + self.loss_scale = Tensor(1 / loss_scale, mstype.float16) self.vector_matmul = CusBatchMatMul() self.pad = P.Pad(((0, 24), (0, 24))) self.pad1 = P.Pad(((0, 8), (0, 8))) @@ -415,14 +433,14 @@ class Dense_Thor(Cell): matrix_G_inv = self.cast(matrix_G_inv, mstype.float16) self.matrix_G_inv = matrix_G_inv return out - + def construct(self, x): if self.thor: inputs = self.cube_matmul(x, x) normalizer = 32 normalizer = self.cast(normalizer, ms.float32) matrix_A = self.mul(inputs, 1.0 / normalizer) - + damping_step = self.gather(self.damping, self.cov_step, self.axis) damping_step = self.cast(damping_step, mstype.float32) damping = self.sqrt(damping_step) @@ -430,11 +448,11 @@ class Dense_Thor(Cell): matrix_A = matrix_A + damping * dampingA matrix_A_inv = self.cholesky(matrix_A) matrix_A_inv = self.vector_matmul(matrix_A_inv, matrix_A_inv) - + matrix_A_inv_max = self.fused_abs_max2(matrix_A_inv) matrix_A_inv_max = self.fused_abs_max2(matrix_A_inv_max) self.A_inv_max = matrix_A_inv_max - + matrix_A_inv = self.matrix_combine(matrix_A_inv) matrix_A_inv_shape = self.shape(matrix_A_inv) matrix_A_inv = self.reshape(matrix_A_inv, (matrix_A_inv_shape[0] / 16, 16, matrix_A_inv_shape[0] / 16, 16)) @@ -446,20 +464,20 @@ class Dense_Thor(Cell): output = self.getG(output) else: output = self.matmul(x, self.weight) - + if self.has_bias: output = self.bias_add(output, self.bias) if self.activation_flag: return self.activation(output) return output - + def extend_repr(self): str_info = 'in_channels={}, out_channels={}, weight={}, has_bias={}' \ - .format(self.in_channels, self.out_channels, self.weight, self.has_bias) + .format(self.in_channels, self.out_channels, self.weight, self.has_bias) if self.has_bias: str_info = str_info + ', bias={}'.format(self.bias) - + if self.activation_flag: str_info = str_info + ', activation={}'.format(self.activation) - + return str_info diff --git a/tests/st/networks/thor_test/run_distribute_train_new.sh b/example/resnet101_imagenet2012_THOR/run_distribute_train_new.sh similarity index 100% rename from tests/st/networks/thor_test/run_distribute_train_new.sh rename to example/resnet101_imagenet2012_THOR/run_distribute_train_new.sh diff --git a/tests/st/networks/thor_test/train.py b/example/resnet101_imagenet2012_THOR/train.py similarity index 89% rename from tests/st/networks/thor_test/train.py rename to example/resnet101_imagenet2012_THOR/train.py index 6e3febd144c..978d0c0dab1 100644 --- a/tests/st/networks/thor_test/train.py +++ b/example/resnet101_imagenet2012_THOR/train.py @@ -13,62 +13,54 @@ # limitations under the License. # ============================================================================ """train_imagenet.""" -import os import argparse +import os import random + +import mindspore.dataset.engine as de import numpy as np -from dataset_imagenet import create_dataset -from lr_generator import get_lr, warmup_cosine_annealing_lr -from config_imagenet import config -from mindspore import context from mindspore import Tensor +from mindspore import context +from mindspore.communication.management import init from mindspore.parallel._auto_parallel_context import auto_parallel_context -from mindspore.nn.optim.momentum import Momentum -from mindspore.nn.loss import SoftmaxCrossEntropyWithLogits - -from mindspore.train.model import ParallelMode - from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMonitor, TimeMonitor from mindspore.train.loss_scale_manager import FixedLossScaleManager -import mindspore.dataset.engine as de -from mindspore.communication.management import init - -import math -import mindspore.nn as nn -from crossentropy import CrossEntropy -from var_init import default_recurisive_init, KaimingNormal -from mindspore.common import initializer as weight_init - -from second_order.thor import THOR +from mindspore.train.model import ParallelMode from second_order.model_second_order import Model from second_order.resnet import resnet50 -from mindspore.train.serialization import load_checkpoint, load_param_into_net - +from second_order.thor import THOR + +from config_imagenet import config +from crossentropy import CrossEntropy +from dataset_imagenet import create_dataset +from lr_generator import get_lr, warmup_cosine_annealing_lr + random.seed(1) np.random.seed(1) de.config.set_seed(1) - + parser = argparse.ArgumentParser(description='Image classification') parser.add_argument('--run_distribute', type=bool, default=False, help='Run distribute') parser.add_argument('--device_num', type=int, default=1, help='Device num.') parser.add_argument('--do_train', type=bool, default=True, help='Do train or not.') parser.add_argument('--do_eval', type=bool, default=False, help='Do eval or not.') parser.add_argument('--dataset_path', type=str, default=None, help='Dataset path') - + args_opt = parser.parse_args() device_id = int(os.getenv('DEVICE_ID')) - + context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=True, device_id=device_id) context.set_context(enable_task_sink=True) context.set_context(enable_loop_sink=True) context.set_context(enable_mem_reuse=True) + def get_second_order_lr(global_step, lr_init, decay, total_epochs, steps_per_epoch): lr_each_step = [] total_steps = steps_per_epoch * total_epochs for i in range(total_steps): - epoch = (i+1)/steps_per_epoch - base = (1.0 - float(epoch)/total_epochs)**decay + epoch = (i + 1) / steps_per_epoch + base = (1.0 - float(epoch) / total_epochs) ** decay lr = lr_init * base lr_each_step.append(lr) current_step = global_step @@ -77,11 +69,12 @@ def get_second_order_lr(global_step, lr_init, decay, total_epochs, steps_per_epo learning_rate = lr_each_step[current_step:] return learning_rate + def get_second_order_damping(global_step, damping_init, decay_rate, total_epochs, steps_per_epoch): damping_each_step = [] total_steps = steps_per_epoch * total_epochs for step in range(total_steps): - epoch = (step+1) / steps_per_epoch + epoch = (step + 1) / steps_per_epoch damping = damping_init * (decay_rate ** (epoch / 10)) damping_each_step.append(damping) @@ -91,6 +84,7 @@ def get_second_order_damping(global_step, damping_init, decay_rate, total_epochs print("damping_is=========", damping) return damping + if __name__ == '__main__': if args_opt.do_eval: print("eval") @@ -104,7 +98,7 @@ if __name__ == '__main__': init() else: print(" ") - + epoch_size = config.epoch_size damping = get_second_order_damping(0, 0.03, 0.87, 50, 5004) net = resnet50(class_num=config.class_num, damping=damping, loss_scale=config.loss_scale, @@ -128,8 +122,8 @@ if __name__ == '__main__': config.eta_min)) else: lr = Tensor(get_lr(global_step=0, lr_init=config.lr_init, lr_end=config.lr_end, lr_max=config.lr_max, - warmup_epochs=config.warmup_epochs, total_epochs=epoch_size, steps_per_epoch=step_size, - lr_decay_mode='poly')) + warmup_epochs=config.warmup_epochs, total_epochs=epoch_size, steps_per_epoch=step_size, + lr_decay_mode='poly')) opt = THOR(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, damping, config.frequency, filter(lambda x: 'matrix_A' in x.name, net.get_parameters()), @@ -137,8 +131,9 @@ if __name__ == '__main__': filter(lambda x: 'spatial_norm' in x.name, net.get_parameters()), config.weight_decay, config.loss_scale) - model = Model(net, loss_fn=loss, optimizer=opt, amp_level='O2', loss_scale_manager=loss_scale, keep_batchnorm_fp32=False, metrics={'acc'}, frequency=config.frequency) - + model = Model(net, loss_fn=loss, optimizer=opt, amp_level='O2', loss_scale_manager=loss_scale, + keep_batchnorm_fp32=False, metrics={'acc'}, frequency=config.frequency) + time_cb = TimeMonitor(data_size=step_size) loss_cb = LossMonitor() cb = [time_cb, loss_cb] From b64c848ba78b41aab4d27aaca6d9f64e52f8f1d7 Mon Sep 17 00:00:00 2001 From: z00478463 Date: Fri, 22 May 2020 10:53:51 +0800 Subject: [PATCH 04/11] for pylint --- .../config_imagenet.py | 0 .../crossentropy.py | 3 +- .../cus_ops/batch_matmul_impl.py | 3 +- .../cus_ops/cholesky_trsm.py | 3 +- .../cus_ops/cus_batch_matmul.py | 6 +- .../cus_ops/cus_cholesky_trsm.py | 12 ++-- .../cus_ops/cus_fused_abs_max1.py | 8 ++- .../cus_ops/cus_img2col.py | 3 +- .../cus_ops/cus_matmul_cube.py | 6 +- .../cus_ops/cus_matmul_cube_dense_left.py | 1 + .../cus_matmul_cube_fracz_right_mul.py | 1 + .../cus_ops/cus_matrix_combine.py | 1 + .../cus_ops/cus_transpose02314.py | 0 .../cus_ops/fused_abs_max1.py | 3 +- .../cus_ops/img2col_impl.py | 3 +- .../cus_ops/matmul_cube_dense_left.py | 7 ++- .../matmul_cube_fracz_left_cast_impl.py | 9 +-- .../matmul_cube_fracz_right_mul_impl.py | 8 +-- .../cus_ops/matmul_cube_impl.py | 9 +-- .../cus_ops/matrix_combine_impl.py | 3 +- .../cus_ops/transpose02314_impl.py | 3 +- .../dataset_imagenet.py | 0 .../lr_generator.py | 3 + .../model/dataset_helper.py | 8 +-- .../model/grad_reducer_thor.py | 1 + .../model/model_thor.py | 61 +++++++++---------- .../model/resnet.py | 4 +- .../model/thor.py | 5 +- .../model/thor_layer.py | 46 +++++++------- .../run_distribute_train_new.sh | 0 .../train.py | 12 ++-- 31 files changed, 133 insertions(+), 99 deletions(-) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/config_imagenet.py (100%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/crossentropy.py (97%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/batch_matmul_impl.py (97%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/cholesky_trsm.py (97%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/cus_batch_matmul.py (96%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/cus_cholesky_trsm.py (85%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/cus_fused_abs_max1.py (92%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/cus_img2col.py (99%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/cus_matmul_cube.py (95%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/cus_matmul_cube_dense_left.py (97%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/cus_matmul_cube_fracz_right_mul.py (97%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/cus_matrix_combine.py (98%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/cus_transpose02314.py (100%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/fused_abs_max1.py (97%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/img2col_impl.py (97%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/matmul_cube_dense_left.py (98%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/matmul_cube_fracz_left_cast_impl.py (98%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/matmul_cube_fracz_right_mul_impl.py (99%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/matmul_cube_impl.py (99%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/matrix_combine_impl.py (97%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/cus_ops/transpose02314_impl.py (97%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/dataset_imagenet.py (100%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/lr_generator.py (98%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/model/dataset_helper.py (99%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/model/grad_reducer_thor.py (99%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/model/model_thor.py (99%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/model/resnet.py (99%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/model/thor.py (99%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/model/thor_layer.py (94%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/run_distribute_train_new.sh (100%) rename example/{resnet101_imagenet2012_THOR => resnet50_imagenet2012_THOR}/train.py (96%) diff --git a/example/resnet101_imagenet2012_THOR/config_imagenet.py b/example/resnet50_imagenet2012_THOR/config_imagenet.py similarity index 100% rename from example/resnet101_imagenet2012_THOR/config_imagenet.py rename to example/resnet50_imagenet2012_THOR/config_imagenet.py diff --git a/example/resnet101_imagenet2012_THOR/crossentropy.py b/example/resnet50_imagenet2012_THOR/crossentropy.py similarity index 97% rename from example/resnet101_imagenet2012_THOR/crossentropy.py rename to example/resnet50_imagenet2012_THOR/crossentropy.py index 24ae022b58f..e8681ff4972 100644 --- a/example/resnet101_imagenet2012_THOR/crossentropy.py +++ b/example/resnet50_imagenet2012_THOR/crossentropy.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - +"""CrossEntropy""" import mindspore.nn as nn from mindspore import Tensor from mindspore.common import dtype as mstype @@ -22,6 +22,7 @@ from mindspore.ops import operations as P class CrossEntropy(_Loss): + """CrossEntropy""" def __init__(self, smooth_factor=0., num_classes=1000): super(CrossEntropy, self).__init__() self.onehot = P.OneHot() diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/batch_matmul_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py similarity index 97% rename from example/resnet101_imagenet2012_THOR/cus_ops/batch_matmul_impl.py rename to example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py index b75f3b49cab..24d7120bd9d 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/batch_matmul_impl.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py @@ -13,7 +13,7 @@ # limitations under the License. # ============================================================================ from mindspore.ops.op_info_register import op_info_register - +"""batch_matmul_impl""" @op_info_register("""{ "op_name": "CusBatchMatMul", @@ -71,4 +71,5 @@ from mindspore.ops.op_info_register import op_info_register ] }""") def CusBatchMatMul(input_x1, input_x2, output, transpose_a=False, transpose_b=True, kernel_name="batchmatmul"): +"""CusBatchMatMul""" return diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/cholesky_trsm.py b/example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py similarity index 97% rename from example/resnet101_imagenet2012_THOR/cus_ops/cholesky_trsm.py rename to example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py index f18d513f17b..0e6be53d3e4 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/cholesky_trsm.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py @@ -13,7 +13,7 @@ # limitations under the License. # ============================================================================ from mindspore.ops.op_info_register import op_info_register - +"""CusCholeskyTrsm""" @op_info_register("""{ "op_name": "CusCholeskyTrsm", @@ -59,4 +59,5 @@ from mindspore.ops.op_info_register import op_info_register ] }""") def CusCholeskyTrsm(input_x, output, kernel_name): +"""CusCholeskyTrsm""" return diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/cus_batch_matmul.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_batch_matmul.py similarity index 96% rename from example/resnet101_imagenet2012_THOR/cus_ops/cus_batch_matmul.py rename to example/resnet50_imagenet2012_THOR/cus_ops/cus_batch_matmul.py index 2379181ac38..7067458d662 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/cus_batch_matmul.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cus_batch_matmul.py @@ -12,11 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ +"""CusBatchMatMul""" from mindspore.ops import prim_attr_register, PrimitiveWithInfer from mindspore.ops.composite import multitype_ops as C - -# y = x^2 class CusBatchMatMul(PrimitiveWithInfer): """CusMatMulCube definition""" @@ -35,4 +34,5 @@ class CusBatchMatMul(PrimitiveWithInfer): return data1_shape def infer_dtype(self, data1_dtype, data2_dtype): - return data1_dtype \ No newline at end of file + return data1_dtype + diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py similarity index 85% rename from example/resnet101_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py rename to example/resnet50_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py index 4328f0f7479..fe7aaabf25d 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ +"""CusCholeskyTrsm""" from mindspore.ops import prim_attr_register, PrimitiveWithInfer @@ -24,11 +25,14 @@ class CusCholeskyTrsm(PrimitiveWithInfer): self.init_prim_io_names(inputs=['x1'], outputs=['y']) def infer_shape(self, data1_shape): - m, n = data1_shape + ll = [] + m, _ = data1_shape if m >= 128: - return [m // 128, 128, 128] + ll = [m // 128, 128, 128] else: - return [1, 64, 64] + ll = [1, 64, 64] + return ll def infer_dtype(self, data1_dtype): - return data1_dtype \ No newline at end of file + return data1_dtype + diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py similarity index 92% rename from example/resnet101_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py rename to example/resnet50_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py index 1bfebedde8f..5e91ba42249 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ +"""CusFusedAbsMax1""" from mindspore.ops import prim_attr_register, PrimitiveWithInfer from mindspore.ops.composite import multitype_ops as C @@ -32,10 +33,13 @@ class CusFusedAbsMax1(PrimitiveWithInfer): return bprop def infer_shape(self, data1_shape): + ll = [] if len(data1_shape) == 2: - return [1, ] + ll = [1,] else: - return [32, 64] + ll = [32, 64] + return ll def infer_dtype(self, data1_dtype): return data1_dtype + diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/cus_img2col.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_img2col.py similarity index 99% rename from example/resnet101_imagenet2012_THOR/cus_ops/cus_img2col.py rename to example/resnet50_imagenet2012_THOR/cus_ops/cus_img2col.py index ce363400ba3..96f943e471a 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/cus_img2col.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cus_img2col.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - +"""CusImg2Col""" from mindspore.ops import prim_attr_register, PrimitiveWithInfer from mindspore.ops.composite import multitype_ops as C @@ -49,3 +49,4 @@ class CusImg2Col(PrimitiveWithInfer): def infer_dtype(self, data1_dtype): return data1_dtype + diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube.py similarity index 95% rename from example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube.py rename to example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube.py index 71933737244..7674d1ab0c0 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ +"""CusMatMulCube""" import mindspore as ms from mindspore.ops import prim_attr_register, PrimitiveWithInfer from mindspore.ops.composite import multitype_ops as C @@ -37,11 +38,11 @@ class CusMatMulCube(PrimitiveWithInfer): def infer_shape(self, data1_shape, data2_shape): # shape = [1, data1_shape[1], data2_shape[2], 16, 16] # return shape - if self.transpose_a == True: + if self.transpose_a: k1, m = data1_shape else: m, k1 = data1_shape - if self.transpose_b == True: + if self.transpose_b: n, k2 = data2_shape else: k2, n = data2_shape @@ -51,3 +52,4 @@ class CusMatMulCube(PrimitiveWithInfer): def infer_dtype(self, data1_dtype, data2_dtype): return ms.common.dtype.tensor_type(getattr(ms, "float32")) + diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube_dense_left.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube_dense_left.py similarity index 97% rename from example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube_dense_left.py rename to example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube_dense_left.py index 603d8487cef..4228e3938a0 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube_dense_left.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube_dense_left.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ +"""CusMatMulCubeDenseLeft""" import mindspore as ms from mindspore.ops import prim_attr_register, PrimitiveWithInfer from mindspore.ops.composite import multitype_ops as C diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube_fracz_right_mul.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube_fracz_right_mul.py similarity index 97% rename from example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube_fracz_right_mul.py rename to example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube_fracz_right_mul.py index f9b68a76551..a80a52797e4 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/cus_matmul_cube_fracz_right_mul.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube_fracz_right_mul.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ +"""CusMatMulCubeFraczRightMul""" import mindspore as ms from mindspore.ops import prim_attr_register, PrimitiveWithInfer from mindspore.ops.composite import multitype_ops as C diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/cus_matrix_combine.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_matrix_combine.py similarity index 98% rename from example/resnet101_imagenet2012_THOR/cus_ops/cus_matrix_combine.py rename to example/resnet50_imagenet2012_THOR/cus_ops/cus_matrix_combine.py index 98add9f542d..373f149dd08 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/cus_matrix_combine.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cus_matrix_combine.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ +"""CusMatrixCombine""" from mindspore.ops import prim_attr_register, PrimitiveWithInfer from mindspore.ops.composite import multitype_ops as C diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/cus_transpose02314.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_transpose02314.py similarity index 100% rename from example/resnet101_imagenet2012_THOR/cus_ops/cus_transpose02314.py rename to example/resnet50_imagenet2012_THOR/cus_ops/cus_transpose02314.py diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/fused_abs_max1.py b/example/resnet50_imagenet2012_THOR/cus_ops/fused_abs_max1.py similarity index 97% rename from example/resnet101_imagenet2012_THOR/cus_ops/fused_abs_max1.py rename to example/resnet50_imagenet2012_THOR/cus_ops/fused_abs_max1.py index 146533131ea..b9a0d452738 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/fused_abs_max1.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/fused_abs_max1.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - +"""CusFusedAbsMax1""" from mindspore.ops.op_info_register import op_info_register @@ -65,4 +65,5 @@ from mindspore.ops.op_info_register import op_info_register ] }""") def CusFusedAbsMax1(input_x, output, origin_shape=None, kernel_name="fused_abs_max1"): + """CusFusedAbsMax1""" return diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/img2col_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/img2col_impl.py similarity index 97% rename from example/resnet101_imagenet2012_THOR/cus_ops/img2col_impl.py rename to example/resnet50_imagenet2012_THOR/cus_ops/img2col_impl.py index 67081823e80..5137d4d7e70 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/img2col_impl.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/img2col_impl.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - +"""CusImg2ColNC1HWC0""" from mindspore.ops.op_info_register import op_info_register @@ -83,4 +83,5 @@ from mindspore.ops.op_info_register import op_info_register ] }""") def CusImg2ColNC1HWC0(input_x, output, ksizes, strides, dilates, padding, kernel_name="img2col"): + """CusImg2ColNC1HWC0""" return diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_dense_left.py b/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_dense_left.py similarity index 98% rename from example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_dense_left.py rename to example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_dense_left.py index 7c4b8c47f3c..91c5336aa0c 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_dense_left.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_dense_left.py @@ -2,11 +2,11 @@ # -*- coding:utf-8 -*- """ copyright 2020 Huawei Technologies Co., Ltd - + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software @@ -14,7 +14,7 @@ distributed under the License == distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - + matmul """ from __future__ import absolute_import @@ -98,4 +98,5 @@ NoneType = type(None) @util.check_input_type(dict, dict, (dict, NoneType), dict, bool, bool, str) def CusMatMulCubeDenseLeft(input_x1, input_x2, bias=None, output_y={}, trans_a=False, trans_b=False, kernel_name="matmulcube"): + """CusMatMulCubeDenseLeft""" return diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_fracz_left_cast_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_left_cast_impl.py similarity index 98% rename from example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_fracz_left_cast_impl.py rename to example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_left_cast_impl.py index af0f6dac706..de2cca10779 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_fracz_left_cast_impl.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_left_cast_impl.py @@ -2,19 +2,19 @@ # -*- coding:utf-8 -*- """ copyright 2020 Huawei Technologies Co., Ltd - + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License == distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - + matmul """ from __future__ import absolute_import @@ -99,4 +99,5 @@ NoneType = type(None) @util.check_input_type(dict, dict, (dict, NoneType), dict, bool, bool, str) def CusMatMulCubeFraczLeftCast(input_x1, input_x2, bias=None, output_y={}, trans_a=False, trans_b=False, kernel_name="CusMatMulCubeFraczLeftCast"): + """CusMatMulCubeFraczLeftCast""" return diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py similarity index 99% rename from example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py rename to example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py index a6afa80b24d..2923afe2bd1 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py @@ -2,19 +2,19 @@ # -*- coding:utf-8 -*- """ copyright 2020 Huawei Technologies Co., Ltd - + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License == distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - + matmul """ from __future__ import absolute_import diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_impl.py similarity index 99% rename from example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_impl.py rename to example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_impl.py index 0b6d91ef79e..7c2d81e1d67 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/matmul_cube_impl.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_impl.py @@ -2,19 +2,19 @@ # -*- coding:utf-8 -*- """ copyright 2020 Huawei Technologies Co., Ltd - + Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License == distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. - + matmul """ from __future__ import absolute_import @@ -110,4 +110,5 @@ NoneType = type(None) # pylint: disable=locally-disabled,too-many-arguments, too-many-locals, too-many-statements @util.check_input_type(dict, dict, (dict, NoneType), dict, bool, bool, str) def CusMatMulCube(input_x1, input_x2, bias=None, output_y={}, trans_a=False, trans_b=False, kernel_name="matmulcube"): + """CusMatMulCube""" return diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/matrix_combine_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/matrix_combine_impl.py similarity index 97% rename from example/resnet101_imagenet2012_THOR/cus_ops/matrix_combine_impl.py rename to example/resnet50_imagenet2012_THOR/cus_ops/matrix_combine_impl.py index f300f0b1a96..32045e7ccbd 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/matrix_combine_impl.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/matrix_combine_impl.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - +"""CusMatrixCombine""" from mindspore.ops.op_info_register import op_info_register @@ -59,4 +59,5 @@ from mindspore.ops.op_info_register import op_info_register ] }""") def CusMatrixCombine(input_x, output, kernel_name="matrix_combine"): + """CusMatrixCombine""" return diff --git a/example/resnet101_imagenet2012_THOR/cus_ops/transpose02314_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/transpose02314_impl.py similarity index 97% rename from example/resnet101_imagenet2012_THOR/cus_ops/transpose02314_impl.py rename to example/resnet50_imagenet2012_THOR/cus_ops/transpose02314_impl.py index fba24f082c8..c5aebe523d5 100644 --- a/example/resnet101_imagenet2012_THOR/cus_ops/transpose02314_impl.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/transpose02314_impl.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - +"""CusTranspose02314""" from mindspore.ops.op_info_register import op_info_register @@ -59,4 +59,5 @@ from mindspore.ops.op_info_register import op_info_register ] }""") def CusTranspose02314(input_x, output, kernel_name="transpose021354"): + """CusTranspose02314""" return diff --git a/example/resnet101_imagenet2012_THOR/dataset_imagenet.py b/example/resnet50_imagenet2012_THOR/dataset_imagenet.py similarity index 100% rename from example/resnet101_imagenet2012_THOR/dataset_imagenet.py rename to example/resnet50_imagenet2012_THOR/dataset_imagenet.py diff --git a/example/resnet101_imagenet2012_THOR/lr_generator.py b/example/resnet50_imagenet2012_THOR/lr_generator.py similarity index 98% rename from example/resnet101_imagenet2012_THOR/lr_generator.py rename to example/resnet50_imagenet2012_THOR/lr_generator.py index ca290fd1585..a447daedd32 100644 --- a/example/resnet101_imagenet2012_THOR/lr_generator.py +++ b/example/resnet50_imagenet2012_THOR/lr_generator.py @@ -19,12 +19,14 @@ import numpy as np def linear_warmup_lr(current_step, warmup_steps, base_lr, init_lr): + """linear_warmup_lr""" lr_inc = (float(base_lr) - float(init_lr)) / float(warmup_steps) lr = float(init_lr) + lr_inc * current_step return lr def cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0, num_periods=0.5): + """linear_warmup_lr""" base_lr = lr warmup_init_lr = 0 total_steps = int(max_epoch * steps_per_epoch) @@ -44,6 +46,7 @@ def cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, et def warmup_cosine_annealing_lr(lr, steps_per_epoch, warmup_epochs, max_epoch, T_max, eta_min=0, num_periods=0.5): + """warmup_cosine_annealing_lr""" base_lr = lr warmup_init_lr = 0 total_steps = int(max_epoch * steps_per_epoch * 0.99) diff --git a/example/resnet101_imagenet2012_THOR/model/dataset_helper.py b/example/resnet50_imagenet2012_THOR/model/dataset_helper.py similarity index 99% rename from example/resnet101_imagenet2012_THOR/model/dataset_helper.py rename to example/resnet50_imagenet2012_THOR/model/dataset_helper.py index 83e7f4e1a2c..b8efd9f682e 100644 --- a/example/resnet101_imagenet2012_THOR/model/dataset_helper.py +++ b/example/resnet50_imagenet2012_THOR/model/dataset_helper.py @@ -25,17 +25,17 @@ from mindspore.train.parallel_utils import ParallelMode class DatasetHelper: """ Help function to use the Minddata dataset. - + According to different context, change the iter of dataset, to use the same for loop in different context. - + Note: The iter of DatasetHelper will give one epoch data. - + Args: dataset (DataSet): The dataset. dataset_sink_mode (bool): If true use GetNext to fetch the data, or else feed the data from host. Default: True. - + Examples: >>> dataset_helper = DatasetHelper(dataset) >>> for inputs in dataset_helper: diff --git a/example/resnet101_imagenet2012_THOR/model/grad_reducer_thor.py b/example/resnet50_imagenet2012_THOR/model/grad_reducer_thor.py similarity index 99% rename from example/resnet101_imagenet2012_THOR/model/grad_reducer_thor.py rename to example/resnet50_imagenet2012_THOR/model/grad_reducer_thor.py index 4714aad17f9..ad8d8dd8e4c 100644 --- a/example/resnet101_imagenet2012_THOR/model/grad_reducer_thor.py +++ b/example/resnet50_imagenet2012_THOR/model/grad_reducer_thor.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ +"""grad_reducer_thor""" import mindspore.common.dtype as mstype from mindspore.communication.management import GlobalComm, get_group_size from mindspore.nn.cell import Cell diff --git a/example/resnet101_imagenet2012_THOR/model/model_thor.py b/example/resnet50_imagenet2012_THOR/model/model_thor.py similarity index 99% rename from example/resnet101_imagenet2012_THOR/model/model_thor.py rename to example/resnet50_imagenet2012_THOR/model/model_thor.py index 1527e28812d..2b33c8602df 100644 --- a/example/resnet101_imagenet2012_THOR/model/model_thor.py +++ b/example/resnet50_imagenet2012_THOR/model/model_thor.py @@ -13,8 +13,6 @@ # limitations under the License. # ============================================================================ """Model.""" -import mindspore.nn as nn -import numpy as np from mindspore import context from mindspore import log as logger from mindspore._c_expression import init_exec_dataset @@ -30,16 +28,17 @@ from mindspore.parallel._utils import _get_parallel_mode, _get_device_num, _get_ from mindspore.train import amp from mindspore.train.callback import _InternalCallbackParam, RunContext, _build_callbacks from mindspore.train.parallel_utils import ParallelMode +import mindspore.nn as nn from second_order.dataset_helper import DatasetHelper - +import numpy as np def _convert_type(types): """ Convert from numpy type to tensor type. - + Args: types (list): Numpy type list of element in dataset. - + Returns: list, list of element in dataset. """ @@ -76,9 +75,9 @@ def _exec_datagraph(exec_dataset, dataset_size, phase='dataset'): class Model: """ High-Level API for Training or Testing. - + `Model` groups layers into an object with training and inference features. - + Args: network (Cell): The training or testing network. loss_fn (Cell): Objective function, if loss_fn is None, the @@ -96,15 +95,15 @@ class Model: metric. Default: None. amp_level (str): Option for argument `level` in `mindspore.amp.build_train_network`, level for mixed precision training. Supports [O0, O2]. Default: "O0". - + - O0: Do not change. - O2: Cast network to float16, keep batchnorm run in float32, using dynamic loss scale. - + loss_scale_manager (Union[None, LossScaleManager]): If None, not scale the loss, or else scale the loss by LossScaleManager. If it is set, overwrite the level setting. It's a eyword argument. e.g. Use `loss_scale_manager=None` to set the value. keep_batchnorm_fp32 (bool): Keep Batchnorm run in `float32`. If set, overwrite the level setting. Default: True. - + Examples: >>> class Net(nn.Cell): >>> def __init__(self): @@ -250,7 +249,7 @@ class Model: def _train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=True): """ Training. - + Args: epoch (int): Total number of iterations on the data. train_dataset (Dataset): A training dataset iterator. If there is no @@ -296,7 +295,7 @@ class Model: def _train_dataset_sink_process(self, epoch, train_dataset, list_callback=None, cb_params=None): """ Training process. The data would be passed to network through dataset channel. - + Args: epoch (int): Total number of iterations on the data. train_dataset (Dataset): A training dataset iterator. If there is no @@ -366,7 +365,7 @@ class Model: def _train_process(self, epoch, train_dataset, list_callback=None, cb_params=None): """ Training process. The data would be passed to network directly. - + Args: epoch (int): Total number of iterations on the data. train_dataset (Dataset): A training dataset iterator. If there is no @@ -426,9 +425,9 @@ class Model: def train(self, epoch, train_dataset, callbacks=None, dataset_sink_mode=True): """ Training API where the iteration is controlled by python front-end. - + When setting pynative mode, the training process will be performed with dataset not sink. - + Note: CPU is not supported when dataset_sink_mode is true. If dataset_sink_mode is True, epoch of training should be equal to the count of repeat @@ -436,7 +435,7 @@ class Model: is not the amount training requires. If dataset_sink_mode is True, data will be sent to device. If device is Ascend, features of data will be transferred one by one. The limitation of data transmission per time is 256M. - + Args: epoch (int): Total number of iterations on the data. train_dataset (Dataset): A training dataset iterator. If there is no @@ -448,8 +447,8 @@ class Model: dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. Configure pynative mode, the training process will be performed with dataset not sink. - - + + Examples: >>> dataset = get_dataset() >>> net = Net() @@ -477,12 +476,12 @@ class Model: def _eval_dataset_sink_process(self, valid_dataset, list_callback=None, cb_params=None): """ Evaluation. The data would be passed to network through dataset channel. - + Args: valid_dataset (Dataset): Dataset to evaluate the model. list_callback (ListCallback): Executor of callback list. Default: None. cb_params (_InternalCallbackParam): Callback parameters. Default: None. - + Returns: Dict, returns the loss value & metrics values for the model in test mode. """ @@ -526,7 +525,7 @@ class Model: def _eval_process(self, valid_dataset, list_callback=None, cb_params=None): """ Evaluation. The data would be passed to network directly. - + Args: valid_dataset (Dataset): Dataset to evaluate the model. list_callback (ListCallback): Executor of callback list. Default: None. @@ -555,23 +554,23 @@ class Model: def eval(self, valid_dataset, callbacks=None, dataset_sink_mode=True): """ Evaluation API where the iteration is controlled by python front-end. - + Configure to pynative mode, the evaluation will be performed with dataset non-sink mode. - + Note: CPU is not supported when dataset_sink_mode is true. If dataset_sink_mode is True, data will be sent to device. If device is Ascend, features of data will be transferred one by one. The limitation of data transmission per time is 256M. - + Args: valid_dataset (Dataset): Dataset to evaluate the model. callbacks (list): List of callback object. Callbacks which should be excuted while training. Default: None. dataset_sink_mode (bool): Determines whether to pass the data through dataset channel. Default: True. - + Returns: Dict, returns the loss value & metrics values for the model in test mode. - + Examples: >>> dataset = get_dataset() >>> net = Net() @@ -603,18 +602,18 @@ class Model: def predict(self, *predict_data): """ Generates output predictions for the input samples. - + Data could be single tensor, or list of tensor, tuple of tensor. - + Note: Batch data should be put together in one tensor. - + Args: predict_data (Tensor): Tensor of predict data. can be array, list or tuple. - + Returns: Tensor, array(s) of predictions. - + Examples: >>> input_data = Tensor(np.random.randint(0, 255, [1, 3, 224, 224]), mindspore.float32) >>> model = Model(Net()) diff --git a/example/resnet101_imagenet2012_THOR/model/resnet.py b/example/resnet50_imagenet2012_THOR/model/resnet.py similarity index 99% rename from example/resnet101_imagenet2012_THOR/model/resnet.py rename to example/resnet50_imagenet2012_THOR/model/resnet.py index a6ba0b5c580..fb053413489 100644 --- a/example/resnet101_imagenet2012_THOR/model/resnet.py +++ b/example/resnet50_imagenet2012_THOR/model/resnet.py @@ -126,8 +126,8 @@ def _bn_last(channel): def _fc(in_channel, out_channel, damping, loss_scale, frequency): weight_shape = (out_channel, in_channel) weight = Tensor(kaiming_uniform(weight_shape, a=math.sqrt(5)) - return Dense_Thor(in_channel, out_channel, has_bias=False, weight_init=weight, bias_init=0, - damping=damping, loss_scale=loss_scale, frequency=frequency) + return Dense_Thor(in_channel, out_channel, has_bias=False, weight_init=weight, + bias_init=0, damping=damping, loss_scale=loss_scale, frequency=frequency) class ResidualBlock(nn.Cell): diff --git a/example/resnet101_imagenet2012_THOR/model/thor.py b/example/resnet50_imagenet2012_THOR/model/thor.py similarity index 99% rename from example/resnet101_imagenet2012_THOR/model/thor.py rename to example/resnet50_imagenet2012_THOR/model/thor.py index 82e4e3b1108..44c0fd45dba 100644 --- a/example/resnet101_imagenet2012_THOR/model/thor.py +++ b/example/resnet50_imagenet2012_THOR/model/thor.py @@ -14,8 +14,6 @@ # ============================================================================ """momentum""" import mindspore.common.dtype as mstype -from cus_ops.cus_matmul_cube_dense_right import CusMatMulCubeDenseRight -from cus_ops.cus_matmul_cube_fracz_left_cast import CusMatMulCubeFraczLeftCast from mindspore.common.initializer import initializer from mindspore.common.parameter import Parameter from mindspore.common.parameter import ParameterTuple @@ -24,6 +22,8 @@ from mindspore.nn.optim.optimizer import Optimizer from mindspore.ops import functional as F, composite as C, operations as P from mindspore.parallel._utils import _get_device_num, _get_mirror_mean +from cus_ops.cus_matmul_cube_dense_right import CusMatMulCubeDenseRight +from cus_ops.cus_matmul_cube_fracz_left_cast import CusMatMulCubeFraczLeftCast from cus_ops.cus_matmul_cube_dense_left import CusMatMulCubeDenseLeft from cus_ops.cus_matmul_cube_fracz_right_mul import CusMatMulCubeFraczRightMul from model.grad_reducer_thor import DistributedGradReducerThor @@ -52,6 +52,7 @@ def _tensor_apply_decay(weight_decay, if_apply, weight, gradient): class THOR(Optimizer): + """THOR""" def __init__(self, params, learning_rate, momentum, matrix_A, matrix_G, A_inv_max, G_inv_max, weight_decay=0.0, loss_scale=1.0, decay_filter=lambda x: x.name not in []): diff --git a/example/resnet101_imagenet2012_THOR/model/thor_layer.py b/example/resnet50_imagenet2012_THOR/model/thor_layer.py similarity index 94% rename from example/resnet101_imagenet2012_THOR/model/thor_layer.py rename to example/resnet50_imagenet2012_THOR/model/thor_layer.py index 7f777aa3360..8add3d74fa9 100644 --- a/example/resnet101_imagenet2012_THOR/model/thor_layer.py +++ b/example/resnet50_imagenet2012_THOR/model/thor_layer.py @@ -12,10 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - +"""thor_layer""" import mindspore as ms import mindspore.common.dtype as mstype -import numpy as np from mindspore._checkparam import check_bool, twice, check_int_positive from mindspore._extends import cell_attr_register from mindspore.common.initializer import initializer @@ -33,6 +32,7 @@ from cus_ops.cus_matmul_cube import CusMatMulCube from cus_ops.cus_matrix_combine import CusMatrixCombine from cus_ops.cus_transpose02314 import CusTranspose02314 +import numpy as np C0 = 16 @@ -91,8 +91,7 @@ class _Conv(Cell): 'attr \'group\' of \'Conv2D\' Op.') self.weight = Parameter(initializer( - weight_init, [out_channels, in_channels // group, *kernel_size]), - name='weight') + weight_init, [out_channels, in_channels // group, *kernel_size]), name='weight') if check_bool(has_bias): self.bias = Parameter(_initializer( @@ -107,6 +106,7 @@ class _Conv(Cell): class Conv2d_Thor(_Conv): + """Conv2d_Thor""" def __init__(self, in_channels, out_channels, @@ -180,7 +180,7 @@ class Conv2d_Thor(_Conv): self.G_inv_max = Parameter(initializer(0, [1], mstype.float32), name="G_inv_max", requires_grad=False) self.fake_G = Tensor( np.reshape(np.identity(self.matrix_G_device_dim).astype(np.float16), self.matrix_G_device_shape)) - self.fake_G_inv_max = Tensor(np.zeros([1, ]).astype(np.float32)) + self.fake_G_inv_max = Tensor(np.zeros([1,]).astype(np.float32)) self.shape = P.Shape() self.reshape = P.Reshape() @@ -196,8 +196,8 @@ class Conv2d_Thor(_Conv): self.channels_slice_flag = True self.padA_flag = False - if ( - self.matrix_A_dim // self.diag_block_dim) * self.diag_block_dim != self.matrix_A_dim and self.matrix_A_dim > self.diag_block_dim: + if (self.matrix_A_dim // self.diag_block_dim) * self.diag_block_dim != self.matrix_A_dim + and self.matrix_A_dim > self.diag_block_dim: self.padA_flag = True pad_dim = self.diag_block_dim - self.matrix_A_dim % self.diag_block_dim self.padA = P.Pad(((0, pad_dim), (0, pad_dim))) @@ -228,6 +228,7 @@ class Conv2d_Thor(_Conv): self.getG = P.InsertGradientOf(self.save_gradient) def save_gradient(self, dout): + """save_gradient""" out = dout dout = self.mul(dout, self.loss_scale) dout = self.mul(dout, 32.0) @@ -252,7 +253,6 @@ class Conv2d_Thor(_Conv): matrix_G_inv_max = self.fused_abs_max2(matrix_G_inv_max) self.G_inv_max = matrix_G_inv_max matrix_G_inv = self.matrix_combine(matrix_G_inv) - matrix_G_inv_shape = self.shape(matrix_G_inv) matrix_G_inv = self.reshape(matrix_G_inv, self.matrix_G_device_temp_shape) matrix_G_inv = self.transpose(matrix_G_inv, (2, 0, 1, 3)) matrix_G = self.cast(matrix_G_inv, mstype.float16) @@ -287,7 +287,6 @@ class Conv2d_Thor(_Conv): self.A_inv_max = matrix_A_inv_max matrix_A_inv = self.matrix_combine(matrix_A_inv) matrix_A_inv = self.cast(matrix_A_inv, mstype.float16) - in_channels = self.in_channels if self.padA_flag: matrix_A_inv = self.slice(matrix_A_inv, (0, 0), (self.matrix_A_dim, self.matrix_A_dim)) @@ -307,22 +306,23 @@ class Conv2d_Thor(_Conv): return out def extra_repr(self): + """extra_repr""" s = 'input_channels={}, output_channels={}, kernel_size={},' \ 'stride={}, pad_mode={}, padding={}, dilation={}, ' \ 'group={}, data_format={}, has_bias={},' \ 'weight_init={}, bias_init={}'.format( - self.in_channels, - self.out_channels, - self.kernel_size, - self.stride, - self.pad_mode, - self.padding, - self.dilation, - self.group, - self.data_format, - self.has_bias, - self.weight, - self.bias) + self.in_channels, + self.out_channels, + self.kernel_size, + self.stride, + self.pad_mode, + self.padding, + self.dilation, + self.group, + self.data_format, + self.has_bias, + self.weight, + self.bias) if self.has_bias: s += ', bias={}'.format(self.bias) @@ -330,6 +330,7 @@ class Conv2d_Thor(_Conv): class Dense_Thor(Cell): + """Dense_Thor""" @cell_attr_register(attrs=['has_bias', 'activation']) def __init__(self, in_channels, @@ -405,6 +406,7 @@ class Dense_Thor(Cell): self.getG = P.InsertGradientOf(self.save_gradient) def save_gradient(self, dout): + """save_gradient""" out = dout dout = self.mul(dout, self.loss_scale) dout = self.mul(dout, 32.0) @@ -435,6 +437,7 @@ class Dense_Thor(Cell): return out def construct(self, x): + """construct""" if self.thor: inputs = self.cube_matmul(x, x) normalizer = 32 @@ -472,6 +475,7 @@ class Dense_Thor(Cell): return output def extend_repr(self): + """extend_repr""" str_info = 'in_channels={}, out_channels={}, weight={}, has_bias={}' \ .format(self.in_channels, self.out_channels, self.weight, self.has_bias) if self.has_bias: diff --git a/example/resnet101_imagenet2012_THOR/run_distribute_train_new.sh b/example/resnet50_imagenet2012_THOR/run_distribute_train_new.sh similarity index 100% rename from example/resnet101_imagenet2012_THOR/run_distribute_train_new.sh rename to example/resnet50_imagenet2012_THOR/run_distribute_train_new.sh diff --git a/example/resnet101_imagenet2012_THOR/train.py b/example/resnet50_imagenet2012_THOR/train.py similarity index 96% rename from example/resnet101_imagenet2012_THOR/train.py rename to example/resnet50_imagenet2012_THOR/train.py index 978d0c0dab1..3692f7b8666 100644 --- a/example/resnet101_imagenet2012_THOR/train.py +++ b/example/resnet50_imagenet2012_THOR/train.py @@ -18,7 +18,6 @@ import os import random import mindspore.dataset.engine as de -import numpy as np from mindspore import Tensor from mindspore import context from mindspore.communication.management import init @@ -30,6 +29,7 @@ from second_order.model_second_order import Model from second_order.resnet import resnet50 from second_order.thor import THOR +import numpy as np from config_imagenet import config from crossentropy import CrossEntropy from dataset_imagenet import create_dataset @@ -56,13 +56,14 @@ context.set_context(enable_mem_reuse=True) def get_second_order_lr(global_step, lr_init, decay, total_epochs, steps_per_epoch): + """get_second_order_lr""" lr_each_step = [] total_steps = steps_per_epoch * total_epochs for i in range(total_steps): epoch = (i + 1) / steps_per_epoch base = (1.0 - float(epoch) / total_epochs) ** decay - lr = lr_init * base - lr_each_step.append(lr) + lr_local = lr_init * base + lr_each_step.append(lr_local) current_step = global_step lr_each_step = np.array(lr_each_step).astype(np.float32) print("learning_rate_is=====", lr_each_step) @@ -71,12 +72,13 @@ def get_second_order_lr(global_step, lr_init, decay, total_epochs, steps_per_epo def get_second_order_damping(global_step, damping_init, decay_rate, total_epochs, steps_per_epoch): + """get_second_order_damping""" damping_each_step = [] total_steps = steps_per_epoch * total_epochs for step in range(total_steps): epoch = (step + 1) / steps_per_epoch - damping = damping_init * (decay_rate ** (epoch / 10)) - damping_each_step.append(damping) + damping_here = damping_init * (decay_rate ** (epoch / 10)) + damping_each_step.append(damping_here) current_step = global_step damping_each_step = np.array(damping_each_step).astype(np.float32) From d6bed5452eb400dbea2b3f3d0ca110085518cf96 Mon Sep 17 00:00:00 2001 From: z00478463 Date: Fri, 22 May 2020 14:59:03 +0800 Subject: [PATCH 05/11] for pylint 2nd --- example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py | 2 +- example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py | 2 +- example/resnet50_imagenet2012_THOR/cus_ops/cus_batch_matmul.py | 1 - example/resnet50_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py | 1 - .../resnet50_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py | 1 - example/resnet50_imagenet2012_THOR/cus_ops/cus_img2col.py | 1 - example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube.py | 1 - 7 files changed, 2 insertions(+), 7 deletions(-) diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py index 24d7120bd9d..bc4ac3a2ebd 100644 --- a/example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py @@ -71,5 +71,5 @@ from mindspore.ops.op_info_register import op_info_register ] }""") def CusBatchMatMul(input_x1, input_x2, output, transpose_a=False, transpose_b=True, kernel_name="batchmatmul"): -"""CusBatchMatMul""" + """CusBatchMatMul""" return diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py b/example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py index 0e6be53d3e4..1a2c4bfb9b2 100644 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py @@ -59,5 +59,5 @@ from mindspore.ops.op_info_register import op_info_register ] }""") def CusCholeskyTrsm(input_x, output, kernel_name): -"""CusCholeskyTrsm""" + """CusCholeskyTrsm""" return diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_batch_matmul.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_batch_matmul.py index 7067458d662..1e72ed9fac1 100644 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_batch_matmul.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cus_batch_matmul.py @@ -35,4 +35,3 @@ class CusBatchMatMul(PrimitiveWithInfer): def infer_dtype(self, data1_dtype, data2_dtype): return data1_dtype - diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py index fe7aaabf25d..d6ea0156223 100644 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py @@ -35,4 +35,3 @@ class CusCholeskyTrsm(PrimitiveWithInfer): def infer_dtype(self, data1_dtype): return data1_dtype - diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py index 5e91ba42249..c49c36ddd47 100644 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py @@ -42,4 +42,3 @@ class CusFusedAbsMax1(PrimitiveWithInfer): def infer_dtype(self, data1_dtype): return data1_dtype - diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_img2col.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_img2col.py index 96f943e471a..e1128291dde 100644 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_img2col.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cus_img2col.py @@ -49,4 +49,3 @@ class CusImg2Col(PrimitiveWithInfer): def infer_dtype(self, data1_dtype): return data1_dtype - diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube.py index 7674d1ab0c0..e41cb6bc98b 100644 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube.py @@ -52,4 +52,3 @@ class CusMatMulCube(PrimitiveWithInfer): def infer_dtype(self, data1_dtype, data2_dtype): return ms.common.dtype.tensor_type(getattr(ms, "float32")) - From 80b738b0da9adc3d8d6f5fec776a6465692b4ea9 Mon Sep 17 00:00:00 2001 From: z00478463 Date: Fri, 22 May 2020 15:17:15 +0800 Subject: [PATCH 06/11] for pylint 3rd --- .../resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py | 2 +- example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py | 2 +- .../cus_ops/matmul_cube_fracz_right_mul_impl.py | 1 + example/resnet50_imagenet2012_THOR/model/model_thor.py | 2 +- example/resnet50_imagenet2012_THOR/model/resnet.py | 2 +- example/resnet50_imagenet2012_THOR/model/thor_layer.py | 2 +- example/resnet50_imagenet2012_THOR/train.py | 6 +++--- 7 files changed, 9 insertions(+), 8 deletions(-) diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py index bc4ac3a2ebd..201d427308e 100644 --- a/example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -from mindspore.ops.op_info_register import op_info_register """batch_matmul_impl""" +from mindspore.ops.op_info_register import op_info_register @op_info_register("""{ "op_name": "CusBatchMatMul", diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py b/example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py index 1a2c4bfb9b2..8f09fab77da 100644 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ -from mindspore.ops.op_info_register import op_info_register """CusCholeskyTrsm""" +from mindspore.ops.op_info_register import op_info_register @op_info_register("""{ "op_name": "CusCholeskyTrsm", diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py index 2923afe2bd1..7fc2ba35d16 100644 --- a/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py @@ -109,4 +109,5 @@ NoneType = type(None) }""") def CusMatMulCubeFraczRightMul(input_x1, input_x2, input_x3, bias=None, output_y={}, trans_a=False, trans_b=False, kernel_name="matmulcube"): + """CusMatMulCubeFraczRightMul""" return diff --git a/example/resnet50_imagenet2012_THOR/model/model_thor.py b/example/resnet50_imagenet2012_THOR/model/model_thor.py index 2b33c8602df..613d15468ff 100644 --- a/example/resnet50_imagenet2012_THOR/model/model_thor.py +++ b/example/resnet50_imagenet2012_THOR/model/model_thor.py @@ -530,7 +530,7 @@ class Model: valid_dataset (Dataset): Dataset to evaluate the model. list_callback (ListCallback): Executor of callback list. Default: None. cb_params (_InternalCallbackParam): Callback parameters. Default: None. - + Returns: Dict, returns the loss value & metrics values for the model in test mode. """ diff --git a/example/resnet50_imagenet2012_THOR/model/resnet.py b/example/resnet50_imagenet2012_THOR/model/resnet.py index fb053413489..7a9b5a8922d 100644 --- a/example/resnet50_imagenet2012_THOR/model/resnet.py +++ b/example/resnet50_imagenet2012_THOR/model/resnet.py @@ -126,7 +126,7 @@ def _bn_last(channel): def _fc(in_channel, out_channel, damping, loss_scale, frequency): weight_shape = (out_channel, in_channel) weight = Tensor(kaiming_uniform(weight_shape, a=math.sqrt(5)) - return Dense_Thor(in_channel, out_channel, has_bias=False, weight_init=weight, + return Dense_Thor(in_channel, out_channel, has_bias=False, weight_init=weight, \ bias_init=0, damping=damping, loss_scale=loss_scale, frequency=frequency) diff --git a/example/resnet50_imagenet2012_THOR/model/thor_layer.py b/example/resnet50_imagenet2012_THOR/model/thor_layer.py index 8add3d74fa9..0850ed1dee6 100644 --- a/example/resnet50_imagenet2012_THOR/model/thor_layer.py +++ b/example/resnet50_imagenet2012_THOR/model/thor_layer.py @@ -196,7 +196,7 @@ class Conv2d_Thor(_Conv): self.channels_slice_flag = True self.padA_flag = False - if (self.matrix_A_dim // self.diag_block_dim) * self.diag_block_dim != self.matrix_A_dim + if (self.matrix_A_dim // self.diag_block_dim) * self.diag_block_dim != self.matrix_A_dim \ and self.matrix_A_dim > self.diag_block_dim: self.padA_flag = True pad_dim = self.diag_block_dim - self.matrix_A_dim % self.diag_block_dim diff --git a/example/resnet50_imagenet2012_THOR/train.py b/example/resnet50_imagenet2012_THOR/train.py index 3692f7b8666..fe192884c41 100644 --- a/example/resnet50_imagenet2012_THOR/train.py +++ b/example/resnet50_imagenet2012_THOR/train.py @@ -82,9 +82,9 @@ def get_second_order_damping(global_step, damping_init, decay_rate, total_epochs current_step = global_step damping_each_step = np.array(damping_each_step).astype(np.float32) - damping = damping_each_step[current_step:] - print("damping_is=========", damping) - return damping + damping_now = damping_each_step[current_step:] + print("damping_is=========", damping_now) + return damping_now if __name__ == '__main__': From 13321abf7e75fb9cae55062fb998a85c740c651b Mon Sep 17 00:00:00 2001 From: z00478463 Date: Fri, 22 May 2020 15:31:52 +0800 Subject: [PATCH 07/11] for pylint 4th --- .../cus_ops/cus_transpose02314.py | 2 +- .../model/thor_layer.py | 33 ++++++++++--------- example/resnet50_imagenet2012_THOR/train.py | 2 +- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_transpose02314.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_transpose02314.py index 944f97a74e2..b856cf7da78 100644 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_transpose02314.py +++ b/example/resnet50_imagenet2012_THOR/cus_ops/cus_transpose02314.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - +"""CusTranspose02314""" from mindspore.ops import prim_attr_register, PrimitiveWithInfer from mindspore.ops.composite import multitype_ops as C diff --git a/example/resnet50_imagenet2012_THOR/model/thor_layer.py b/example/resnet50_imagenet2012_THOR/model/thor_layer.py index 0850ed1dee6..8097d729ea1 100644 --- a/example/resnet50_imagenet2012_THOR/model/thor_layer.py +++ b/example/resnet50_imagenet2012_THOR/model/thor_layer.py @@ -37,13 +37,14 @@ C0 = 16 def caculate_device_shape(matrix_dim, channel, is_A): + ll = (0) if is_A: if channel // C0 == 0: matrix_dim = (matrix_dim / channel) * C0 - return (int(matrix_dim // C0), int(matrix_dim // C0), C0, C0), int(matrix_dim) + ll = (int(matrix_dim // C0), int(matrix_dim // C0), C0, C0), int(matrix_dim) else: - return (int(matrix_dim // C0), int(matrix_dim // C0), C0, C0), int(matrix_dim) - + ll = (int(matrix_dim // C0), int(matrix_dim // C0), C0, C0), int(matrix_dim) + return ll class _Conv(Cell): r"""Applies a N-D convolution over an input signal composed of several input @@ -91,7 +92,7 @@ class _Conv(Cell): 'attr \'group\' of \'Conv2D\' Op.') self.weight = Parameter(initializer( - weight_init, [out_channels, in_channels // group, *kernel_size]), name='weight') + weight_init, [out_channels, in_channels // group, *kernel_size]), name='weight') if check_bool(has_bias): self.bias = Parameter(_initializer( @@ -311,18 +312,18 @@ class Conv2d_Thor(_Conv): 'stride={}, pad_mode={}, padding={}, dilation={}, ' \ 'group={}, data_format={}, has_bias={},' \ 'weight_init={}, bias_init={}'.format( - self.in_channels, - self.out_channels, - self.kernel_size, - self.stride, - self.pad_mode, - self.padding, - self.dilation, - self.group, - self.data_format, - self.has_bias, - self.weight, - self.bias) + self.in_channels, + self.out_channels, + self.kernel_size, + self.stride, + self.pad_mode, + self.padding, + self.dilation, + self.group, + self.data_format, + self.has_bias, + self.weight, + self.bias) if self.has_bias: s += ', bias={}'.format(self.bias) diff --git a/example/resnet50_imagenet2012_THOR/train.py b/example/resnet50_imagenet2012_THOR/train.py index fe192884c41..421cda38860 100644 --- a/example/resnet50_imagenet2012_THOR/train.py +++ b/example/resnet50_imagenet2012_THOR/train.py @@ -125,7 +125,7 @@ if __name__ == '__main__': else: lr = Tensor(get_lr(global_step=0, lr_init=config.lr_init, lr_end=config.lr_end, lr_max=config.lr_max, warmup_epochs=config.warmup_epochs, total_epochs=epoch_size, steps_per_epoch=step_size, - lr_decay_mode='poly')) + )) opt = THOR(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, damping, config.frequency, filter(lambda x: 'matrix_A' in x.name, net.get_parameters()), From 78b133b08d63fadcd00dcd87f987374d46ca08c7 Mon Sep 17 00:00:00 2001 From: z00478463 Date: Fri, 22 May 2020 15:38:06 +0800 Subject: [PATCH 08/11] for pylint 5th --- example/resnet50_imagenet2012_THOR/model/resnet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/resnet50_imagenet2012_THOR/model/resnet.py b/example/resnet50_imagenet2012_THOR/model/resnet.py index 7a9b5a8922d..fb053413489 100644 --- a/example/resnet50_imagenet2012_THOR/model/resnet.py +++ b/example/resnet50_imagenet2012_THOR/model/resnet.py @@ -126,7 +126,7 @@ def _bn_last(channel): def _fc(in_channel, out_channel, damping, loss_scale, frequency): weight_shape = (out_channel, in_channel) weight = Tensor(kaiming_uniform(weight_shape, a=math.sqrt(5)) - return Dense_Thor(in_channel, out_channel, has_bias=False, weight_init=weight, \ + return Dense_Thor(in_channel, out_channel, has_bias=False, weight_init=weight, bias_init=0, damping=damping, loss_scale=loss_scale, frequency=frequency) From f4f2b58d25d289c9f93857f04a2a79e27642a086 Mon Sep 17 00:00:00 2001 From: z00478463 Date: Fri, 22 May 2020 15:49:38 +0800 Subject: [PATCH 09/11] for pylint 6th --- example/resnet50_imagenet2012_THOR/train.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/example/resnet50_imagenet2012_THOR/train.py b/example/resnet50_imagenet2012_THOR/train.py index 421cda38860..020d56d55b4 100644 --- a/example/resnet50_imagenet2012_THOR/train.py +++ b/example/resnet50_imagenet2012_THOR/train.py @@ -115,17 +115,12 @@ if __name__ == '__main__': step_size = dataset.get_dataset_size() loss_scale = FixedLossScaleManager(config.loss_scale, drop_overflow_update=False) - if config.lr_decay_mode == 'cosine': - lr = Tensor(warmup_cosine_annealing_lr(0.035, - step_size, - config.warmup_epochs, - 50, - config.T_max, - config.eta_min)) - else: - lr = Tensor(get_lr(global_step=0, lr_init=config.lr_init, lr_end=config.lr_end, lr_max=config.lr_max, - warmup_epochs=config.warmup_epochs, total_epochs=epoch_size, steps_per_epoch=step_size, - )) + lr = Tensor(warmup_cosine_annealing_lr(0.035, + step_size, + config.warmup_epochs, + 50, + config.T_max, + config.eta_min)) opt = THOR(filter(lambda x: x.requires_grad, net.get_parameters()), lr, config.momentum, damping, config.frequency, filter(lambda x: 'matrix_A' in x.name, net.get_parameters()), From 81d7cc484507df481471b43ef712ba3be5817ff2 Mon Sep 17 00:00:00 2001 From: z00478463 Date: Fri, 22 May 2020 15:57:30 +0800 Subject: [PATCH 10/11] for pylint 7th --- example/resnet50_imagenet2012_THOR/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/resnet50_imagenet2012_THOR/train.py b/example/resnet50_imagenet2012_THOR/train.py index 020d56d55b4..3843338a517 100644 --- a/example/resnet50_imagenet2012_THOR/train.py +++ b/example/resnet50_imagenet2012_THOR/train.py @@ -33,7 +33,7 @@ import numpy as np from config_imagenet import config from crossentropy import CrossEntropy from dataset_imagenet import create_dataset -from lr_generator import get_lr, warmup_cosine_annealing_lr +from lr_generator import warmup_cosine_annealing_lr random.seed(1) np.random.seed(1) From 270d348daeae28b898425023e05259c71502ef61 Mon Sep 17 00:00:00 2001 From: z00478463 Date: Sat, 23 May 2020 10:47:35 +0800 Subject: [PATCH 11/11] for comments 1st --- .../cus_ops/batch_matmul_impl.py | 75 ------------ .../cus_ops/cholesky_trsm.py | 63 ---------- .../cus_ops/cus_batch_matmul.py | 37 ------ .../cus_ops/cus_cholesky_trsm.py | 37 ------ .../cus_ops/cus_fused_abs_max1.py | 44 ------- .../cus_ops/cus_img2col.py | 51 -------- .../cus_ops/cus_matmul_cube.py | 54 --------- .../cus_ops/cus_matmul_cube_dense_left.py | 40 ------ .../cus_matmul_cube_fracz_right_mul.py | 39 ------ .../cus_ops/cus_matrix_combine.py | 41 ------- .../cus_ops/cus_transpose02314.py | 44 ------- .../cus_ops/fused_abs_max1.py | 69 ----------- .../cus_ops/img2col_impl.py | 87 ------------- .../cus_ops/matmul_cube_dense_left.py | 102 ---------------- .../matmul_cube_fracz_left_cast_impl.py | 103 ---------------- .../matmul_cube_fracz_right_mul_impl.py | 113 ----------------- .../cus_ops/matmul_cube_impl.py | 114 ------------------ .../cus_ops/matrix_combine_impl.py | 63 ---------- .../cus_ops/transpose02314_impl.py | 63 ---------- 19 files changed, 1239 deletions(-) delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/cus_batch_matmul.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/cus_img2col.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube_dense_left.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube_fracz_right_mul.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/cus_matrix_combine.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/cus_transpose02314.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/fused_abs_max1.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/img2col_impl.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_dense_left.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_left_cast_impl.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_impl.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/matrix_combine_impl.py delete mode 100644 example/resnet50_imagenet2012_THOR/cus_ops/transpose02314_impl.py diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py deleted file mode 100644 index 201d427308e..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/batch_matmul_impl.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""batch_matmul_impl""" -from mindspore.ops.op_info_register import op_info_register - -@op_info_register("""{ - "op_name": "CusBatchMatMul", - "imply_type": "TBE", - "fusion_type": "OPAQUE", - "async_flag": false, - "binfile_name": "batchmatmul.so", - "compute_cost": 10, - "kernel_name": "CusBatchMatMul", - "partial_flag": true, - "attr": [ - ], - "inputs": [ - { - "index": 0, - "dtype": [ - "float32" - ], - "format": [ - "DefaultFormat" - ], - "name": "x1", - "need_compile": false, - "param_type": "required", - "shape": "all" - }, - { - "index": 1, - "dtype": [ - "float32" - ], - "format": [ - "DefaultFormat" - ], - "name": "x2", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ], - "outputs": [ - { - "index": 0, - "dtype": [ - "float32" - ], - "format": [ - "DefaultFormat" - ], - "name": "y", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ] -}""") -def CusBatchMatMul(input_x1, input_x2, output, transpose_a=False, transpose_b=True, kernel_name="batchmatmul"): - """CusBatchMatMul""" - return diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py b/example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py deleted file mode 100644 index 8f09fab77da..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cholesky_trsm.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CusCholeskyTrsm""" -from mindspore.ops.op_info_register import op_info_register - -@op_info_register("""{ - "op_name": "CusCholeskyTrsm", - "imply_type": "TBE", - "fusion_type": "OPAQUE", - "async_flag": false, - "binfile_name": "choleskytrsm.so", - "compute_cost": 10, - "kernel_name": "CusCholeskyTrsm", - "partial_flag": true, - "attr": [ - - ], - "inputs": [ - { - "index": 0, - "dtype": [ - "float32" - ], - "format": [ - "DefaultFormat" - ], - "name": "x1", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ], - "outputs": [ - { - "index": 0, - "dtype": [ - "float32" - ], - "format": [ - "DefaultFormat" - ], - "name": "y", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ] -}""") -def CusCholeskyTrsm(input_x, output, kernel_name): - """CusCholeskyTrsm""" - return diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_batch_matmul.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_batch_matmul.py deleted file mode 100644 index 1e72ed9fac1..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_batch_matmul.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CusBatchMatMul""" -from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore.ops.composite import multitype_ops as C - -class CusBatchMatMul(PrimitiveWithInfer): - """CusMatMulCube definition""" - - @prim_attr_register - def __init__(self): - """init CusMatMulCube""" - self.init_prim_io_names(inputs=['x1', 'x2'], outputs=['y']) - - def get_bprop(self): - def bprop(x1, x2, out, dout): - return (C.zeros_like(x1), C.zeros_like(x2)) - - return bprop - - def infer_shape(self, data1_shape, data2_shape): - return data1_shape - - def infer_dtype(self, data1_dtype, data2_dtype): - return data1_dtype diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py deleted file mode 100644 index d6ea0156223..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_cholesky_trsm.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CusCholeskyTrsm""" -from mindspore.ops import prim_attr_register, PrimitiveWithInfer - - -class CusCholeskyTrsm(PrimitiveWithInfer): - """CusCholeskyTrsm definition""" - - @prim_attr_register - def __init__(self): - """init CusCholeskyTrsm""" - self.init_prim_io_names(inputs=['x1'], outputs=['y']) - - def infer_shape(self, data1_shape): - ll = [] - m, _ = data1_shape - if m >= 128: - ll = [m // 128, 128, 128] - else: - ll = [1, 64, 64] - return ll - - def infer_dtype(self, data1_dtype): - return data1_dtype diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py deleted file mode 100644 index c49c36ddd47..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_fused_abs_max1.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CusFusedAbsMax1""" -from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore.ops.composite import multitype_ops as C - - -class CusFusedAbsMax1(PrimitiveWithInfer): - """CusCholeskyTrsm definition""" - - @prim_attr_register - def __init__(self, origin_shape=[-1, -1]): - """init CusCholeskyTrsm""" - self.init_prim_io_names(inputs=['x1'], outputs=['y']) - self.origin_shape = origin_shape - - def get_bprop(self): - def bprop(x, out, dout): - return (C.zeros_like(x),) - - return bprop - - def infer_shape(self, data1_shape): - ll = [] - if len(data1_shape) == 2: - ll = [1,] - else: - ll = [32, 64] - return ll - - def infer_dtype(self, data1_dtype): - return data1_dtype diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_img2col.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_img2col.py deleted file mode 100644 index e1128291dde..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_img2col.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CusImg2Col""" -from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore.ops.composite import multitype_ops as C - - -class CusImg2Col(PrimitiveWithInfer): - """CusImg2Col definition""" - - @prim_attr_register - def __init__(self, ksizes, strides, dilates=(1, 1, 1, 1), mode="NC1HWC0"): - """init CusImg2Col""" - self.init_prim_io_names(inputs=['x1'], outputs=['y']) - self.ksizes = ksizes - self.strides = strides - self.dilates = dilates - self.mode = mode - - def get_bprop(self): - def bprop(x, out, dout): - return (C.zeros_like(x),) - - return bprop - - def infer_shape(self, data1_shape): - bs, c, h, w = data1_shape - _, stride_h, stride_w, _ = self.strides - _, k_w, k_h, _ = self.ksizes - # assert m == n - c0 = 16 - c1 = c // 16 - if c1 == 0: - c1 = 1 - shape = [bs * int(h // stride_h) * int(w // stride_w), k_w * k_h * c1 * c0] - return shape - - def infer_dtype(self, data1_dtype): - return data1_dtype diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube.py deleted file mode 100644 index e41cb6bc98b..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CusMatMulCube""" -import mindspore as ms -from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore.ops.composite import multitype_ops as C - - -# y = x^2 -class CusMatMulCube(PrimitiveWithInfer): - """CusMatMulCube definition""" - - @prim_attr_register - def __init__(self, transpose_a=False, transpose_b=False): - """init CusMatMulCube""" - self.init_prim_io_names(inputs=['x1', 'x2'], outputs=['y']) - self.transpose_a = transpose_a - self.transpose_b = transpose_b - - def get_bprop(self): - def bprop(x1, x2, out, dout): - return (C.zeros_like(x1), C.zeros_like(x2)) - - return bprop - - def infer_shape(self, data1_shape, data2_shape): - # shape = [1, data1_shape[1], data2_shape[2], 16, 16] - # return shape - if self.transpose_a: - k1, m = data1_shape - else: - m, k1 = data1_shape - if self.transpose_b: - n, k2 = data2_shape - else: - k2, n = data2_shape - assert k1 == k2 - shape = [m, n] - return shape - - def infer_dtype(self, data1_dtype, data2_dtype): - return ms.common.dtype.tensor_type(getattr(ms, "float32")) diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube_dense_left.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube_dense_left.py deleted file mode 100644 index 4228e3938a0..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube_dense_left.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CusMatMulCubeDenseLeft""" -import mindspore as ms -from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore.ops.composite import multitype_ops as C - - -# y = x^2 -class CusMatMulCubeDenseLeft(PrimitiveWithInfer): - """CusMatMulCube definition""" - - @prim_attr_register - def __init__(self): - """init CusMatMulCube""" - self.init_prim_io_names(inputs=['x1', 'x2'], outputs=['y']) - - def get_bprop(self): - def bprop(x1, x2, out, dout): - return (C.zeros_like(x1), C.zeros_like(x2)) - - return bprop - - def infer_shape(self, data1_shape, data2_shape): - return data2_shape - - def infer_dtype(self, data1_dtype, data2_dtype): - return ms.common.dtype.tensor_type(getattr(ms, "float16")) diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube_fracz_right_mul.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube_fracz_right_mul.py deleted file mode 100644 index a80a52797e4..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_matmul_cube_fracz_right_mul.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CusMatMulCubeFraczRightMul""" -import mindspore as ms -from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore.ops.composite import multitype_ops as C - - -class CusMatMulCubeFraczRightMul(PrimitiveWithInfer): - """CusMatMulCubeFraczRightMul definition""" - - @prim_attr_register - def __init__(self): - """init CusMatMulCubeFraczRightMul""" - self.init_prim_io_names(inputs=['x1', 'x2', 'x3'], outputs=['y']) - - def get_bprop(self): - def bprop(x1, x2, x3, out, dout): - return (C.zeros_like(x1), C.zeros_like(x2), C.zeros_like(x3)) - - return bprop - - def infer_shape(self, data1_shape, data2_shape, data3_shape): - return data1_shape - - def infer_dtype(self, data1_dtype, data2_dtype, data3_dtype): - return ms.common.dtype.tensor_type(getattr(ms, "float32")) diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_matrix_combine.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_matrix_combine.py deleted file mode 100644 index 373f149dd08..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_matrix_combine.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CusMatrixCombine""" -from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore.ops.composite import multitype_ops as C - - -class CusMatrixCombine(PrimitiveWithInfer): - """CusMatMulCube definition""" - - @prim_attr_register - def __init__(self): - """init CusMatMulCube""" - self.init_prim_io_names(inputs=['x'], outputs=['y']) - - def get_bprop(self): - def bprop(x, out, dout): - return (C.zeros_like(x),) - - return bprop - - def infer_shape(self, data_shape): - a, b, c = data_shape - shape = [a * b, a * c] - - return shape - - def infer_dtype(self, data_dtype): - return data_dtype diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/cus_transpose02314.py b/example/resnet50_imagenet2012_THOR/cus_ops/cus_transpose02314.py deleted file mode 100644 index b856cf7da78..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/cus_transpose02314.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CusTranspose02314""" - -from mindspore.ops import prim_attr_register, PrimitiveWithInfer -from mindspore.ops.composite import multitype_ops as C - - -class CusTranspose02314(PrimitiveWithInfer): - """CusTranspose02314 definition""" - - @prim_attr_register - def __init__(self): - """init CusTranspose02314""" - self.init_prim_io_names(inputs=['x1'], outputs=['y']) - - def get_bprop(self): - def bprop(x, out, dout): - return (C.zeros_like(x),) - - return bprop - - def infer_shape(self, data1_shape): - assert len(data1_shape) == 4 - n, c, h, w = data1_shape - c0 = 16 - c1 = c // 16 - shape = (n * h * w, c1 * c0) - return shape - - def infer_dtype(self, data1_dtype): - return data1_dtype diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/fused_abs_max1.py b/example/resnet50_imagenet2012_THOR/cus_ops/fused_abs_max1.py deleted file mode 100644 index b9a0d452738..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/fused_abs_max1.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CusFusedAbsMax1""" -from mindspore.ops.op_info_register import op_info_register - - -@op_info_register("""{ - "op_name": "CusFusedAbsMax1", - "imply_type": "TBE", - "fusion_type": "OPAQUE", - "async_flag": false, - "binfile_name": "fusedabsmax1.so", - "compute_cost": 10, - "kernel_name": "CusFusedAbsMax1", - "partial_flag": true, - "attr": [ - { - "name": "origin_shape", - "param_type": "required", - "type": "listInt", - "value": "all" - } - ], - "inputs": [ - { - "index": 0, - "dtype": [ - "float32" - ], - "format": [ - "DefaultFormat" - ], - "name": "x1", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ], - "outputs": [ - { - "index": 0, - "dtype": [ - "float32" - ], - "format": [ - "DefaultFormat" - ], - "name": "y", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ] -}""") -def CusFusedAbsMax1(input_x, output, origin_shape=None, kernel_name="fused_abs_max1"): - """CusFusedAbsMax1""" - return diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/img2col_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/img2col_impl.py deleted file mode 100644 index 5137d4d7e70..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/img2col_impl.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CusImg2ColNC1HWC0""" -from mindspore.ops.op_info_register import op_info_register - - -@op_info_register("""{ - "op_name": "CusImg2ColNC1HWC0", - "imply_type": "TBE", - "fusion_type": "OPAQUE", - "async_flag": false, - "binfile_name": "img2colnc1hwc0.so", - "compute_cost": 10, - "kernel_name": "CusImg2ColNC1HWC0", - "partial_flag": true, - "attr": [ - { - "name": "ksizes", - "param_type": "required", - "type": "listInt", - "value": "all" - }, - { - "name": "strides", - "param_type": "required", - "type": "listInt", - "value": "all" - }, - { - "name": "dilates", - "param_type": "required", - "type": "listInt", - "value": "all" - }, - { - "name": "padding", - "param_type": "required", - "type": "str", - "value": "all" - } - ], - "inputs": [ - { - "index": 0, - "dtype": [ - "float16" - ], - "format": [ - "NC1HWC0" - ], - "name": "x1", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ], - "outputs": [ - { - "index": 0, - "dtype": [ - "float16" - ], - "format": [ - "FRACTAL_NZ" - ], - "name": "y", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ] -}""") -def CusImg2ColNC1HWC0(input_x, output, ksizes, strides, dilates, padding, kernel_name="img2col"): - """CusImg2ColNC1HWC0""" - return diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_dense_left.py b/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_dense_left.py deleted file mode 100644 index 91c5336aa0c..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_dense_left.py +++ /dev/null @@ -1,102 +0,0 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -""" -copyright 2020 Huawei Technologies Co., Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License == distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -matmul -""" -from __future__ import absolute_import - -from mindspore.ops.op_info_register import op_info_register -from topi.cce import util - -# General limitation of the size for input shape: 2**31 -SHAPE_SIZE_LIMIT = 2147483648 -NoneType = type(None) - - -@op_info_register("""{ - "op_name": "CusMatMulCubeDenseLeft", - "imply_type": "TBE", - "fusion_type": "OPAQUE", - "async_flag": false, - "binfile_name": "matmulcubedenseleft.so", - "compute_cost": 10, - "kernel_name": "CusMatMulCubeDenseLeft", - "partial_flag": true, - "attr": [ - ], - "inputs": [ - { - "index": 0, - "dtype": [ - "float16" - ], - "format": [ - "DefaultFormat" - ], - "name": "x1", - "need_compile": false, - "param_type": "required", - "shape": "all" - }, - { - "index": 1, - "dtype": [ - "float16" - ], - "format": [ - "FRACTAL_NZ" - ], - "name": "x2", - "need_compile": false, - "param_type": "required", - "shape": "all" - }, - { - "index": 2, - "dtype": [ - "float16" - ], - "format": [ - "DefaultFormat" - ], - "name": "x3", - "need_compile": false, - "param_type": "optional", - "shape": "all" - } - ], - "outputs": [ - { - "index": 0, - "dtype": [ - "float16" - ], - "format": [ - "FRACTAL_NZ" - ], - "name": "y", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ] -}""") -@util.check_input_type(dict, dict, (dict, NoneType), dict, bool, bool, str) -def CusMatMulCubeDenseLeft(input_x1, input_x2, bias=None, output_y={}, trans_a=False, trans_b=False, - kernel_name="matmulcube"): - """CusMatMulCubeDenseLeft""" - return diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_left_cast_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_left_cast_impl.py deleted file mode 100644 index de2cca10779..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_left_cast_impl.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -""" -copyright 2020 Huawei Technologies Co., Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License == distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -matmul -""" -from __future__ import absolute_import - -from mindspore.ops.op_info_register import op_info_register -from topi.cce import util - -# General limitation of the size for input shape: 2**31 -SHAPE_SIZE_LIMIT = 2147483648 -NoneType = type(None) - - -@op_info_register("""{ - "op_name": "CusMatMulCubeFraczLeftCast", - "imply_type": "TBE", - "fusion_type": "OPAQUE", - "async_flag": false, - "binfile_name": "matmulcubefraczleftcast.so", - "compute_cost": 10, - "kernel_name": "CusMatMulCubeFraczLeftCast", - "partial_flag": true, - "attr": [ - ], - "inputs": [ - { - "index": 0, - "dtype": [ - "float16" - ], - "format": [ - "DefaultFormat" - ], - "name": "x1", - "need_compile": false, - "param_type": "required", - "shape": "all" - }, - { - "index": 1, - "dtype": [ - "float32" - ], - "format": [ - "FracZ" - ], - "name": "x2", - "need_compile": false, - "param_type": "required", - "shape": "all" - }, - { - "index": 2, - "dtype": [ - "float16" - ], - "format": [ - "DefaultFormat" - ], - "name": "x3", - "need_compile": false, - "param_type": "optional", - "shape": "all" - } - ], - "outputs": [ - { - "index": 0, - "dtype": [ - "float16" - ], - "format": [ - "FracZ" - ], - "name": "y", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ] -}""") -# pylint: disable=locally-disabled,too-many-arguments, too-many-locals, too-many-statements -@util.check_input_type(dict, dict, (dict, NoneType), dict, bool, bool, str) -def CusMatMulCubeFraczLeftCast(input_x1, input_x2, bias=None, output_y={}, trans_a=False, trans_b=False, - kernel_name="CusMatMulCubeFraczLeftCast"): - """CusMatMulCubeFraczLeftCast""" - return diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py deleted file mode 100644 index 7fc2ba35d16..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_fracz_right_mul_impl.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -""" -copyright 2020 Huawei Technologies Co., Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License == distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -matmul -""" -from __future__ import absolute_import - -from mindspore.ops.op_info_register import op_info_register - -# General limitation of the size for input shape: 2**31 -SHAPE_SIZE_LIMIT = 2147483648 -NoneType = type(None) - - -@op_info_register("""{ - "op_name": "CusMatMulCubeFraczRightMul", - "imply_type": "TBE", - "fusion_type": "OPAQUE", - "async_flag": false, - "binfile_name": "matmulcubefraczrightmul.so", - "compute_cost": 10, - "kernel_name": "CusMatMulCubeFraczRightMul", - "partial_flag": true, - "attr": [ - ], - "inputs": [ - { - "index": 0, - "dtype": [ - "float16" - ], - "format": [ - "FracZ" - ], - "name": "x1", - "need_compile": false, - "param_type": "required", - "shape": "all" - }, - { - "index": 1, - "dtype": [ - "float16" - ], - "format": [ - "DefaultFormat" - ], - "name": "x2", - "need_compile": false, - "param_type": "required", - "shape": "all" - }, - { - "index": 2, - "dtype": [ - "float32" - ], - "format": [ - "DefaultFormat" - ], - "name": "x3", - "need_compile": false, - "param_type": "required", - "shape": "all" - }, - { - "index": 3, - "dtype": [ - "float16" - ], - "format": [ - "DefaultFormat" - ], - "name": "x4", - "need_compile": false, - "param_type": "optional", - "shape": "all" - } - ], - "outputs": [ - { - "index": 0, - "dtype": [ - "float32" - ], - "format": [ - "FracZ" - ], - "name": "y", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ] -}""") -def CusMatMulCubeFraczRightMul(input_x1, input_x2, input_x3, bias=None, output_y={}, trans_a=False, trans_b=False, - kernel_name="matmulcube"): - """CusMatMulCubeFraczRightMul""" - return diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_impl.py deleted file mode 100644 index 7c2d81e1d67..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/matmul_cube_impl.py +++ /dev/null @@ -1,114 +0,0 @@ -#!/usr/bin/env python -# -*- coding:utf-8 -*- -""" -copyright 2020 Huawei Technologies Co., Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License == distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. - -matmul -""" -from __future__ import absolute_import - -from mindspore.ops.op_info_register import op_info_register -from topi.cce import util - -# General limitation of the size for input shape: 2**31 -SHAPE_SIZE_LIMIT = 2147483648 -NoneType = type(None) - - -@op_info_register("""{ - "op_name": "CusMatMulCube", - "imply_type": "TBE", - "fusion_type": "OPAQUE", - "async_flag": false, - "binfile_name": "matmulcube.so", - "compute_cost": 10, - "kernel_name": "CusMatMulCube", - "partial_flag": true, - "attr": [ - { - "name": "transpose_a", - "param_type": "required", - "type": "bool", - "value": "all" - }, - { - "name": "transpose_b", - "param_type": "required", - "type": "bool", - "value": "all" - } - ], - "inputs": [ - { - "index": 0, - "dtype": [ - "float16" - ], - "format": [ - "FRACTAL_NZ" - ], - "name": "x1", - "need_compile": false, - "param_type": "required", - "shape": "all" - }, - { - "index": 1, - "dtype": [ - "float16" - ], - "format": [ - "FRACTAL_NZ" - ], - "name": "x2", - "need_compile": false, - "param_type": "required", - "shape": "all" - }, - { - "index": 2, - "dtype": [ - "float16" - ], - "format": [ - "DefaultFormat" - ], - "name": "x3", - "need_compile": false, - "param_type": "optional", - "shape": "all" - } - ], - "outputs": [ - { - "index": 0, - "dtype": [ - "float32" - ], - "format": [ - "FRACTAL_NZ" - ], - "name": "y", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ] -}""") -# pylint: disable=locally-disabled,too-many-arguments, too-many-locals, too-many-statements -@util.check_input_type(dict, dict, (dict, NoneType), dict, bool, bool, str) -def CusMatMulCube(input_x1, input_x2, bias=None, output_y={}, trans_a=False, trans_b=False, kernel_name="matmulcube"): - """CusMatMulCube""" - return diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/matrix_combine_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/matrix_combine_impl.py deleted file mode 100644 index 32045e7ccbd..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/matrix_combine_impl.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CusMatrixCombine""" -from mindspore.ops.op_info_register import op_info_register - - -@op_info_register("""{ - "op_name": "CusMatrixCombine", - "imply_type": "TBE", - "fusion_type": "OPAQUE", - "async_flag": false, - "binfile_name": "matrixcombine.so", - "compute_cost": 10, - "kernel_name": "CusMatrixCombine", - "partial_flag": true, - "attr": [ - ], - "inputs": [ - { - "index": 0, - "dtype": [ - "float32" - ], - "format": [ - "DefaultFormat" - ], - "name": "x1", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ], - "outputs": [ - { - "index": 0, - "dtype": [ - "float32" - ], - "format": [ - "DefaultFormat" - ], - "name": "y", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ] -}""") -def CusMatrixCombine(input_x, output, kernel_name="matrix_combine"): - """CusMatrixCombine""" - return diff --git a/example/resnet50_imagenet2012_THOR/cus_ops/transpose02314_impl.py b/example/resnet50_imagenet2012_THOR/cus_ops/transpose02314_impl.py deleted file mode 100644 index c5aebe523d5..00000000000 --- a/example/resnet50_imagenet2012_THOR/cus_ops/transpose02314_impl.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2020 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""CusTranspose02314""" -from mindspore.ops.op_info_register import op_info_register - - -@op_info_register("""{ - "op_name": "CusTranspose02314", - "imply_type": "TBE", - "fusion_type": "OPAQUE", - "async_flag": false, - "binfile_name": "transpose02314.so", - "compute_cost": 10, - "kernel_name": "CusTranspose02314", - "partial_flag": true, - "attr": [ - ], - "inputs": [ - { - "index": 0, - "dtype": [ - "float16" - ], - "format": [ - "NC1HWC0" - ], - "name": "x1", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ], - "outputs": [ - { - "index": 0, - "dtype": [ - "float16" - ], - "format": [ - "DefaultFormat" - ], - "name": "y", - "need_compile": false, - "param_type": "required", - "shape": "all" - } - ] -}""") -def CusTranspose02314(input_x, output, kernel_name="transpose021354"): - """CusTranspose02314""" - return