From 2dea092baff7d0c59af57278a5e98f1d0e65c79d Mon Sep 17 00:00:00 2001 From: chaijun Date: Thu, 16 Jun 2022 16:21:05 +0800 Subject: [PATCH] add unique_with_pad functional --- .jenkins/check/config/whitelizard.txt | 1 + .../api_python/mindspore.ops.functional.rst | 1 + .../api_python/mindspore/mindspore.Tensor.rst | 21 +++++++++ .../ops/mindspore.ops.UniqueWithPad.rst | 12 +++++ .../mindspore.ops.func_unique_with_pad.rst | 24 ++++++++++ mindspore/ccsrc/pipeline/jit/resource.cc | 1 + mindspore/lite/src/common/file_utils.cc | 6 +-- .../lite/tools/converter/micro/coder/config.h | 2 +- .../_extends/parse/standard_method.py | 7 +++ mindspore/python/mindspore/common/tensor.py | 40 ++++++++++++++++ .../python/mindspore/ops/function/__init__.py | 1 + .../mindspore/ops/function/array_func.py | 47 +++++++++++++++++++ mindspore/python/mindspore/ops/functional.py | 1 + .../mindspore/ops/operations/array_ops.py | 15 +----- 14 files changed, 162 insertions(+), 17 deletions(-) create mode 100644 docs/api/api_python/ops/mindspore.ops.UniqueWithPad.rst create mode 100644 docs/api/api_python/ops/mindspore.ops.func_unique_with_pad.rst diff --git a/.jenkins/check/config/whitelizard.txt b/.jenkins/check/config/whitelizard.txt index f7f073287f8..dcdc4242a80 100644 --- a/.jenkins/check/config/whitelizard.txt +++ b/.jenkins/check/config/whitelizard.txt @@ -23,6 +23,7 @@ mindspore/model_zoo/official/recommend/wide_and_deep/src/wide_and_deep.py:__init mindspore/model_zoo/official/recommend/wide_and_deep_multitable/src/wide_and_deep.py:__init__ mindspore/mindspore/ccsrc/pipeline/jit/resource.cc:mindspore::pipeline::GetMethodMap mindspore/mindspore/python/mindspore/ops/operations/array_ops.py:_compute_slicing_shape +mindspore/mindspore/python/mindspore/ops/function/array_func.py:scatter_nd mindspore/mindspore/python/mindspore/context.py:set_auto_parallel_context mindspore/mindspore/python/mindspore/common/tensor.py:__init__ mindspore/mindspore/python/mindspore/common/parameter.py:set_data diff --git a/docs/api/api_python/mindspore.ops.functional.rst b/docs/api/api_python/mindspore.ops.functional.rst index be598c74208..7bf9e3e8bcc 100644 --- a/docs/api/api_python/mindspore.ops.functional.rst +++ b/docs/api/api_python/mindspore.ops.functional.rst @@ -346,6 +346,7 @@ Array操作 mindspore.ops.transpose mindspore.ops.unique mindspore.ops.unique_consecutive + mindspore.ops.unique_with_pad .. list-table:: :widths: 50 50 diff --git a/docs/api/api_python/mindspore/mindspore.Tensor.rst b/docs/api/api_python/mindspore/mindspore.Tensor.rst index c9486aa7d35..be092c677f2 100644 --- a/docs/api/api_python/mindspore/mindspore.Tensor.rst +++ b/docs/api/api_python/mindspore/mindspore.Tensor.rst @@ -1458,6 +1458,27 @@ mindspore.Tensor - **TypeError** - 输入参数类型有误。 - **ValueError** - `axes` 的数量不等于Tensor.ndim。 + .. py:method:: unique_with_pad(pad_num) + + 对当前一维张量中元素去重，返回一维张量中的唯一元素（使用pad_num填充）和相对索引。 + + 基本操作与unique相同，但unique_with_pad多了pad操作。 + unique运算符对张量处理后所返回的元组（`y`，`idx`），`y` 与 `idx` 的shape通常会有差别，因此，为了解决上述情况， + unique_with_pad操作符将用用户指定的`pad_num`填充`y`张量，使其具有与张量`idx`相同的形状。 + + **参数：** + + - **pad_num** (int) - 填充值。数据类型为int32或int64。 + + **返回：** + + Tuple， `(y, idx)` 。 `y` 是与当前张量形状和数据类型相同的Tensor，包含当前张量中去重后的元素，并用 `pad_num` 填充。 `idx` 为索引Tensor，包含当前张量中的元素在 `y` 中的索引，与当前张量的shape相同。 + + **异常：** + + - **TypeError** - 当前张量的数据类型既不是int32也不是int64。 + - **ValueError** - 当前张量不是一维张量。 + .. py:method:: unique_consecutive(return_idx=False, return_counts=False, axis=None) 返回输入张量中每个连续等效元素组中唯一的元素。 diff --git a/docs/api/api_python/ops/mindspore.ops.UniqueWithPad.rst b/docs/api/api_python/ops/mindspore.ops.UniqueWithPad.rst new file mode 100644 index 00000000000..acb092b4cb2 --- /dev/null +++ b/docs/api/api_python/ops/mindspore.ops.UniqueWithPad.rst @@ -0,0 +1,12 @@ +mindspore.ops.UniqueWithPad +==================== + +.. py:class:: mindspore.ops.UniqueWithPad + + 对输入一维张量中元素去重，返回一维张量中的唯一元素（使用pad_num填充）和相对索引。 + + 基本操作与Unique相同，但UniqueWithPad多了Pad操作。 + Unique运算符处理输入张量`x`后所返回的元组（`y`，`idx`），`y` 与 `idx` 的shape通常会有差别，因此，为了解决上述情况， + UniqueWithPad操作符将用用户指定的`pad_num`填充`y`张量，使其具有与张量`idx`相同的形状。 + + 更多参考详见 :func:`mindspore.ops.unique_with_pad`。 diff --git a/docs/api/api_python/ops/mindspore.ops.func_unique_with_pad.rst b/docs/api/api_python/ops/mindspore.ops.func_unique_with_pad.rst new file mode 100644 index 00000000000..ef39654045b --- /dev/null +++ b/docs/api/api_python/ops/mindspore.ops.func_unique_with_pad.rst @@ -0,0 +1,24 @@ +mindspore.ops.unique_with_pad +==================== + +.. py:function:: mindspore.ops.unique_with_pad(x, pad_num) + + 对输入一维张量中元素去重，返回一维张量中的唯一元素（使用pad_num填充）和相对索引。 + + 基本操作与unique相同，但unique_with_pad多了pad操作。 + unique运算符对张量处理后所返回的元组（`y`，`idx`），`y` 与 `idx` 的shape通常会有差别，因此，为了解决上述情况， + unique_with_pad操作符将用用户指定的`pad_num`填充`y`张量，使其具有与张量`idx`相同的形状。 + + **参数：** + + - **x** (Tensor) - 需要被去重的Tensor。必须是类型为int32或int64的一维向量。 + - **pad_num** (int) - 填充值。数据类型为int32或int64。 + + **返回：** + + Tuple， `(y, idx)` 。 `y` 是与 `x` 形状和数据类型相同的Tensor，包含 `x` 中去重后的元素，并用 `pad_num` 填充。 `idx` 为索引Tensor，包含 `x` 中的元素在 `y` 中的索引，与 `x` 的shape相同。 + + **异常：** + + - **TypeError** - 'x'的数据类型既不是int32也不是int64。 + - **ValueError** - 'x'不是一维张量。 diff --git a/mindspore/ccsrc/pipeline/jit/resource.cc b/mindspore/ccsrc/pipeline/jit/resource.cc index 46d2900f1ed..4453d10dcc8 100644 --- a/mindspore/ccsrc/pipeline/jit/resource.cc +++ b/mindspore/ccsrc/pipeline/jit/resource.cc @@ -252,6 +252,7 @@ BuiltInTypeMap &GetMethodMap() { {"one_hot", std::string("one_hot")}, // P.OneHot {"gather_nd", std::string("gather_nd")}, // P.GatherNd() {"unique_consecutive", std::string("unique_consecutive")}, // UniqueConsecutive() + {"unique_with_pad", std::string("unique_with_pad")}, // P.UniqueWithPad() {"diag", std::string("diag")}, // P.Diag() {"adaptive_max_pool2d", std::string("adaptive_max_pool2d")}, // P.AdaptiveMaxPool2D {"to_coo", std::string("to_coo")}, // dense_to_sparse_coo() diff --git a/mindspore/lite/src/common/file_utils.cc b/mindspore/lite/src/common/file_utils.cc index 337bbc0700a..f178299ef58 100644 --- a/mindspore/lite/src/common/file_utils.cc +++ b/mindspore/lite/src/common/file_utils.cc @@ -172,7 +172,7 @@ char *ReadFile(const char *file, size_t *size) { ifs->seekg(0, std::ios::end); *size = ifs->tellg(); - auto buf = std::make_unique(*size); + auto buf = new (std::nothrow) char[*size]; if (buf == nullptr) { MS_LOG(ERROR) << "malloc buf failed, file: " << file; ifs->close(); @@ -181,10 +181,10 @@ char *ReadFile(const char *file, size_t *size) { } ifs->seekg(0, std::ios::beg); - ifs->read(buf.get(), *size); + ifs->read(buf, *size); ifs->close(); delete ifs; - return buf.release(); + return buf; } std::string RealPath(const char *path) { diff --git a/mindspore/lite/tools/converter/micro/coder/config.h b/mindspore/lite/tools/converter/micro/coder/config.h index a854347cced..84285932818 100644 --- a/mindspore/lite/tools/converter/micro/coder/config.h +++ b/mindspore/lite/tools/converter/micro/coder/config.h @@ -24,7 +24,7 @@ enum Target { kX86 = 0, kCortex_M = 1, kARM32 = 2, kARM64 = 3, kAllTargets = 4, enum CodeMode { Inference = 0, Train = 1, Code_Unknown = 99 }; struct MicroParam { - std::string codegen_mode; + std::string codegen_mode = "Inference"; std::string target; bool enable_micro{false}; bool support_parallel{false}; diff --git a/mindspore/python/mindspore/_extends/parse/standard_method.py b/mindspore/python/mindspore/_extends/parse/standard_method.py index f8cac6416a1..69b76ca443e 100644 --- a/mindspore/python/mindspore/_extends/parse/standard_method.py +++ b/mindspore/python/mindspore/_extends/parse/standard_method.py @@ -730,6 +730,13 @@ def unique_consecutive(x, return_idx=False, return_counts=False, axis=None): return F.unique_consecutive(x, return_idx, return_counts, axis) +def unique_with_pad(x, pad_num): + """ + Returns unique elements and relative indexes in 1-D tensor, filled with padding num. + """ + return F.unique_with_pad(x, pad_num) + + def resize(x, *new_shape): """ Changes shape and size of array in-place. diff --git a/mindspore/python/mindspore/common/tensor.py b/mindspore/python/mindspore/common/tensor.py index bb693714b3a..ca8dc7c0411 100644 --- a/mindspore/python/mindspore/common/tensor.py +++ b/mindspore/python/mindspore/common/tensor.py @@ -3837,6 +3837,46 @@ class Tensor(Tensor_): return output, counts return output + def unique_with_pad(self, pad_num): + """ + Returns unique elements and relative indexes in 1-D tensor, filled with padding num. + + The basic function is the same as the Unique operator, but the UniqueWithPad operator adds a Pad function. + The returned tuple(`y`, `idx`) after the self tensor is processed by the unique operator, + in which the shapes of `y` and `idx` are mostly not equal. Therefore, in order to solve the above situation, + the UniqueWithPad operator will fill the `y` Tensor with the `pad_num` specified by the user + to make it have the same shape as the Tensor `idx`. + + Args: + pad_num (int): Pad num. The data type is an int. + + Returns: + tuple(Tensor), tuple of 2 tensors, `y` and `idx`. + - y (Tensor) - The unique elements filled with pad_num, the shape and data type same as self tensor. + - idx (Tensor) - The index of each value of self tensor in the unique output `y`, + the shape and data type same as self tensor. + + Raises: + TypeError: If dtype of self tensor is neither int32 nor int64. + ValueError: If length of shape of self tensor is not equal to 1. + + Supported Platforms: + ``Ascend`` ``GPU`` ``CPU`` + + Examples: + >>> import numpy as np + >>> from mindspore import Tensor + >>> from mindspore import dtype as mstype + >>> x = Tensor(np.array([1, 1, 2, 2, 3, 1]), mstype.int32) + >>> output, idx = x.unique_with_pad(pad_num=0) + >>> print(output) + [1 2 3 0 0 0] + >>> print(idx) + [0 0 1 1 2 0] + """ + self._init_check() + return tensor_operator_registry.get("unique_with_pad")()(self, pad_num) + def diag(self): r""" Constructs a diagonal tensor with a given diagonal values. diff --git a/mindspore/python/mindspore/ops/function/__init__.py b/mindspore/python/mindspore/ops/function/__init__.py index 2af317abb75..6a7fb2b0a63 100644 --- a/mindspore/python/mindspore/ops/function/__init__.py +++ b/mindspore/python/mindspore/ops/function/__init__.py @@ -28,6 +28,7 @@ from . import ( ) from .array_func import ( unique, + unique_with_pad, eye, matrix_band_part, padding, diff --git a/mindspore/python/mindspore/ops/function/array_func.py b/mindspore/python/mindspore/ops/function/array_func.py index 7d27ce8660a..302f76be420 100644 --- a/mindspore/python/mindspore/ops/function/array_func.py +++ b/mindspore/python/mindspore/ops/function/array_func.py @@ -39,6 +39,7 @@ fills_ = Fills() ones_ = P.Ones() ones_like_ = P.OnesLike() tile_ = P.Tile() +unique_with_pad_ = P.UniqueWithPad() size_ = P.Size() shape_ = P.Shape() rank_ = P.Rank() @@ -561,6 +562,52 @@ def unique(x): return y, idx +def unique_with_pad(x, pad_num): + """ + Returns unique elements and relative indexes in 1-D tensor, filled with padding num. + + The basic function is the same as the Unique operator, but the UniqueWithPad operator adds a Pad function. + The returned tuple(`y`, `idx`) after the input Tensor `x` is processed by the unique operator, + in which the shapes of `y` and `idx` are mostly not equal. Therefore, in order to solve the above situation, + the UniqueWithPad operator will fill the `y` Tensor with the `pad_num` specified by the user + to make it have the same shape as the Tensor `idx`. + + Args: + x (Tensor): The tensor need to be unique. Must be 1-D vector with types: int32, int64. + pad_num (int): Pad num. The data type is an int. + + Returns: + tuple(Tensor), tuple of 2 tensors, `y` and `idx`. + - y (Tensor) - The unique elements filled with pad_num, the shape and data type same as `x`. + - idx (Tensor) - The index of each value of `x` in the unique output `y`, the shape and data type same as `x`. + + Raises:x + TypeError: If dtype of `x` is neither int32 nor int64. + ValueError: If length of shape of `x` is not equal to 1. + + Supported Platforms: + ``Ascend`` ``GPU`` ``CPU`` + + Examples: + >>> import mindspore + >>> import numpy as np + >>> from mindspore import Tensor, nn + >>> from mindspore import ops + >>> x = Tensor(np.array([1, 2, 5, 2, 3, 5]), mindspore.int32) + >>> output = ops.unique_with_pad(x, 0) + >>> print(output) + (Tensor(shape=[6], dtype=Int32, value= [1, 2, 5, 3, 0, 0]), + Tensor(shape=[6], dtype=Int32, value= [0, 1, 2, 1, 3, 2])) + >>> y = output[0] + >>> print(y) + [1 2 5 3 0 0] + >>> idx = output[1] + >>> print(idx) + [0 1 2 1 3 2] + """ + return unique_with_pad_(x, pad_num) + + def unique_consecutive(x, return_idx=False, return_counts=False, axis=None): """ Returns the elements that are unique in each consecutive group of equivalent elements in the input tensor. diff --git a/mindspore/python/mindspore/ops/functional.py b/mindspore/python/mindspore/ops/functional.py index 798194d2c0f..3b0fc6ed973 100644 --- a/mindspore/python/mindspore/ops/functional.py +++ b/mindspore/python/mindspore/ops/functional.py @@ -964,6 +964,7 @@ tensor_operator_registry.register('soft_shrink', P.SoftShrink) tensor_operator_registry.register('svd', linalg_ops.Svd) tensor_operator_registry.register('diag', P.Diag) tensor_operator_registry.register('unique_consecutive', UniqueConsecutive) +tensor_operator_registry.register('unique_with_pad', P.UniqueWithPad) tensor_operator_registry.register('inplace_update', P.InplaceUpdate) tensor_operator_registry.register('col2im', col2im) tensor_operator_registry.register('standard_laplace', P.StandardLaplace) diff --git a/mindspore/python/mindspore/ops/operations/array_ops.py b/mindspore/python/mindspore/ops/operations/array_ops.py index 64acfd951cc..28ee33265a3 100755 --- a/mindspore/python/mindspore/ops/operations/array_ops.py +++ b/mindspore/python/mindspore/ops/operations/array_ops.py @@ -1217,21 +1217,10 @@ class UniqueWithPad(PrimitiveWithInfer): the UniqueWithPad operator will fill the `y` Tensor with the `pad_num` specified by the user to make it have the same shape as the Tensor `idx`. - Inputs: - - **x** (Tensor) - The tensor need to be unique. Must be 1-D vector with types: int32, int64. - - **pad_num** (int) - Pad num. The data type is an int. - - Outputs: - tuple(Tensor), tuple of 2 tensors, `y` and `idx`. - - y (Tensor) - The unique elements filled with pad_num, the shape and data type same as `x`. - - idx (Tensor) - The index of each value of `x` in the unique output `y`, the shape and data type same as `x`. - - Raises: - TypeError: If dtype of `x` is neither int32 nor int64. - ValueError: If length of shape of `x` is not equal to 1. + Refer to :func:`mindspore.ops.unique_with_pad` for more detail. Supported Platforms: - ``Ascend`` ``CPU`` + ``Ascend`` ``GPU`` ``CPU`` Examples: >>> x = Tensor(np.array([1, 1, 5, 5, 4, 4, 3, 3, 2, 2,]), mindspore.int32)