From 475d2277c24cd0b88df82b567564b896e7262d0c Mon Sep 17 00:00:00 2001 From: reku1997 Date: Thu, 5 Jan 2023 11:30:14 +0800 Subject: [PATCH] fix cpu dropout seed --- .../ops/mindspore.ops.func_dropout.rst | 2 +- .../device/cpu/kernel/dropout_cpu_kernel.cc | 12 ++++- .../device/cpu/kernel/dropout_cpu_kernel.h | 2 + .../python/mindspore/ops/function/nn_func.py | 2 +- tests/st/ops/cpu/test_dropout_op.py | 46 +++++++++++++++++++ 5 files changed, 60 insertions(+), 4 deletions(-) diff --git a/docs/api/api_python/ops/mindspore.ops.func_dropout.rst b/docs/api/api_python/ops/mindspore.ops.func_dropout.rst index f9d519aa531..c5a9b933524 100644 --- a/docs/api/api_python/ops/mindspore.ops.func_dropout.rst +++ b/docs/api/api_python/ops/mindspore.ops.func_dropout.rst @@ -3,7 +3,7 @@ mindspore.ops.dropout .. py:function:: mindspore.ops.dropout(x, p=0.5, seed0=0, seed1=0) - 在训练期间，以服从伯努利分布的概率 `p` 随机将输入Tensor的某些值归零，起到减少神经元相关性的作用，避免过拟合。此概率与 `ops.dropout` 和 `nn.dropout` 中的含义相反。 + 在训练期间，以服从伯努利分布的概率 `p` 随机将输入Tensor的某些值归零，起到减少神经元相关性的作用，避免过拟合。此概率与 `ops.Dropout` 和 `nn.Dropout` 中的含义相反。 参数： - **x** (Tensor) - dropout的输入，任意维度的Tensor，其数据类型为float16或float32。 diff --git a/mindspore/ccsrc/plugin/device/cpu/kernel/dropout_cpu_kernel.cc b/mindspore/ccsrc/plugin/device/cpu/kernel/dropout_cpu_kernel.cc index 01d81ed600d..d4e457cc97a 100644 --- a/mindspore/ccsrc/plugin/device/cpu/kernel/dropout_cpu_kernel.cc +++ b/mindspore/ccsrc/plugin/device/cpu/kernel/dropout_cpu_kernel.cc @@ -51,6 +51,14 @@ bool DropoutCpuKernelMod::Init(const BaseOperatorPtr &base_operator, const std:: if (keep_prob_ <= 0.0 || keep_prob_ > 1.0) { MS_LOG(EXCEPTION) << "For '" << kernel_name_ << ", the 'keep_prob' must be in (0.0, 1.0], but got " << keep_prob_; } + auto seed = GetValue(base_operator->GetAttr("Seed0")); + if (seed == 0) { + seed = GetValue(base_operator->GetAttr("Seed1")); + if (seed == 0) { + seed = time(nullptr); + } + } + seed_ = static_cast(seed); return MatchKernelFunc(base_operator, inputs, outputs); } @@ -78,8 +86,7 @@ bool DropoutCpuKernelMod::LaunchKernel(const std::vector &in auto *output_addr = reinterpret_cast(outputs[0]->addr); auto mask_addr = reinterpret_cast(outputs[1]->addr); T scale = static_cast(1.f / keep_prob_); - std::random_device rd; - std::default_random_engine generator(rd()); + std::default_random_engine generator(seed_ + seed_offset_); std::uniform_real_distribution uniform(0.f, 1.f); auto task = [input_addr, output_addr, mask_addr, scale, &uniform, &generator, this](size_t start, size_t end) { for (size_t i = start; i < end; i++) { @@ -88,6 +95,7 @@ bool DropoutCpuKernelMod::LaunchKernel(const std::vector &in } }; ParallelLaunchAutoSearch(task, tensor_size_, this, ¶llel_search_info_); + seed_offset_ += 1; return true; } diff --git a/mindspore/ccsrc/plugin/device/cpu/kernel/dropout_cpu_kernel.h b/mindspore/ccsrc/plugin/device/cpu/kernel/dropout_cpu_kernel.h index d182d1f5cc3..d072d6e6dee 100644 --- a/mindspore/ccsrc/plugin/device/cpu/kernel/dropout_cpu_kernel.h +++ b/mindspore/ccsrc/plugin/device/cpu/kernel/dropout_cpu_kernel.h @@ -53,6 +53,8 @@ class DropoutCpuKernelMod : public NativeCpuKernelMod, public MatchKernelHelper< ShapeVector input_shape_; float keep_prob_{0.0}; + uint64_t seed_{0}; + uint64_t seed_offset_{0}; size_t tensor_size_{1}; }; } // namespace kernel diff --git a/mindspore/python/mindspore/ops/function/nn_func.py b/mindspore/python/mindspore/ops/function/nn_func.py index c9344d6e975..03fd0a2f32e 100644 --- a/mindspore/python/mindspore/ops/function/nn_func.py +++ b/mindspore/python/mindspore/ops/function/nn_func.py @@ -1071,7 +1071,7 @@ def dropout(x, p=0.5, seed0=0, seed1=0): During training, randomly zeroes some of the elements of the input tensor with probability `p` from a Bernoulli distribution. It plays the role of reducing neuron correlation and avoid overfitting. The meaning of probability - here is opposite to that in `ops.dropout` and `nn.dropout`. + here is opposite to that in `ops.Dropout` and `nn.Dropout`. Args: x (Tensor): The input of Dropout, a Tensor of any shape with data type of float16 or float32. diff --git a/tests/st/ops/cpu/test_dropout_op.py b/tests/st/ops/cpu/test_dropout_op.py index 06b0155fe66..0d3f6465f37 100644 --- a/tests/st/ops/cpu/test_dropout_op.py +++ b/tests/st/ops/cpu/test_dropout_op.py @@ -20,6 +20,7 @@ import mindspore.context as context import mindspore.nn as nn from mindspore import Tensor from mindspore.ops import operations as P +from mindspore import ops context.set_context(mode=context.GRAPH_MODE, device_target='CPU') @@ -87,7 +88,52 @@ def test_net2(): print(mask) +class Net3(nn.Cell): + def __init__(self): + super(Net3, self).__init__() + self.dropout = P.Dropout(keep_prob=0.5) + + def construct(self, x): + return self.dropout(x) + + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_net3(): + """ + Feature: test dropout mask diff by diff step. + Description: dropout. + Expectation: No exception. + """ + x = np.arange(0, 12).reshape(3, 4).astype(np.float16) + dropout = Net3() + output1, mask1 = dropout(Tensor(x)) + output2, mask2 = dropout(Tensor(x)) + assert np.allclose(mask1.asnumpy(), mask2.asnumpy()) is False + assert np.allclose(output1.asnumpy(), output2.asnumpy()) is False + + +@pytest.mark.level0 +@pytest.mark.platform_x86_cpu +@pytest.mark.env_onecard +def test_op1(): + """ + Feature: test dropout mask equal by equal seed. + Description: dropout. + Expectation: No exception. + """ + x = Tensor(np.arange(0, 12).reshape(3, 4).astype(np.float16)) + output1, mask1 = ops.dropout(x, p=0.5, seed0=1, seed1=100) + output2, mask2 = ops.dropout(x, p=0.5, seed0=1, seed1=100) + + assert mask1.shape == mask2.shape + assert np.allclose(output1.asnumpy(), output2.asnumpy()) + assert np.allclose(mask1.asnumpy(), mask2.asnumpy()) + + if __name__ == '__main__': test_net() test_net1() test_net2() + test_op1()