xxq250
/
mindspore
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add DropoutExt/DropoutGenMaskExt/DropoutDoMaskExt 

type: feature
target: ascend
reason:

------

Signed-off-by: wang_ziqi <wangziqi4@huawei.com>
This commit is contained in:
wang_ziqi 2024-04-20 18:05:23 +08:00
parent 4ec81c944a
commit 4ff5fd4d05
39 changed files with 1833 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
mindspore/ccsrc/frontend/expander/bprop/grad_ops/grad_nn_ops.cc
View File

@ -645,6 +645,18 @@ REG_BPROP_BUILDER("Dropout").SetUnusedInputs({i0}).SetBody(BODYFUNC(ib) {
return {dx, ib->OutZeros(keep_prob), ib->OutZeros(seed0), ib->OutZeros(seed1)};
});
REG_BPROP_BUILDER("DropoutExt").SetUnusedInputs({i0}).SetBody(BODYFUNC(ib) {
auto p = ib->GetInput(kIndex1);
auto seed = ib->GetInput(kIndex2);
auto offset = ib->GetInput(kIndex3);
auto out = ib->GetInput(kIndex4);
auto dout = ib->GetInput(kIndex5);
auto mask = ib->TupleGetItem(out, kIndex1);
auto dy = ib->TupleGetItem(dout, kIndex0);
auto dx = ib->Emit("DropoutGradExt", {dy, mask, p});
return {dx, ib->OutZeros(p), ib->OutZeros(seed), ib->OutZeros(offset)};
});
REG_BPROP_BUILDER("BinaryCrossEntropy").SetUnusedInputs({i3}).SetBody(BODYFUNC(ib) {
auto x = ib->GetInput(kIndex0);
auto y = ib->GetInput(kIndex1);

59
mindspore/ccsrc/plugin/device/ascend/kernel/opapi/aclnn/dropout_do_mask_ext_aclnn_kernel.cc Normal file
View File

@ -0,0 +1,59 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "plugin/device/ascend/kernel/opapi/aclnn/dropout_do_mask_ext_aclnn_kernel.h"
#include "ir/tensor.h"
#include "runtime/device/kernel_runtime.h"
namespace mindspore {
namespace kernel {
void DropoutDoMaskExtAscend::GetWorkSpaceInfo(const std::vector<KernelTensor *> &inputs,
const std::vector<KernelTensor *> &outputs) {
MS_EXCEPTION_IF_NULL(primitive_);
if (primitive_->HasAttr("enable_keep_prob") && GetValue<bool>(primitive_->GetAttr("enable_keep_prob"))) {
auto keep_prob_dtype = inputs[kIndex2]->dtype_id();
if (keep_prob_dtype == kNumberTypeFloat16) {
p_value_ = static_cast<double>(1.0) - static_cast<double>(inputs[kIndex2]->GetValueWithCheck<float16>());
} else if (keep_prob_dtype == kNumberTypeBFloat16) {
p_value_ = static_cast<double>(1.0) - static_cast<double>(inputs[kIndex2]->GetValueWithCheck<bfloat16>());
} else if (keep_prob_dtype == kNumberTypeFloat32) {
p_value_ = static_cast<double>(1.0) - static_cast<double>(inputs[kIndex2]->GetValueWithCheck<float>());
} else {
MS_LOG(EXCEPTION)
<< "For 'DropoutDoMaskExt', the 'keep_prob' type must be in [Float16, BFloat16, Float32], but got "
<< TypeIdToString(keep_prob_dtype);
}
} else {
p_value_ = static_cast<double>(inputs[kIndex2]->GetValueWithCheck<float>());
}
MS_LOG(DEBUG) << primitive_->name() << " got a (" + TypeIdToString(inputs[kIndex2]->dtype_id()) + ")p = " << p_value_;
GetWorkspaceForResize(inputs[kIndex0], inputs[kIndex1], p_value_, outputs[kIndex0]);
}
bool DropoutDoMaskExtAscend::Launch(const std::vector<KernelTensor *> &inputs,
const std::vector<KernelTensor *> &workspace,
const std::vector<KernelTensor *> &outputs, void *stream_ptr) {
MS_EXCEPTION_IF_NULL(stream_ptr);
ParseGenExecutor(
GEN_EXECUTOR_BOOST(op_type_, hash_id_, inputs[kIndex0], inputs[kIndex1], p_value_, outputs[kIndex0]));
RunOp(stream_ptr, workspace);
return true;
}
MS_ACLNN_KERNEL_FACTORY_REG(DropoutDoMaskExt, DropoutDoMaskExtAscend);
} // namespace kernel
} // namespace mindspore

42
mindspore/ccsrc/plugin/device/ascend/kernel/opapi/aclnn/dropout_do_mask_ext_aclnn_kernel.h Normal file
View File

@ -0,0 +1,42 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_OPAPI_ACLNN_DROPOUT_DO_MASK_EXT_ACLNN_KERNEL_H_
#define MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_OPAPI_ACLNN_DROPOUT_DO_MASK_EXT_ACLNN_KERNEL_H_
#include <vector>
#include <utility>
#include "ops/base_operator.h"
#include "plugin/device/ascend/kernel/opapi/aclnn_kernel_mod.h"
#include "transform/acl_ir/acl_convert.h"
namespace mindspore {
namespace kernel {
class DropoutDoMaskExtAscend : public AclnnKernelMod {
public:
DropoutDoMaskExtAscend() : AclnnKernelMod(std::move("aclnnDropoutDoMask")) {}
~DropoutDoMaskExtAscend() = default;
bool Launch(const std::vector<KernelTensor *> &inputs, const std::vector<KernelTensor *> &workspace,
const std::vector<KernelTensor *> &outputs, void *stream_ptr) override;
void GetWorkSpaceInfo(const std::vector<KernelTensor *> &inputs, const std::vector<KernelTensor *> &outputs) override;
private:
DEFINE_GET_WORKSPACE_FOR_RESIZE()
double p_value_;
};
} // namespace kernel
} // namespace mindspore
#endif // MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_OPAPI_ACLNN_DROPOUT_DO_MASK_EXT_ACLNN_KERNEL_H_

51
mindspore/ccsrc/plugin/device/ascend/kernel/opapi/aclnn/dropout_ext_aclnn_kernel.cc Normal file
View File

@ -0,0 +1,51 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "plugin/device/ascend/kernel/opapi/aclnn/dropout_ext_aclnn_kernel.h"
#include "ir/tensor.h"
#include "runtime/device/kernel_runtime.h"
namespace mindspore {
namespace kernel {
void DropoutExtAscend::GetWorkSpaceInfo(const std::vector<KernelTensor *> &inputs,
const std::vector<KernelTensor *> &outputs) {
MS_EXCEPTION_IF_NULL(primitive_);
p_value_ = static_cast<double>(inputs[kIndex1]->GetValueWithCheck<float>());
seed_value_ = inputs[kIndex2]->GetValueWithCheck<int64_t>();
offset_value_ = inputs[kIndex3]->GetValueWithCheck<int64_t>();
dtype_value_ = inputs[kIndex0]->dtype_id();
MS_LOG(DEBUG) << primitive_->name() << " got a (" + TypeIdToString(inputs[kIndex1]->dtype_id()) + ")p = " << p_value_;
GetWorkspaceForResize(inputs[kIndex0], p_value_, seed_value_, offset_value_, dtype_value_, outputs[kIndex0],
outputs[kIndex1]);
}
bool DropoutExtAscend::Launch(const std::vector<KernelTensor *> &inputs, const std::vector<KernelTensor *> &workspace,
const std::vector<KernelTensor *> &outputs, void *stream_ptr) {
MS_EXCEPTION_IF_NULL(stream_ptr);
ParseGenExecutor(GEN_EXECUTOR_BOOST(dropout_gen_mask_, gen_mask_hash_id_, inputs[kIndex0]->GetShapeVector(), p_value_,
seed_value_, offset_value_, dtype_value_, outputs[kIndex1]));
RunOp(stream_ptr, workspace);
ParseGenExecutor(GEN_EXECUTOR_BOOST(dropout_do_mask_, do_mask_hash_id_, inputs[kIndex0], outputs[kIndex1], p_value_,
outputs[kIndex0]));
RunOp(stream_ptr, workspace);
return true;
}
MS_ACLNN_KERNEL_FACTORY_REG(DropoutExt, DropoutExtAscend);
} // namespace kernel
} // namespace mindspore

50
mindspore/ccsrc/plugin/device/ascend/kernel/opapi/aclnn/dropout_ext_aclnn_kernel.h Normal file
View File

@ -0,0 +1,50 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_OPAPI_ACLNN_DROPOUT_EXT_ACLNN_KERNEL_H_
#define MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_OPAPI_ACLNN_DROPOUT_EXT_ACLNN_KERNEL_H_
#include <vector>
#include <string>
#include <utility>
#include "ops/base_operator.h"
#include "plugin/device/ascend/kernel/opapi/aclnn_kernel_mod.h"
#include "transform/acl_ir/acl_convert.h"
namespace mindspore {
namespace kernel {
class DropoutExtAscend : public AclnnKernelMod {
public:
DropoutExtAscend() : AclnnKernelMod(std::move("aclnnDropout")) {}
~DropoutExtAscend() = default;
bool Launch(const std::vector<KernelTensor *> &inputs, const std::vector<KernelTensor *> &workspace,
const std::vector<KernelTensor *> &outputs, void *stream_ptr) override;
void GetWorkSpaceInfo(const std::vector<KernelTensor *> &inputs, const std::vector<KernelTensor *> &outputs) override;
private:
DEFINE_GET_WORKSPACE_FOR_RESIZE()
const std::string dropout_gen_mask_{"aclnnDropoutGenMaskV2"};
const std::string dropout_do_mask_{"aclnnDropoutDoMask"};
uint64_t gen_mask_hash_id_{0};
uint64_t do_mask_hash_id_{0};
double p_value_;
int64_t seed_value_;
int64_t offset_value_;
TypeId dtype_value_;
};
} // namespace kernel
} // namespace mindspore
#endif // MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_OPAPI_ACLNN_DROPOUT_EXT_ACLNN_KERNEL_H_

64
mindspore/ccsrc/plugin/device/ascend/kernel/opapi/aclnn/dropout_gen_mask_ext_aclnn_kernel.cc Normal file
View File

@ -0,0 +1,64 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "plugin/device/ascend/kernel/opapi/aclnn/dropout_gen_mask_ext_aclnn_kernel.h"
#include "ir/tensor.h"
#include "runtime/device/kernel_runtime.h"
namespace mindspore {
namespace kernel {
void DropoutGenMaskExtAscend::GetWorkSpaceInfo(const std::vector<KernelTensor *> &inputs,
const std::vector<KernelTensor *> &outputs) {
MS_EXCEPTION_IF_NULL(primitive_);
if (primitive_->HasAttr("enable_keep_prob") && GetValue<bool>(primitive_->GetAttr("enable_keep_prob"))) {
auto keep_prob_dtype = inputs[kIndex1]->dtype_id();
if (keep_prob_dtype == kNumberTypeFloat16) {
p_value_ = static_cast<double>(1.0) - static_cast<double>(inputs[kIndex1]->GetValueWithCheck<float16>());
} else if (keep_prob_dtype == kNumberTypeBFloat16) {
p_value_ = static_cast<double>(1.0) - static_cast<double>(inputs[kIndex1]->GetValueWithCheck<bfloat16>());
} else if (keep_prob_dtype == kNumberTypeFloat32) {
p_value_ = static_cast<double>(1.0) - static_cast<double>(inputs[kIndex1]->GetValueWithCheck<float>());
} else {
MS_LOG(EXCEPTION)
<< "For 'DropoutGenMaskExt', the 'keep_prob' type must be in [Float16, BFloat16, Float32], but got "
<< TypeIdToString(keep_prob_dtype);
}
} else {
p_value_ = static_cast<double>(inputs[kIndex1]->GetValueWithCheck<float>());
}
MS_LOG(DEBUG) << primitive_->name() << " got a (" + TypeIdToString(inputs[kIndex1]->dtype_id()) + ")p = " << p_value_;
shape_ = transform::ConvertKernelTensor<ShapeVector>(inputs[kIndex0]);
seed_value_ = inputs[kIndex2]->GetValueWithCheck<int64_t>();
offset_value_ = inputs[kIndex3]->GetValueWithCheck<int64_t>();
dtype_value_ = static_cast<TypeId>(inputs[kIndex4]->GetValueWithCheck<int64_t>());
GetWorkspaceForResize(shape_, p_value_, seed_value_, offset_value_, dtype_value_, outputs[kIndex0]);
}
bool DropoutGenMaskExtAscend::Launch(const std::vector<KernelTensor *> &inputs,
const std::vector<KernelTensor *> &workspace,
const std::vector<KernelTensor *> &outputs, void *stream_ptr) {
MS_EXCEPTION_IF_NULL(stream_ptr);
ParseGenExecutor(GEN_EXECUTOR_BOOST(op_type_, hash_id_, shape_, p_value_, seed_value_, offset_value_, dtype_value_,
outputs[kIndex0]));
RunOp(stream_ptr, workspace);
return true;
}
MS_ACLNN_KERNEL_FACTORY_REG(DropoutGenMaskExt, DropoutGenMaskExtAscend);
} // namespace kernel
} // namespace mindspore

46
mindspore/ccsrc/plugin/device/ascend/kernel/opapi/aclnn/dropout_gen_mask_ext_aclnn_kernel.h Normal file
View File

@ -0,0 +1,46 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_OPAPI_ACLNN_DROPOUT_GEN_MASK_EXT_ACLNN_KERNEL_H_
#define MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_OPAPI_ACLNN_DROPOUT_GEN_MASK_EXT_ACLNN_KERNEL_H_
#include <vector>
#include <utility>
#include "ops/base_operator.h"
#include "plugin/device/ascend/kernel/opapi/aclnn_kernel_mod.h"
#include "transform/acl_ir/acl_convert.h"
namespace mindspore {
namespace kernel {
class DropoutGenMaskExtAscend : public AclnnKernelMod {
public:
DropoutGenMaskExtAscend() : AclnnKernelMod(std::move("aclnnDropoutGenMaskV2")) {}
~DropoutGenMaskExtAscend() = default;
bool Launch(const std::vector<KernelTensor *> &inputs, const std::vector<KernelTensor *> &workspace,
const std::vector<KernelTensor *> &outputs, void *stream_ptr) override;
void GetWorkSpaceInfo(const std::vector<KernelTensor *> &inputs, const std::vector<KernelTensor *> &outputs) override;
private:
DEFINE_GET_WORKSPACE_FOR_RESIZE()
ShapeVector shape_;
double p_value_;
int64_t seed_value_;
int64_t offset_value_;
TypeId dtype_value_;
};
} // namespace kernel
} // namespace mindspore
#endif // MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_OPAPI_ACLNN_DROPOUT_GEN_MASK_EXT_ACLNN_KERNEL_H_

44
mindspore/ccsrc/plugin/device/ascend/kernel/opapi/aclnn/dropout_grad_ext_aclnn_kernel.cc Normal file
View File

@ -0,0 +1,44 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "plugin/device/ascend/kernel/opapi/aclnn/dropout_grad_ext_aclnn_kernel.h"
#include "ir/tensor.h"
#include "runtime/device/kernel_runtime.h"
namespace mindspore {
namespace kernel {
void DropoutGradExtAscend::GetWorkSpaceInfo(const std::vector<KernelTensor *> &inputs,
const std::vector<KernelTensor *> &outputs) {
MS_EXCEPTION_IF_NULL(primitive_);
p_value_ = static_cast<double>(inputs[kIndex2]->GetValueWithCheck<float>());
MS_LOG(DEBUG) << primitive_->name() << " got a (" + TypeIdToString(inputs[kIndex2]->dtype_id()) + ")p = " << p_value_;
GetWorkspaceForResize(inputs[kIndex0], inputs[kIndex1], p_value_, outputs[kIndex0]);
}
bool DropoutGradExtAscend::Launch(const std::vector<KernelTensor *> &inputs,
const std::vector<KernelTensor *> &workspace,
const std::vector<KernelTensor *> &outputs, void *stream_ptr) {
MS_EXCEPTION_IF_NULL(stream_ptr);
ParseGenExecutor(GEN_EXECUTOR_BOOST(dropout_do_mask_, do_mask_hash_id_, inputs[kIndex0], inputs[kIndex1], p_value_,
outputs[kIndex0]));
RunOp(stream_ptr, workspace);
return true;
}
MS_ACLNN_KERNEL_FACTORY_REG(DropoutGradExt, DropoutGradExtAscend);
} // namespace kernel
} // namespace mindspore

45
mindspore/ccsrc/plugin/device/ascend/kernel/opapi/aclnn/dropout_grad_ext_aclnn_kernel.h Normal file
View File

@ -0,0 +1,45 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_OPAPI_ACLNN_DROPOUT_GRAD_EXT_ACLNN_KERNEL_H_
#define MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_OPAPI_ACLNN_DROPOUT_GRAD_EXT_ACLNN_KERNEL_H_
#include <string>
#include <vector>
#include <utility>
#include "ops/base_operator.h"
#include "plugin/device/ascend/kernel/opapi/aclnn_kernel_mod.h"
#include "transform/acl_ir/acl_convert.h"
namespace mindspore {
namespace kernel {
class DropoutGradExtAscend : public AclnnKernelMod {
public:
DropoutGradExtAscend() : AclnnKernelMod(std::move("aclnnDropoutBackward")) {}
~DropoutGradExtAscend() = default;
bool Launch(const std::vector<KernelTensor *> &inputs, const std::vector<KernelTensor *> &workspace,
const std::vector<KernelTensor *> &outputs, void *stream_ptr) override;
void GetWorkSpaceInfo(const std::vector<KernelTensor *> &inputs, const std::vector<KernelTensor *> &outputs) override;
private:
DEFINE_GET_WORKSPACE_FOR_RESIZE()
const std::string dropout_do_mask_{"aclnnDropoutDoMask"};
uint64_t do_mask_hash_id_{0};
double p_value_;
};
} // namespace kernel
} // namespace mindspore
#endif // MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_OPAPI_ACLNN_DROPOUT_GRAD_EXT_ACLNN_KERNEL_H_

48
mindspore/ccsrc/plugin/device/ascend/kernel/pyboost/customize/dropout_do_mask_ext.cc Normal file
View File

@ -0,0 +1,48 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "plugin/device/ascend/kernel/pyboost/customize/dropout_do_mask_ext.h"
#include "plugin/device/ascend/hal/device/ascend_stream_manager.h"
#include "kernel/pyboost/pyboost_utils.h"
#include "plugin/device/ascend/kernel/pyboost/aclnn_utils.h"
#include "runtime/device/device_address_utils.h"
namespace mindspore {
namespace kernel {
namespace pyboost {
tensor::BaseTensorPtr DropoutDoMaskExtAscendCustomize(const std::shared_ptr<OpRunner> &op, const BaseTensorPtr &input,
const BaseTensorPtr &mask, const FP32ImmPtr &p) {
OpRunner::InferOpOutput(op, input, mask, p);
// Create device address for input/output tensors.
PyBoostUtils::PrepareOpInputs(op->device_context(), op->stream_id(), input, mask);
PyBoostUtils::PrepareOpOutputs(op->device_context(), op->stream_id(), op->outputs());
// Async
PyBoostUtils::DispatchRun(std::make_shared<runtime::PyBoostDeviceTask>([op, input, mask, p]() {
auto device_context = op->device_context();
const auto &outputs = op->outputs();
// Malloc for input tensors
PyBoostUtils::MallocOpInputs(op->device_context(), input, mask);
// Malloc for output tensors
PyBoostUtils::MallocOpOutputs(op->device_context(), outputs);
auto p_value = static_cast<double>(p->value());
LAUNCH_ACLNN(aclnnDropoutDoMask, device_context, op->stream_id(), input, mask, p_value, outputs[kIndex0]);
}));
return op->output(0);
}
} // namespace pyboost
} // namespace kernel
} // namespace mindspore

34
mindspore/ccsrc/plugin/device/ascend/kernel/pyboost/customize/dropout_do_mask_ext.h Normal file
View File

@ -0,0 +1,34 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_PYBOOST_DROPOUT_DO_MASK_EXT_H_
#define MINDSPORE_MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_PYBOOST_DROPOUT_DO_MASK_EXT_H_
#include <memory>
#include <vector>
#include "ir/tensor.h"
#include "ir/value.h"
#include "kernel/pyboost/op_runner.h"
namespace mindspore {
namespace kernel {
namespace pyboost {
tensor::BaseTensorPtr DropoutDoMaskExtAscendCustomize(const std::shared_ptr<OpRunner> &op, const BaseTensorPtr &input,
const BaseTensorPtr &mask, const FP32ImmPtr &p);
} // namespace pyboost
} // namespace kernel
} // namespace mindspore
#endif // MINDSPORE_MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_PYBOOST_DROPOUT_DO_MASK_EXT_H_

59
mindspore/ccsrc/plugin/device/ascend/kernel/pyboost/customize/dropout_ext.cc Normal file
View File

@ -0,0 +1,59 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "plugin/device/ascend/kernel/pyboost/customize/dropout_ext.h"
#include "plugin/device/ascend/hal/device/ascend_stream_manager.h"
#include "kernel/pyboost/pyboost_utils.h"
#include "plugin/device/ascend/kernel/pyboost/aclnn_utils.h"
#include "runtime/device/device_address_utils.h"
namespace mindspore {
namespace kernel {
namespace pyboost {
tensor::BaseTensorPtr DropoutExtAscendCustomize(const std::shared_ptr<OpRunner> &op, const BaseTensorPtr &input,
const FP32ImmPtr &p, const Int64ImmPtr &seed,
const Int64ImmPtr &offset) {
OpRunner::InferOpOutput(op, input, p, seed, offset);
// Create device address for input/output tensors.
PyBoostUtils::PrepareOpInputs(op->device_context(), op->stream_id(), input);
PyBoostUtils::PrepareOpOutputs(op->device_context(), op->stream_id(), op->outputs());
// Async
PyBoostUtils::DispatchRun(std::make_shared<runtime::PyBoostDeviceTask>([op, input, p, seed, offset]() {
auto device_context = op->device_context();
const auto &outputs = op->outputs();
// Malloc for input tensors
PyBoostUtils::MallocOpInputs(op->device_context(), input);
// Malloc for output tensors
PyBoostUtils::MallocOpOutputs(op->device_context(), outputs);
auto p_value = static_cast<double>(p->value());
auto seed_value = static_cast<int64_t>(seed->value());
auto offset_value = static_cast<int64_t>(offset->value());
std::vector<int64_t> input_shape = input->shape();
auto tensor_type = op->input_abs()[kIndex0]->GetType()->cast<TensorTypePtr>();
MS_EXCEPTION_IF_NULL(tensor_type);
auto dtype = tensor_type->element()->type_id();
MS_LOG(DEBUG) << "DropoutExt input dtype = " << TypeIdToString(dtype);
LAUNCH_ACLNN(aclnnDropoutGenMaskV2, device_context, op->stream_id(), input_shape, p_value, seed_value, offset_value,
dtype, outputs[kIndex1]);
LAUNCH_ACLNN(aclnnDropoutDoMask, device_context, op->stream_id(), input, outputs[kIndex1], p_value,
outputs[kIndex0]);
}));
return op->output(0);
}
} // namespace pyboost
} // namespace kernel
} // namespace mindspore

35
mindspore/ccsrc/plugin/device/ascend/kernel/pyboost/customize/dropout_ext.h Normal file
View File

@ -0,0 +1,35 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_PYBOOST_DROPOUT_EXT_H_
#define MINDSPORE_MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_PYBOOST_DROPOUT_EXT_H_
#include <memory>
#include <vector>
#include "ir/tensor.h"
#include "ir/value.h"
#include "kernel/pyboost/op_runner.h"
namespace mindspore {
namespace kernel {
namespace pyboost {
tensor::BaseTensorPtr DropoutExtAscendCustomize(const std::shared_ptr<OpRunner> &op, const BaseTensorPtr &input,
const FP32ImmPtr &p, const Int64ImmPtr &seed,
const Int64ImmPtr &offset);
} // namespace pyboost
} // namespace kernel
} // namespace mindspore
#endif // MINDSPORE_MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_PYBOOST_DROPOUT_EXT_H_

51
mindspore/ccsrc/plugin/device/ascend/kernel/pyboost/customize/dropout_gen_mask_ext.cc Normal file
View File

@ -0,0 +1,51 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "plugin/device/ascend/kernel/pyboost/customize/dropout_gen_mask_ext.h"
#include "plugin/device/ascend/hal/device/ascend_stream_manager.h"
#include "kernel/pyboost/pyboost_utils.h"
#include "plugin/device/ascend/kernel/pyboost/aclnn_utils.h"
#include "runtime/device/device_address_utils.h"
namespace mindspore {
namespace kernel {
namespace pyboost {
tensor::BaseTensorPtr DropoutGenMaskExtAscendCustomize(const std::shared_ptr<OpRunner> &op, const ValueTuplePtr &shape,
const FP32ImmPtr &p, const Int64ImmPtr &seed,
const Int64ImmPtr &offset, const Int64ImmPtr &dtype) {
OpRunner::InferOpOutput(op, shape, p, seed, offset, dtype);
auto shape_vector = ConvertValueTupleToVector<int64_t>(shape);
// Create device address for output tensors.
PyBoostUtils::PrepareOpOutputs(op->device_context(), op->stream_id(), op->outputs());
// Async
PyBoostUtils::DispatchRun(std::make_shared<runtime::PyBoostDeviceTask>([op, shape_vector, p, seed, offset, dtype]() {
auto device_context = op->device_context();
const auto &outputs = op->outputs();
// Malloc for output tensors
PyBoostUtils::MallocOpOutputs(op->device_context(), outputs);
auto p_value = static_cast<double>(p->value());
auto seed_value = static_cast<int64_t>(seed->value());
auto offset_value = static_cast<int64_t>(offset->value());
auto dtype_value = static_cast<TypeId>(dtype->value());
LAUNCH_ACLNN(aclnnDropoutGenMaskV2, device_context, op->stream_id(), shape_vector, p_value, seed_value,
offset_value, dtype_value, outputs[kIndex0]);
}));
return op->output(0);
}
} // namespace pyboost
} // namespace kernel
} // namespace mindspore

35
mindspore/ccsrc/plugin/device/ascend/kernel/pyboost/customize/dropout_gen_mask_ext.h Normal file
View File

@ -0,0 +1,35 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_PYBOOST_DROPOUT_GEN_MASK_EXT_H_
#define MINDSPORE_MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_PYBOOST_DROPOUT_GEN_MASK_EXT_H_
#include <memory>
#include <vector>
#include "ir/tensor.h"
#include "ir/value.h"
#include "kernel/pyboost/op_runner.h"
namespace mindspore {
namespace kernel {
namespace pyboost {
tensor::BaseTensorPtr DropoutGenMaskExtAscendCustomize(const std::shared_ptr<OpRunner> &op, const ValueTuplePtr &shape,
const FP32ImmPtr &p, const Int64ImmPtr &seed,
const Int64ImmPtr &offset, const Int64ImmPtr &dtype);
} // namespace pyboost
} // namespace kernel
} // namespace mindspore
#endif // MINDSPORE_MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_PYBOOST_DROPOUT_GEN_MASK_EXT_H_

48
mindspore/ccsrc/plugin/device/ascend/kernel/pyboost/customize/dropout_grad_ext.cc Normal file
View File

@ -0,0 +1,48 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "plugin/device/ascend/kernel/pyboost/customize/dropout_grad_ext.h"
#include "plugin/device/ascend/hal/device/ascend_stream_manager.h"
#include "kernel/pyboost/pyboost_utils.h"
#include "plugin/device/ascend/kernel/pyboost/aclnn_utils.h"
#include "runtime/device/device_address_utils.h"
namespace mindspore {
namespace kernel {
namespace pyboost {
tensor::BaseTensorPtr DropoutGradExtAscendCustomize(const std::shared_ptr<OpRunner> &op, const BaseTensorPtr &input,
const BaseTensorPtr &mask, const FP32ImmPtr &p) {
OpRunner::InferOpOutput(op, input, mask, p);
// Create device address for input/output tensors.
PyBoostUtils::PrepareOpInputs(op->device_context(), op->stream_id(), input, mask);
PyBoostUtils::PrepareOpOutputs(op->device_context(), op->stream_id(), op->outputs());
// Async
PyBoostUtils::DispatchRun(std::make_shared<runtime::PyBoostDeviceTask>([op, input, mask, p]() {
auto device_context = op->device_context();
const auto &outputs = op->outputs();
// Malloc for input tensors
PyBoostUtils::MallocOpInputs(op->device_context(), input, mask);
// Malloc for output tensors
PyBoostUtils::MallocOpOutputs(op->device_context(), outputs);
auto p_value = static_cast<double>(p->value());
LAUNCH_ACLNN(aclnnDropoutDoMask, device_context, op->stream_id(), input, mask, p_value, outputs[kIndex0]);
}));
return op->output(0);
}
} // namespace pyboost
} // namespace kernel
} // namespace mindspore

34
mindspore/ccsrc/plugin/device/ascend/kernel/pyboost/customize/dropout_grad_ext.h Normal file
View File

@ -0,0 +1,34 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_PYBOOST_DROPOUT_GRAD_EXT_H_
#define MINDSPORE_MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_PYBOOST_DROPOUT_GRAD_EXT_H_
#include <memory>
#include <vector>
#include "ir/tensor.h"
#include "ir/value.h"
#include "kernel/pyboost/op_runner.h"
namespace mindspore {
namespace kernel {
namespace pyboost {
tensor::BaseTensorPtr DropoutGradExtAscendCustomize(const std::shared_ptr<OpRunner> &op, const BaseTensorPtr &input,
const BaseTensorPtr &mask, const FP32ImmPtr &p);
} // namespace pyboost
} // namespace kernel
} // namespace mindspore
#endif // MINDSPORE_MINDSPORE_CCSRC_PLUGIN_DEVICE_ASCEND_KERNEL_PYBOOST_DROPOUT_GRAD_EXT_H_

2
mindspore/ccsrc/plugin/device/ascend/optimizer/backend_common_unify_mindir.cc
View File

@ -102,6 +102,8 @@ void GetBackendCommonUnifyMindIRPassManager(PassManagerPtr *unify_mindir_pm) {
(*unify_mindir_pm)->AddPass(std::make_shared<opt::PynativeSparseSoftmaxCrossEntropyWithLogitsUnifyMindIR>());
}
(*unify_mindir_pm)->AddPass(std::make_shared<opt::DropoutExtUnifyMindIR1>());
(*unify_mindir_pm)->AddPass(std::make_shared<opt::DropoutGradExtUnifyMindIR>());
(*unify_mindir_pm)->AddPass(std::make_shared<opt::DropoutUnifyMindIR1>());
(*unify_mindir_pm)->AddPass(std::make_shared<opt::DropoutGradUnifyMindIR>());

160
mindspore/ccsrc/plugin/device/ascend/optimizer/mindir/dropout_unify_mindir.cc
View File

@ -47,6 +47,7 @@ constexpr int64_t kMaskAlignNum = 128;
constexpr int64_t kMaskMultiNum = 16;
constexpr int64_t kV3ShapeLimitSize = 1 << 30;
constexpr size_t kDropoutGradInputTensorNum = 2;
constexpr size_t kDropoutGradExtInputTensorNum = 3;
constexpr size_t kFloat16Len = 2; // size of float16
constexpr size_t kInt64Len = 8; // size of int64
constexpr auto kX1 = "X1";
@ -71,19 +72,34 @@ TypeId GetOriginInputXDataType(const AnfNodePtr &node) {
return common::AnfAlgo::GetPrevNodeOutputInferDataType(node, 0);
}
ValueNodePtr CreateKeepProbValueNode(const FuncGraphPtr &func_graph, const AnfNodePtr &node, TypeId type_id) {
ValueNodePtr CreateKeepProbValueNode(const FuncGraphPtr &func_graph, const AnfNodePtr &node, TypeId type_id,
bool enable_keep_prob = false) {
MS_EXCEPTION_IF_NULL(func_graph);
MS_EXCEPTION_IF_NULL(node);
auto cnode = node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(cnode);
// Step1: get keep_prob
float keep_prob = 0.5;
if (common::AnfAlgo::GetCNodeName(cnode) == kDropoutOpName) {
auto keep_prob_v = cnode->input(kIndex2)->cast<ValueNodePtr>()->value();
keep_prob = ops::GetScalarValue<float>(keep_prob_v).value();
auto cnode_name = common::AnfAlgo::GetCNodeName(cnode);
if (cnode_name == kDropoutOpName || cnode_name == prim::kPrimDropoutExt->name()) {
auto keep_prob_v = cnode->input(kIndex2)->cast<ValueNodePtr>();
MS_EXCEPTION_IF_NULL(keep_prob_v);
auto keep_prob_opt = ops::GetScalarValue<float>(keep_prob_v->value());
MS_EXCEPTION_IF_CHECK_FAIL(keep_prob_opt.has_value(), "can't get keep_prob value from " + cnode_name);
keep_prob = keep_prob_opt.value();
} else if (cnode_name == prim::kPrimDropoutGradExt->name()) {
auto keep_prob_v = cnode->input(kIndex3)->cast<ValueNodePtr>();
MS_EXCEPTION_IF_NULL(keep_prob_v);
auto keep_prob_opt = ops::GetScalarValue<float>(keep_prob_v->value());
MS_EXCEPTION_IF_CHECK_FAIL(keep_prob_opt.has_value(), "can't get keep_prob value from " + cnode_name);
keep_prob = keep_prob_opt.value();
} else {
keep_prob = common::AnfAlgo::GetNodeAttr<float>(node, kAttrKeepProb);
}
if (enable_keep_prob) {
keep_prob = static_cast<float>(1.0) - keep_prob;
}
MS_LOG(DEBUG) << "Keep_prob value: " << keep_prob;
std::vector<int64_t> keep_prob_shape = {};
@ -187,7 +203,8 @@ CNodePtr GetRecomputeDropoutGenMask(const FuncGraphPtr &func_graph, const CNodeP
auto recompute_id = GetValue<int64_t>(dropout->GetAttr(kAttrRecomputeId));
const auto &node_list = TopoSort(func_graph->get_return());
auto find_recompute_genmask = [recompute_id](const AnfNodePtr &node) {
if (!node->isa<CNode>() || !IsOneOfPrimitiveCNode(node, {prim::kPrimDropoutGenMask, prim::kPrimDropoutGenMaskV3})) {
if (!node->isa<CNode>() || !IsOneOfPrimitiveCNode(node, {prim::kPrimDropoutGenMask, prim::kPrimDropoutGenMaskV3,
prim::kPrimDropoutGenMaskExt})) {
return false;
}
auto recompute_id_val = node->cast<CNodePtr>()->GetAttr(kAttrRecomputeId);
@ -203,7 +220,7 @@ CNodePtr GetRecomputeDropoutGenMask(const FuncGraphPtr &func_graph, const CNodeP
CNodePtr CreateDropoutGenMaskCNode(const FuncGraphPtr &func_graph, const CNodePtr &dropout,
const ValueNodePtr &keep_prob_value, const abstract::ShapePtr &input_shape,
const bool use_v3) {
const bool use_v3, bool enable_keep_prob = false) {
MS_EXCEPTION_IF_NULL(func_graph);
MS_EXCEPTION_IF_NULL(dropout);
MS_EXCEPTION_IF_NULL(input_shape);
@ -220,6 +237,14 @@ CNodePtr CreateDropoutGenMaskCNode(const FuncGraphPtr &func_graph, const CNodePt
dropout_gen_mask_inputs.push_back(shape_value);
dropout_gen_mask_inputs.push_back(keep_prob_value);
}
if (enable_keep_prob) {
dropout_gen_mask_inputs[kIndex0] = NewValueNode(std::make_shared<Primitive>(prim::kPrimDropoutGenMaskExt->name()));
dropout_gen_mask_inputs.push_back(dropout->input(kIndex3)); // seed
dropout_gen_mask_inputs.push_back(dropout->input(kIndex4)); // offset
auto dtype_value = static_cast<int64_t>(GetInputXDataType(dropout));
auto dtype_node = CreateValueNodeWithKernelInfo(func_graph, std::make_shared<Int64Imm>(dtype_value));
dropout_gen_mask_inputs.push_back(dtype_node); // dtype
}
CNodePtr dropout_gen_mask = opt::NewCNode(dropout_gen_mask_inputs, func_graph, {dropout});
MS_EXCEPTION_IF_NULL(dropout_gen_mask);
if (dropout->HasPrimalAttr(kAttrFusion)) {
@ -232,10 +257,14 @@ CNodePtr CreateDropoutGenMaskCNode(const FuncGraphPtr &func_graph, const CNodePt
dropout_gen_mask->set_scope(dropout->scope());
common::AnfAlgo::CopyNodeAttrs(dropout, dropout_gen_mask);
auto dropout_gen_mask_primitive = common::AnfAlgo::GetCNodePrimitive(dropout_gen_mask);
if (enable_keep_prob) {
dropout_gen_mask_primitive->set_attr("enable_keep_prob", MakeValue(true));
} else {
auto seed0 = dropout->input(kIndex3)->cast<ValueNodePtr>()->value();
auto seed1 = dropout->input(kIndex4)->cast<ValueNodePtr>()->value();
dropout_gen_mask_primitive->set_attr(kAttrSeed0, seed0);
dropout_gen_mask_primitive->set_attr(kAttrSeed1, seed1);
}
if (dropout->HasPrimalAttr(kAttrMicro)) {
dropout_gen_mask->AddPrimalAttr(kAttrMicro, dropout->GetPrimalAttr(kAttrMicro));
}
@ -247,7 +276,7 @@ CNodePtr CreateDropoutGenMaskCNode(const FuncGraphPtr &func_graph, const CNodePt
CNodePtr CreateDropoutDoMaskCNode(const FuncGraphPtr &func_graph, const CNodePtr &dropout,
const std::vector<AnfNodePtr> &inputs, const abstract::AbstractBasePtr &abstract,
const bool use_v3) {
const bool use_v3, bool enable_keep_prob = false) {
MS_EXCEPTION_IF_NULL(func_graph);
MS_EXCEPTION_IF_NULL(dropout);
std::vector<AnfNodePtr> dropout_do_mask_inputs =
@ -255,6 +284,11 @@ CNodePtr CreateDropoutDoMaskCNode(const FuncGraphPtr &func_graph, const CNodePtr
inputs[kIndex1]}
: std::vector<AnfNodePtr>{NewValueNode(std::make_shared<Primitive>(kDropoutDoMaskOpName)), inputs[kIndex0],
inputs[kIndex1], inputs[kIndex2]};
if (enable_keep_prob) {
dropout_do_mask_inputs =
std::vector<AnfNodePtr>{NewValueNode(std::make_shared<Primitive>(prim::kPrimDropoutDoMaskExt->name())),
inputs[kIndex0], inputs[kIndex1], inputs[kIndex2]};
}
auto dropout_do_mask = opt::NewCNode(dropout_do_mask_inputs, func_graph, {dropout});
MS_EXCEPTION_IF_NULL(dropout_do_mask);
dropout_do_mask->set_abstract(abstract);
@ -266,6 +300,10 @@ CNodePtr CreateDropoutDoMaskCNode(const FuncGraphPtr &func_graph, const CNodePtr
dropout_do_mask->AddPrimalAttr(primal_attr, dropout->GetPrimalAttr(primal_attr));
}
}
if (enable_keep_prob) {
auto dropout_do_mask_primitive = common::AnfAlgo::GetCNodePrimitive(dropout_do_mask);
dropout_do_mask_primitive->set_attr("enable_keep_prob", MakeValue(true));
}
return dropout_do_mask;
}
@ -593,9 +631,9 @@ const AnfNodePtr DropoutUnifyMindIR1::Process(const FuncGraphPtr &func_graph, co
dropout_gen_mask = GetRecomputeDropoutGenMask(func_graph, dropout_cnode);
}
if (dropout_gen_mask == nullptr) {
dropout_gen_mask = CreateDropoutGenMaskCNode(func_graph, dropout_cnode,
CreateKeepProbValueNode(func_graph, dropout_cnode, inputx_type_id),
input_shape, use_v3);
dropout_gen_mask = CreateDropoutGenMaskCNode(
func_graph, dropout_cnode, CreateKeepProbValueNode(func_graph, dropout_cnode, inputx_type_id, enable_keep_prob_),
input_shape, use_v3, enable_keep_prob_);
}
// CreateDropoutDoMask
@ -603,10 +641,11 @@ const AnfNodePtr DropoutUnifyMindIR1::Process(const FuncGraphPtr &func_graph, co
inputx_type_id = kNumberTypeBFloat16;
}
auto do_mask_abstract = std::make_shared<abstract::AbstractTensor>(TypeIdToType(inputx_type_id), input_shape);
auto dropout_do_mask = CreateDropoutDoMaskCNode(
func_graph, dropout_cnode,
{dropout_input, dropout_gen_mask, CreateKeepProbValueNode(func_graph, dropout_cnode, inputx_type_id)},
do_mask_abstract, use_v3);
auto dropout_do_mask =
CreateDropoutDoMaskCNode(func_graph, dropout_cnode,
{dropout_input, dropout_gen_mask,
CreateKeepProbValueNode(func_graph, dropout_cnode, inputx_type_id, enable_keep_prob_)},
do_mask_abstract, use_v3, enable_keep_prob_);
std::vector<AnfNodePtr> make_tuple_inputs{NewValueNode(prim::kPrimMakeTuple), dropout_do_mask, dropout_gen_mask};
auto make_tuple = func_graph->NewCNode(make_tuple_inputs);
@ -631,4 +670,97 @@ void DropoutGradUnifyMindIR::DefineDstPattern(DstPattern *dst_pattern) {
.AddCNode(kRDropoutDoMask, {std::make_shared<Primitive>(kDropoutDoMaskOpName), kX1, kX2, kKeepProbValue},
BuildDropoutDoMask);
}
const BaseRef DropoutExtUnifyMindIR1::DefinePattern() const {
VarPtr X = std::make_shared<Var>();
VarPtr P = std::make_shared<Var>();
VarPtr SEED = std::make_shared<Var>();
VarPtr OFFSET = std::make_shared<Var>();
return VectorRef({prim::kPrimDropoutExt, X, P, SEED, OFFSET});
}
std::vector<std::string> DropoutExtUnifyMindIR1::MustExistPrimitiveName() const {
std::vector<std::string> ret;
ret.emplace_back(prim::kPrimDropoutExt->name());
return ret;
}
const BaseRef DropoutGradExtUnifyMindIR::DefinePattern() const {
VarPtr X = std::make_shared<Var>();
VarPtr M = std::make_shared<Var>();
VarPtr P = std::make_shared<Var>();
return VectorRef({prim::kPrimDropoutGradExt, X, M, P});
}
const AnfNodePtr DropoutGradExtUnifyMindIR::Process(const FuncGraphPtr &func_graph, const AnfNodePtr &node,
const EquivPtr &) const {
MS_EXCEPTION_IF_NULL(func_graph);
MS_EXCEPTION_IF_NULL(node);
auto dropout_grad_cnode = node->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(dropout_grad_cnode);
bool enable_keep_prob = true;
CheckCNodeInputSize(dropout_grad_cnode, kDropoutGradExtInputTensorNum);
auto grad_input_type_id = GetOriginInputXDataType(dropout_grad_cnode);
if (grad_input_type_id != kNumberTypeBFloat16) {
grad_input_type_id = GetInputXDataType(dropout_grad_cnode);
}
auto keep_prob_value = CreateKeepProbValueNode(func_graph, dropout_grad_cnode, grad_input_type_id, enable_keep_prob);
auto grad_input_shape = GetDropoutInputShape(dropout_grad_cnode->input(kIndex1));
auto use_v3 = WhetherUseDropoutV3(dropout_grad_cnode, grad_input_shape);
// DropoutGrad may not in the same graph with Dropout in heterogeneous scene, and mask input which is a parameter
// in that scene, need to be updated.
auto mask_input = dropout_grad_cnode->input(kIndex2);
MS_EXCEPTION_IF_NULL(mask_input);
if (mask_input->isa<Parameter>()) {
// update abstract
auto mask_abstract = mask_input->abstract();
MS_EXCEPTION_IF_NULL(mask_abstract);
mask_abstract = GetDropoutMaskShapeAbstract(grad_input_shape, nullptr, use_v3);
mask_input->set_abstract(mask_abstract);
// update kernel info
auto kernel_build_info_builder = std::make_shared<kernel::KernelBuildInfo::KernelBuildInfoBuilder>();
kernel_build_info_builder->SetOutputsFormat(std::vector<std::string>{kOpFormat_DEFAULT});
kernel_build_info_builder->SetOutputsDeviceType(std::vector<TypeId>{kNumberTypeUInt8});
kernel_build_info_builder->SetOutputsKernelObjectType({kernel::KernelObjectType::TENSOR});
AnfAlgo::SetSelectKernelBuildInfo(kernel_build_info_builder->Build(), mask_input.get());
} else if (IsPrimitiveCNode(mask_input, prim::kPrimTupleGetItem)) {
auto mask_input_cnode = mask_input->cast<CNodePtr>();
MS_EXCEPTION_IF_NULL(mask_input_cnode);
auto tuple_input = mask_input_cnode->input(kIndex1);
MS_EXCEPTION_IF_NULL(tuple_input);
if (IsValueNode<ValueTuple>(tuple_input)) {
auto tuple_abstract = tuple_input->abstract();
MS_EXCEPTION_IF_NULL(tuple_abstract);
abstract::AbstractSequencePtr sequence_abstract_ptr = tuple_abstract->cast<abstract::AbstractSequencePtr>();
MS_EXCEPTION_IF_NULL(sequence_abstract_ptr);
// Dropout's outputs only have two elements.
if (sequence_abstract_ptr->size() != kIndex2) {
MS_LOG(INTERNAL_EXCEPTION) << "Dropout's outputs have more than two elements, "
<< sequence_abstract_ptr->size();
}
abstract::AbstractBasePtrList abs{};
abs.push_back(sequence_abstract_ptr->elements()[0]);
// modify mask abstract
auto mask_abstract = mask_input->abstract();
MS_EXCEPTION_IF_NULL(mask_abstract);
mask_abstract = GetDropoutMaskShapeAbstract(grad_input_shape, nullptr, use_v3);
mask_input->set_abstract(mask_abstract);
abs.push_back(mask_abstract);
auto new_abstract = std::make_shared<abstract::AbstractTuple>(abs);
tuple_input->set_abstract(new_abstract);
}
}
// CreateDropoutDoMask
auto do_mask_abstract =
std::make_shared<abstract::AbstractTensor>(TypeIdToType(grad_input_type_id), grad_input_shape);
auto dropout_do_mask = CreateDropoutDoMaskCNode(func_graph, dropout_grad_cnode,
{dropout_grad_cnode->input(kIndex1), mask_input, keep_prob_value},
do_mask_abstract, use_v3, enable_keep_prob);
return dropout_do_mask;
}
} // namespace mindspore::opt

27
mindspore/ccsrc/plugin/device/ascend/optimizer/mindir/dropout_unify_mindir.h
View File

@ -48,13 +48,16 @@ class DropoutUnifyMindIR0 : public PatternProcessPass {
class DropoutUnifyMindIR1 : public PatternProcessPass {
public:
explicit DropoutUnifyMindIR1(bool multigraph = true) : PatternProcessPass("dropout_unify_mindir1", multigraph) {}
explicit DropoutUnifyMindIR1(const std::string &name = "dropout_unify_mindir1", bool multigraph = true)
: PatternProcessPass(name, multigraph) {}
~DropoutUnifyMindIR1() override = default;
const BaseRef DefinePattern() const override;
const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override;
void EnableKeepProb() { enable_keep_prob_ = true; }
private:
std::vector<std::string> MustExistPrimitiveName() const override;
bool enable_keep_prob_ = false;
};
class DropoutGradUnifyMindIR : public PatternToPatternPass {
@ -65,6 +68,28 @@ class DropoutGradUnifyMindIR : public PatternToPatternPass {
void DefineSrcPattern(SrcPattern *src_pattern) override;
void DefineDstPattern(DstPattern *dst_pattern) override;
};
class DropoutExtUnifyMindIR1 : public DropoutUnifyMindIR1 {
public:
explicit DropoutExtUnifyMindIR1(bool multigraph = true)
: DropoutUnifyMindIR1("dropout_ext_unify_mindir1", multigraph) {
EnableKeepProb();
}
~DropoutExtUnifyMindIR1() override = default;
const BaseRef DefinePattern() const override;
private:
std::vector<std::string> MustExistPrimitiveName() const override;
};
class DropoutGradExtUnifyMindIR : public PatternProcessPass {
public:
explicit DropoutGradExtUnifyMindIR(bool multigraph = true)
: PatternProcessPass("dropoutgrad_ext_unify_mindir", multigraph) {}
~DropoutGradExtUnifyMindIR() override = default;
const BaseRef DefinePattern() const override;
const AnfNodePtr Process(const FuncGraphPtr &, const AnfNodePtr &, const EquivPtr &) const override;
};
} // namespace opt
} // namespace mindspore
#endif // MINDSPORE_CCSRC_BACKEND_OPTIMIZER_ASCEND_MINDIR_DROPOUT_UNIFY_MINDIR_H_

2
mindspore/core/ops/op_utils.cc
View File

@ -1113,6 +1113,7 @@ template MS_CORE_API std::optional<ArrayValue<float>> GetArrayValue(const ValueP
template MS_CORE_API std::optional<ArrayValue<bool>> GetArrayValue(const ValuePtr &value);
template MS_CORE_API std::optional<ArrayValue<std::string>> GetArrayValue(const ValuePtr &value);
template MS_CORE_API std::optional<ArrayValue<float16>> GetArrayValue(const ValuePtr &value);
template MS_CORE_API std::optional<ArrayValue<bfloat16>> GetArrayValue(const ValuePtr &value);
template MS_CORE_API std::optional<ArrayValue<int64_t>> GetArrayValue(const AbstractBasePtr &abs_base);
template MS_CORE_API std::optional<ArrayValue<int32_t>> GetArrayValue(const AbstractBasePtr &abs_base);
@ -1127,5 +1128,6 @@ template MS_CORE_API std::optional<ArrayValue<float>> GetArrayValue(const Abstra
template MS_CORE_API std::optional<ArrayValue<bool>> GetArrayValue(const AbstractBasePtr &abs_base);
template MS_CORE_API std::optional<ArrayValue<std::string>> GetArrayValue(const AbstractBasePtr &abs_base);
template MS_CORE_API std::optional<ArrayValue<float16>> GetArrayValue(const AbstractBasePtr &abs_base);
template MS_CORE_API std::optional<ArrayValue<bfloat16>> GetArrayValue(const AbstractBasePtr &abs_base);
} // namespace ops
} // namespace mindspore

17
mindspore/core/ops/ops_def/dropout_do_mask_ext_op.yaml Normal file
View File

@ -0,0 +1,17 @@
#operator dropout_do_mask_ext
dropout_do_mask_ext:
args:
input:
dtype: tensor
mask:
dtype: tensor
p:
dtype: float
returns:
output:
dtype: tensor
dispatch:
enable: True
Ascend: DropoutDoMaskExtAscend
function:
disable: True

28
mindspore/core/ops/ops_def/dropout_ext_op.yaml Normal file
View File

@ -0,0 +1,28 @@
#operator dropout_ext
dropout_ext:
args:
input:
dtype: tensor
p:
dtype: float
default: 0.5
seed:
dtype: int
type_cast: tensor
default: 0
offset:
dtype: int
type_cast: tensor
default: 0
labels:
side_effect_hidden: True
returns:
output:
dtype: tensor
mask:
dtype: tensor
dispatch:
enable: True
Ascend: DropoutExtAscend
function:
disable: True

26
mindspore/core/ops/ops_def/dropout_gen_mask_ext_op.yaml Normal file
View File

@ -0,0 +1,26 @@
#operator dropout_gen_mask_ext
dropout_gen_mask_ext:
args:
shape:
dtype: tuple[int]
p:
dtype: float
seed:
dtype: int
type_cast: tensor
offset:
dtype: int
type_cast: tensor
dtype:
dtype: TypeId
arg_handler: dtype_to_type_id
labels:
side_effect_hidden: True
returns:
output:
dtype: tensor
dispatch:
enable: True
Ascend: DropoutGenMaskExtAscend
function:
disable: True

17
mindspore/core/ops/ops_def/dropout_grad_ext_op.yaml Normal file
View File

@ -0,0 +1,17 @@
#operator dropout_grad_ext
dropout_grad_ext:
args:
x:
dtype: tensor
mask:
dtype: tensor
p:
dtype: float
returns:
output:
dtype: tensor
dispatch:
enable: True
Ascend: DropoutGradExtAscend
function:
disable: True

100
mindspore/core/ops/ops_func_impl/dropout_do_mask_ext.cc Normal file
View File

@ -0,0 +1,100 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "ops/ops_func_impl/dropout_do_mask_ext.h"
#include <functional>
#include <memory>
#include <numeric>
#include <set>
#include "ops/op_utils.h"
namespace mindspore {
namespace ops {
namespace {
template <typename T>
bool IsKeepProbValid(const AbstractBasePtr &keep_prob_abs) {
auto keep_prob_opt = GetArrayValue<T>(keep_prob_abs);
if (!keep_prob_opt.has_value()) {
return false;
}
auto keep_prob = keep_prob_opt.value()[kIndex0];
if (keep_prob > static_cast<T>(1.0) || keep_prob < static_cast<T>(0.0)) {
MS_EXCEPTION(ValueError) << "For 'DropoutDoMaskExt', the 'keep_prob(1-p)' must be in range [0, 1], but got "
<< keep_prob << ".";
}
return true;
}
} // namespace
int32_t DropoutDoMaskExtFuncImpl::CheckValidation(const PrimitivePtr &primitive,
const std::vector<AbstractBasePtr> &input_args) const {
if (input_args[kIndex2]->GetType()->object_type() == kObjectTypeTensorType) {
// p will be replaced with keep_prob after some pass.
auto keep_prob_shape = input_args[kIndex2]->GetShape()->GetShapeVector();
MS_CHECK_VALUE(keep_prob_shape.empty(),
"For 'DropoutDoMaskExt', the dim of 'keep_prob' must be 0(scalar), but got " +
std::to_string(keep_prob_shape.size()) + ".");
auto keep_prob_dtype = input_args[kIndex2]->GetType()->cast<TensorTypePtr>()->element()->type_id();
static std::set<TypeId> valid_dtype_set = {kNumberTypeFloat32, kNumberTypeFloat16, kNumberTypeBFloat16};
MS_CHECK_VALUE(valid_dtype_set.find(keep_prob_dtype) != valid_dtype_set.end(),
"For 'DropoutDoMaskExt', the keep_prob type must be in [Float32, Float16, BFloat16], but got " +
TypeIdToString(keep_prob_dtype));
if (MS_UNLIKELY(keep_prob_dtype == kNumberTypeFloat16 && !IsKeepProbValid<float16>(input_args[kIndex2]))) {
return OP_CHECK_RETRY;
} else if (MS_UNLIKELY(keep_prob_dtype == kNumberTypeBFloat16 && !IsKeepProbValid<bfloat16>(input_args[kIndex2]))) {
return OP_CHECK_RETRY;
} else if (MS_UNLIKELY(keep_prob_dtype == kNumberTypeFloat32 && !IsKeepProbValid<float>(input_args[kIndex2]))) {
return OP_CHECK_RETRY;
}
} else {
auto p_opt = GetScalarValue<float>(input_args[kIndex2]->GetValue());
if (MS_UNLIKELY(!p_opt.has_value())) {
return OP_CHECK_RETRY;
}
MS_CHECK_VALUE(
p_opt.value() >= static_cast<float>(0.0) && p_opt.value() <= static_cast<float>(1.0),
"For 'DropoutDoMaskExt', the 'p' must be in range [0, 1], but got " + std::to_string(p_opt.value()) + ".");
}
auto input_shape = input_args[kIndex0]->GetShape()->GetShapeVector();
auto mask_shape = input_args[kIndex1]->GetShape()->GetShapeVector();
if (MS_UNLIKELY(IsDynamic(input_shape) || IsDynamic(mask_shape))) {
return OP_CHECK_RETRY;
}
MS_CHECK_VALUE(mask_shape.size() == 1, "For 'DropoutDoMaskExt', the 'mask' must be 1-D, but got " +
std::to_string(mask_shape.size()) + "-D.");
auto input_size = std::accumulate(input_shape.cbegin(), input_shape.cend(), 1, std::multiplies<int64_t>());
auto mask_size = mask_shape[kIndex0] * 8;
if (input_size > mask_size) {
MS_EXCEPTION(ValueError)
<< "For 'DropoutDoMaskExt', the input 'mask' must be less than or equal to match input, but got 'input' shape: "
<< ShapeVectorToString(input_shape) << ", 'mask' shape: " << ShapeVectorToString(mask_shape) << ".";
}
return OP_CHECK_SUCCESS;
}
BaseShapePtr DropoutDoMaskExtFuncImpl::InferShape(const PrimitivePtr &primitive,
const std::vector<AbstractBasePtr> &input_args) const {
return input_args[kIndex0]->GetShape()->Clone();
}
TypePtr DropoutDoMaskExtFuncImpl::InferType(const PrimitivePtr &primitive,
const std::vector<AbstractBasePtr> &input_args) const {
return input_args[kIndex0]->GetType();
}
} // namespace ops
} // namespace mindspore

34
mindspore/core/ops/ops_func_impl/dropout_do_mask_ext.h Normal file
View File

@ -0,0 +1,34 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CORE_OPS_OPS_FUNC_IMPL_DROPOUT_DO_MASK_EXT_H_
#define MINDSPORE_CORE_OPS_OPS_FUNC_IMPL_DROPOUT_DO_MASK_EXT_H_
#include <vector>
#include "ops/ops_func_impl/op_func_impl.h"
namespace mindspore {
namespace ops {
class MIND_API DropoutDoMaskExtFuncImpl : public OpFuncImpl {
public:
BaseShapePtr InferShape(const PrimitivePtr &primitive, const std::vector<AbstractBasePtr> &input_args) const override;
TypePtr InferType(const PrimitivePtr &primitive, const std::vector<AbstractBasePtr> &input_args) const override;
int32_t CheckValidation(const PrimitivePtr &primitive, const std::vector<AbstractBasePtr> &input_args) const override;
};
} // namespace ops
} // namespace mindspore
#endif // MINDSPORE_CORE_OPS_OPS_FUNC_IMPL_DROPOUT_DO_MASK_EXT_H_

70
mindspore/core/ops/ops_func_impl/dropout_ext.cc Normal file
View File

@ -0,0 +1,70 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "ops/ops_func_impl/dropout_ext.h"
#include <limits>
#include <memory>
#include "ops/op_utils.h"
namespace mindspore {
namespace ops {
int64_t CalMaskShape(const PrimitivePtr &primitive, const ShapeVector &shape_vec) {
constexpr int64_t kDropoutGenMaskMaskConvertLen = 128;
int64_t count = 1;
for (size_t i = 0; i < shape_vec.size(); i++) {
auto dim_value = shape_vec[i];
if (dim_value <= 0) {
MS_LOG(EXCEPTION) << "For '" << primitive->name()
<< "', each dim of 'shape' must be greater than 0, but got shape[" << i << "]: " << dim_value
<< ".";
}
if (std::numeric_limits<int64_t>::max() / count / dim_value < 1) {
MS_LOG(EXCEPTION) << "For '" << primitive->name() << "', integer multiply integer overflow.";
}
count *= shape_vec[i];
}
int64_t n128s = count / kDropoutGenMaskMaskConvertLen;
if ((count % kDropoutGenMaskMaskConvertLen) != 0) {
n128s++;
}
int64_t bytes_count = n128s * 16;
return bytes_count;
}
BaseShapePtr DropoutExtFuncImpl::InferShape(const PrimitivePtr &primitive,
const std::vector<AbstractBasePtr> &input_args) const {
auto x_shape_ptr = input_args[kIndex0]->GetShape();
auto x_shape = input_args[kIndex0]->GetShape()->GetShapeVector();
ShapeVector mask_shape;
if (x_shape_ptr->IsDynamic()) {
mask_shape.push_back(abstract::TensorShape::kShapeDimAny);
} else {
mask_shape.push_back(CalMaskShape(primitive, x_shape));
}
auto mask_shape_ptr = std::make_shared<abstract::TensorShape>(mask_shape);
return std::make_shared<abstract::TupleShape>(std::vector<abstract::BaseShapePtr>{x_shape_ptr, mask_shape_ptr});
}
TypePtr DropoutExtFuncImpl::InferType(const PrimitivePtr &primitive,
const std::vector<AbstractBasePtr> &input_args) const {
auto x_type = input_args[0]->GetType();
return std::make_shared<Tuple>(std::vector<TypePtr>{x_type, kUInt8});
}
} // namespace ops
} // namespace mindspore

35
mindspore/core/ops/ops_func_impl/dropout_ext.h Normal file
View File

@ -0,0 +1,35 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CORE_OPS_OPS_FUNC_IMPL_DROPOUT_EXT_H_
#define MINDSPORE_CORE_OPS_OPS_FUNC_IMPL_DROPOUT_EXT_H_
#include <vector>
#include "ops/ops_func_impl/op_func_impl.h"
namespace mindspore {
namespace ops {
class MIND_API DropoutExtFuncImpl : public OpFuncImpl {
public:
BaseShapePtr InferShape(const PrimitivePtr &primitive, const std::vector<AbstractBasePtr> &input_args) const override;
TypePtr InferType(const PrimitivePtr &primitive, const std::vector<AbstractBasePtr> &input_args) const override;
};
int64_t CalMaskShape(const PrimitivePtr &primitive, const ShapeVector &shape_vec);
} // namespace ops
} // namespace mindspore
#endif // MINDSPORE_CORE_OPS_OPS_FUNC_IMPL_DROPOUT_EXT_H_

80
mindspore/core/ops/ops_func_impl/dropout_gen_mask_ext.cc Normal file
View File

@ -0,0 +1,80 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "ops/ops_func_impl/dropout_gen_mask_ext.h"
#include <memory>
#include <set>
#include "ops/ops_func_impl/dropout_ext.h"
#include "ops/op_utils.h"
namespace mindspore {
namespace ops {
BaseShapePtr DropoutGenMaskExtFuncImpl::InferShape(const PrimitivePtr &primitive,
const std::vector<AbstractBasePtr> &input_args) const {
auto shape = input_args[kIndex0];
if (MS_UNLIKELY(shape->GetType()->object_type() == kObjectTypeTensorType &&
IsDynamic(shape->GetShape()->GetShapeVector()))) {
return std::make_shared<abstract::TensorShape>(ShapeVector({abstract::TensorShape::kShapeDimAny}));
} else if (MS_UNLIKELY(shape->GetShape()->isa<abstract::DynamicSequenceShape>())) {
return std::make_shared<abstract::TensorShape>(ShapeVector({abstract::TensorShape::kShapeDimAny}));
}
auto shape_opt = GetArrayValue<int64_t>(shape);
if (MS_UNLIKELY(!shape_opt.has_value())) {
return std::make_shared<abstract::TensorShape>(ShapeVector({abstract::TensorShape::kShapeDimAny}));
}
auto shape_array = shape_opt.value();
if (MS_UNLIKELY(shape_array.HasUnknownValue())) {
return std::make_shared<abstract::TensorShape>(ShapeVector({abstract::TensorShape::kShapeDimAny}));
}
return std::make_shared<abstract::TensorShape>(ShapeVector({CalMaskShape(primitive, shape_array.ToVector())}));
}
TypePtr DropoutGenMaskExtFuncImpl::InferType(const PrimitivePtr &primitive,
const std::vector<AbstractBasePtr> &input_args) const {
return kUInt8;
}
int32_t DropoutGenMaskExtFuncImpl::CheckValidation(const PrimitivePtr &primitive,
const std::vector<AbstractBasePtr> &input_args) const {
auto shape_type = input_args[kIndex0]->GetType();
if (shape_type->object_type() == kObjectTypeTensorType) {
// shape will be replaced with tensor after some pass.
auto shape_tensor_type = shape_type->cast<TensorTypePtr>();
MS_EXCEPTION_IF_NULL(shape_tensor_type);
auto shape_elem_typeid = shape_tensor_type->element()->type_id();
MS_CHECK_VALUE(shape_elem_typeid == kNumberTypeInt64,
"For 'DropoutGenMaskExt', the element of shape tensor must be int64, but got " +
TypeIdToString(shape_elem_typeid));
}
auto dtype_opt = GetScalarValue<int64_t>(input_args[kIndex4]->GetValue());
if (MS_UNLIKELY(!dtype_opt.has_value())) {
return OP_CHECK_RETRY;
}
static std::set<TypeId> valid_dtype_set = {kNumberTypeFloat32, kNumberTypeFloat16, kNumberTypeBFloat16};
auto dtype_value = static_cast<TypeId>(dtype_opt.value());
MS_CHECK_VALUE(valid_dtype_set.find(dtype_value) != valid_dtype_set.end(),
"For 'DropoutGenMaskExt', the dtype must be in [Float32, Float16, BFloat16], but got " +
TypeIdToString(dtype_value));
return OP_CHECK_SUCCESS;
}
} // namespace ops
} // namespace mindspore

34
mindspore/core/ops/ops_func_impl/dropout_gen_mask_ext.h Normal file
View File

@ -0,0 +1,34 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CORE_OPS_OPS_FUNC_IMPL_DROPOUT_GEN_MASK_EXT_H_
#define MINDSPORE_CORE_OPS_OPS_FUNC_IMPL_DROPOUT_GEN_MASK_EXT_H_
#include <vector>
#include "ops/ops_func_impl/op_func_impl.h"
namespace mindspore {
namespace ops {
class MIND_API DropoutGenMaskExtFuncImpl : public OpFuncImpl {
public:
BaseShapePtr InferShape(const PrimitivePtr &primitive, const std::vector<AbstractBasePtr> &input_args) const override;
TypePtr InferType(const PrimitivePtr &primitive, const std::vector<AbstractBasePtr> &input_args) const override;
int32_t CheckValidation(const PrimitivePtr &primitive, const std::vector<AbstractBasePtr> &input_args) const override;
};
} // namespace ops
} // namespace mindspore
#endif // MINDSPORE_CORE_OPS_OPS_FUNC_IMPL_DROPOUT_GEN_MASK_EXT_H_

31
mindspore/core/ops/ops_func_impl/dropout_grad_ext.cc Normal file
View File

@ -0,0 +1,31 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "ops/ops_func_impl/dropout_grad_ext.h"
namespace mindspore {
namespace ops {
BaseShapePtr DropoutGradExtFuncImpl::InferShape(const PrimitivePtr &primitive,
const std::vector<AbstractBasePtr> &input_args) const {
return input_args[kIndex0]->GetShape()->Clone();
}
TypePtr DropoutGradExtFuncImpl::InferType(const PrimitivePtr &primitive,
const std::vector<AbstractBasePtr> &input_args) const {
return input_args[kIndex0]->GetType();
}
} // namespace ops
} // namespace mindspore

33
mindspore/core/ops/ops_func_impl/dropout_grad_ext.h Normal file
View File

@ -0,0 +1,33 @@
/**
* Copyright 2024 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CORE_OPS_OPS_FUNC_IMPL_DROPOUT_GRAD_EXT_H_
#define MINDSPORE_CORE_OPS_OPS_FUNC_IMPL_DROPOUT_GRAD_EXT_H_
#include <vector>
#include "ops/ops_func_impl/op_func_impl.h"
namespace mindspore {
namespace ops {
class MIND_API DropoutGradExtFuncImpl : public OpFuncImpl {
public:
BaseShapePtr InferShape(const PrimitivePtr &primitive, const std::vector<AbstractBasePtr> &input_args) const override;
TypePtr InferType(const PrimitivePtr &primitive, const std::vector<AbstractBasePtr> &input_args) const override;
};
} // namespace ops
} // namespace mindspore
#endif // MINDSPORE_CORE_OPS_OPS_FUNC_IMPL_DROPOUT_GRAD_EXT_H_

2
mindspore/python/mindspore/mint/nn/__init__.py
View File

@ -49,7 +49,7 @@ from mindspore.nn.layer.basic import UpsampleExt as Upsample
# 13
# 14
from mindspore.nn.layer.basic import DropoutExt as Dropout
# 15
# 16

4
mindspore/python/mindspore/mint/nn/functional.py
View File

@ -46,7 +46,7 @@ from mindspore.ops.functional import relu
# 13
# 14
from mindspore.ops.function.nn_func import dropout_ext as dropout
# 15
# 16
@ -249,7 +249,7 @@ __all__ = [
# 13
# 14
'dropout',
# 15
# 16

68
mindspore/python/mindspore/nn/layer/basic.py
View File

@ -36,10 +36,12 @@ from mindspore import _checkparam as Validator
from mindspore.nn.cell import Cell
from mindspore.nn.layer.activation import get_activation
from mindspore.common._decorator import deprecated
from mindspore.ops.auto_generate import dropout_ext_op
from mindspore.nn.generator import default_generator
__all__ = ['Dropout', 'Flatten', 'Dense', 'ClipByNorm', 'Norm', 'OneHot', 'Pad', 'Unfold', 'Tril', 'Triu',
'MatrixDiag', 'MatrixDiagPart', 'MatrixSetDiag', 'L1Regularizer', 'Dropout1d',
'Dropout2d', 'Dropout3d', 'Upsample', 'Roll', 'Identity', 'Unflatten']
'Dropout2d', 'Dropout3d', 'Upsample', 'Roll', 'Identity', 'Unflatten', 'DropoutExt']
class L1Regularizer(Cell):
@ -202,6 +204,70 @@ class Dropout(Cell):
return f'p={self.p}'
class DropoutExt(Cell):
r"""
Dropout layer for the input.
Dropout is a means of regularization that reduces overfitting by preventing correlations between neuronal nodes.
The operator randomly sets some neurons output to 0 according to `p`, which means the probability of discarding
during training. And the return will be multiplied by :math:`\frac{1}{1-p}` during training.
During the reasoning, this layer returns the same Tensor as the `x`.
This technique is proposed in paper `Dropout: A Simple Way to Prevent Neural Networks from Overfitting
<http://www.cs.toronto.edu/~rsalakhu/papers/srivastava14a.pdf>`_ and proved to be effective to reduce
over-fitting and prevents neurons from co-adaptation. See more details in `Improving neural networks by
preventing co-adaptation of feature detectors
<https://arxiv.org/pdf/1207.0580.pdf>`_.
Note:
- Each channel will be zeroed out independently on every construct call.
Parameter `p` means the probability of the element of the input tensor to be zeroed.
Args:
p (float): The dropout rate, greater than or equal to 0 and less than 1.
E.g. rate=0.9, dropping out 90% of input neurons. Default: ``0.5`` .
Inputs:
- **x** (Tensor) - The input of Dropout with data type of float16 or float32.
Outputs:
Tensor, output tensor with the same shape as the `x`.
Raises:
ValueError: If `p` is not in range [0, 1).
ValueError: If length of shape of `x` is less than 1.
Supported Platforms:
``Ascend``
Examples:
>>> import mindspore
>>> from mindspore import Tensor, nn
>>> import numpy as np
>>> x = Tensor(np.ones([2, 2, 3]), mindspore.float32)
>>> net = nn.DropoutExt(p=0.2)
>>> net.set_train()
>>> output = net(x)
>>> print(output.shape)
(2, 2, 3)
"""
def __init__(self, p=0.5):
"""Initialize DropoutExt."""
super(DropoutExt, self).__init__()
self.generator = default_generator()
self.dropout = dropout_ext_op
self.p = p
def construct(self, x):
if not self.training or self.p == 0:
return x
seed, offset = self.generator(1)
out, _ = self.dropout(x, self.p, seed, offset)
return out
class Dropout1d(Cell):
r"""
During training, randomly zeroes entire channels of the input tensor with probability `p`

1
mindspore/python/mindspore/ops/function/__init__.py
View File

@ -548,6 +548,7 @@ from .nn_func import (
channel_shuffle,
hardsigmoid,
group_norm,
dropout_ext,
)
from .linalg_func import (
cond,

51
mindspore/python/mindspore/ops/function/nn_func.py
View File

@ -46,8 +46,9 @@ from mindspore.ops.auto_generate import log_softmax, dense, prelu, celu, relu, f
from mindspore.ops.auto_generate.gen_ops_prim import GroupNorm
from mindspore.ops.auto_generate import (reflection_pad_1d_op, reflection_pad_2d_op, reflection_pad_3d_op,
replication_pad_1d_op, replication_pad_2d_op, replication_pad_3d_op,
constant_pad_nd_op)
constant_pad_nd_op, dropout_ext_op)
from mindspore.ops.auto_generate.gen_ops_prim import embedding_op, Convolution
from mindspore.nn.generator import default_generator
abs_ = P.Abs()
add_ = P.Add()
@ -1363,6 +1364,50 @@ def dropout(input, p=0.5, training=True, seed=None):
return out
@_function_forbid_reuse
def dropout_ext(input, p=0.5, training=True, seed=None):
r"""
During training, randomly zeroes some of the elements of the input tensor
with probability `p` from a Bernoulli distribution. It plays the role of reducing neuron correlation and
avoid overfitting. And the return will be multiplied by :math:`\frac{1}{1-p}` during training.
During the reasoning, this operation returns the same Tensor as the `x`.
Args:
input (Tensor): The input Tensor of shape :math:`(*, N)`, with data type of float16, float32 or float64.
p (float, optional): The dropping rate, between 0 and 1, e.g. p = 0.1,
means dropping out 10% of input units. Default: ``0.5`` .
training (bool): Apply dropout_ext if is True. Default: ``True``.
seed (int, optional): Seed is used as entropy source for Random number engines generating pseudo-random numbers.
Default: ``None`` , which will be treated as ``0`` .
Returns:
- **output** (Tensor) - Zeroed tensor, with the same shape and data type as `input`.
Raises:
TypeError: If `p` is not a float.
TypeError: If dtype of `input` is not float16, float32 or float64.
TypeError: If `input` is not a Tensor.
Supported Platforms:
``Ascend``
Examples:
>>> import mindspore
>>> from mindspore import Tensor, ops
>>> input = Tensor(((20, 16), (50, 50)), mindspore.float32)
>>> output = ops.dropout_ext(input, p=0.5)
>>> print(output.shape)
(2, 2)
"""
check_bool_const(training, "training", "dropout_ext")
if training is False:
return input
generator = default_generator()
seed, offset = generator(1)
out, _ = dropout_ext_op(input, p, seed, offset)
return out
def dropout1d(input, p=0.5, training=True):
r"""
During training, randomly zeroes some channels of the input tensor with probability `p`
@ -2155,6 +2200,7 @@ def _is_dim_unknown(shape):
return isinstance(shape, tuple) and -2 in shape
# pylint: disable=missing-docstring
@_primexpr
def _interploate_make_tuple(rank, value):
s = tuple_to_tensor_((rank,), mstype.int32)
@ -7958,6 +8004,7 @@ __all__ = [
'triplet_margin_loss',
'channel_shuffle',
'hardsigmoid',
'group_norm'
'group_norm',
'dropout_ext',
]
__all__.sort()

271
tests/st/ops/test_ops_dropout_ext.py Normal file
View File

@ -0,0 +1,271 @@
# Copyright 2024 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import os
from functools import reduce
import numpy as np
import pytest
import mindspore as ms
import mindspore.common.dtype as mstype
from mindspore import ops
from mindspore.ops.auto_generate import dropout_ext_op
from mindspore.nn import DropoutExt
from mindspore.ops.function import dropout_ext
from mindspore.nn import Cell
from tests.st.utils import test_utils
def generate_random_input(shape, dtype):
return np.random.random(shape).astype(dtype)
@test_utils.run_with_cell
def dropout_forward_func(x, p=0.4):
return dropout_ext(x, p)
@test_utils.run_with_cell
def dropout_backward_func(x, p=0.4):
return ops.grad(dropout_forward_func, (0))(x, p)
def compare_output(x, p, output):
# check output
keep_prob = 1 - p
if output.dtype == mstype.bfloat16:
output_np = output.astype(mstype.float32).asnumpy()
else:
output_np = output.asnumpy()
elem_count = x.size
nonzero_count = np.count_nonzero(output_np)
assert (elem_count * (keep_prob - 0.1)) < nonzero_count < (elem_count * (keep_prob + 0.1))
output_sum = np.sum(output_np)
x_sum = np.sum(x)
assert abs(output_sum - x_sum) / x_sum < 0.1
def compare_grad(x, p, grad):
# check grad
keep_prob = 1 - p
if grad.dtype == mstype.bfloat16:
grad_np = grad.astype(mstype.float32).asnumpy()
else:
grad_np = grad.asnumpy()
elem_count = x.size
nonzero_count = np.count_nonzero(grad_np)
assert (elem_count * (keep_prob - 0.1)) < nonzero_count < (elem_count * (keep_prob + 0.1))
grad_sum = np.sum(grad_np)
np.testing.assert_allclose(grad_sum * keep_prob, nonzero_count, rtol=1e-3)
@pytest.mark.level0
@pytest.mark.env_onecard
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_arm_ascend910b_training
@pytest.mark.parametrize('context_mode', [ms.PYNATIVE_MODE])
@pytest.mark.parametrize('dtype', [np.float16, np.float32])
def test_func_dropout_normal(context_mode, dtype):
"""
Feature: pyboost function.
Description: test function dropout normal.
Expectation: expect correct result.
"""
ms.context.set_context(mode=context_mode)
if context_mode == ms.GRAPH_MODE:
os.environ['GRAPH_OP_RUN'] = "1"
x = generate_random_input((128, 128), dtype)
p = 0.4
output = dropout_forward_func(ms.Tensor(x), p)
compare_output(x, p, output)
x1 = generate_random_input((64, 64), dtype)
p1 = 0.3
grad = dropout_backward_func(ms.Tensor(x1), p1)
compare_grad(x1, p1, grad)
if context_mode == ms.GRAPH_MODE:
del os.environ['GRAPH_OP_RUN']
@pytest.mark.level0
@pytest.mark.env_onecard
@pytest.mark.platform_arm_ascend910b_training
@pytest.mark.parametrize('context_mode', [ms.PYNATIVE_MODE])
def test_func_dropout_bfloat16(context_mode):
"""
Feature: pyboost function.
Description: test function dropout normal.
Expectation: expect correct result.
"""
ms.context.set_context(mode=context_mode)
if context_mode == ms.GRAPH_MODE:
os.environ['GRAPH_OP_RUN'] = "1"
x = generate_random_input((128, 128), np.float32)
p = 0.4
output = dropout_forward_func(ms.Tensor(x).astype(mstype.bfloat16), p)
compare_output(x, p, output)
x1 = generate_random_input((64, 64), np.float32)
p1 = 0.3
grad = dropout_backward_func(ms.Tensor(x1).astype(mstype.bfloat16), p1)
compare_grad(x1, p1, grad)
if context_mode == ms.GRAPH_MODE:
del os.environ['GRAPH_OP_RUN']
def compare_func(x, p, output, mask=None):
device_target = ms.context.get_context("device_target")
keep_prob = 1 - p
if device_target != "Ascend":
# check output
output_np = output.asnumpy()
elem_count = x.size
nonzero_count = np.count_nonzero(output_np)
assert (elem_count * (keep_prob - 0.1)) < nonzero_count < (elem_count * (keep_prob + 0.1))
output_sum = np.sum(output_np)
x_sum = np.sum(x)
assert abs(output_sum - x_sum) / x_sum < 0.1
# check mask
if mask is not None:
mask_np = mask.asnumpy()
mask_sum = np.sum(mask_np)
assert np.count_nonzero(mask_np) == nonzero_count
assert abs(mask_sum - nonzero_count) / nonzero_count < 0.1
else:
# check output
output_np = output.asnumpy()
elem_count = x.size
nonzero_count = np.count_nonzero(output_np)
assert (elem_count * (keep_prob - 0.1)) < nonzero_count < (elem_count * (keep_prob + 0.1))
output_sum = np.sum(output_np)
x_sum = np.sum(x)
assert abs(output_sum - x_sum) / x_sum < 0.1
# check mask
if mask is not None:
assert len(mask.shape) == 1
assert np.ceil(reduce(lambda a, b: a * b, x.shape) / 128) * 16 == mask.shape[0]
@pytest.mark.level0
@pytest.mark.env_onecard
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_arm_ascend910b_training
@pytest.mark.parametrize('context_mode', [ms.PYNATIVE_MODE])
def test_nn_DropoutExt_normal(context_mode):
"""
Feature: nn.DropoutExt
Description: forward
Expectation: success
"""
os.environ["GRAPH_OP_RUN"] = "1"
ms.context.set_context(mode=context_mode)
x = np.array(np.random.random((16, 16, 16, 16)), np.float32)
p = 0.4
net = DropoutExt(p)
net.set_train()
output = net(ms.tensor(x))
compare_func(x, p, output)
del os.environ["GRAPH_OP_RUN"]
@pytest.mark.level1
@pytest.mark.env_onecard
@pytest.mark.platform_arm_ascend910b_training
@pytest.mark.parametrize('context_mode', [ms.PYNATIVE_MODE])
def test_nn_DropoutExt_bf16(context_mode):
"""
Feature: nn.DropoutExt
Description: bf16
Expectation: success
"""
os.environ["GRAPH_OP_RUN"] = "1"
ms.context.set_context(mode=context_mode)
x = np.array(np.random.random((128, 128)), np.float32)
p = 0.4
net = DropoutExt(p)
net.set_train()
output = net(ms.tensor(x, mstype.bfloat16))
compare_func(x, p, output.float())
del os.environ["GRAPH_OP_RUN"]
class DropoutExtCell(Cell):
def __init__(self):
super().__init__()
self.dropout_ext = dropout_ext_op
def construct(self, x, p):
return self.dropout_ext(x, p)
@pytest.mark.level0
@pytest.mark.env_onecard
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_arm_ascend910b_training
@pytest.mark.parametrize('context_mode', [ms.PYNATIVE_MODE])
def test_ops_DropoutExt_normal(context_mode):
"""
Feature: ops.DropoutExt
Description: forward
Expectation: success
"""
os.environ["GRAPH_OP_RUN"] = "1"
ms.context.set_context(mode=context_mode)
dropout_cell = DropoutExtCell()
x = np.array(np.random.random((128, 128)), np.float32)
p = 0.4
output, mask = dropout_cell(ms.tensor(x), p)
compare_func(x, p, output, mask)
if context_mode == ms.GRAPH_MODE:
dropout_cell.set_inputs(ms.tensor(shape=[None, None], dtype=ms.float32), p)
else:
dropout_cell.set_inputs(ms.tensor(shape=[None, None], dtype=ms.float32), ms.mutable(p))
x = np.array(np.random.random((256, 128)), np.float32)
output, mask = dropout_cell(ms.tensor(x), p)
compare_func(x, p, output, mask)
x = np.array(np.random.random((128, 256)), np.float32)
output, mask = dropout_cell(ms.tensor(x), p)
compare_func(x, p, output, mask)
if context_mode == ms.GRAPH_MODE:
dropout_cell.set_inputs(ms.tensor(shape=None, dtype=ms.float32), p)
else:
dropout_cell.set_inputs(ms.tensor(shape=None, dtype=ms.float32), ms.mutable(p))
x = np.array(np.random.random((128, 128, 128)), np.float32)
output, mask = dropout_cell(ms.tensor(x), p)
compare_func(x, p, output, mask)
x = np.array(np.random.random((16, 16, 16, 16)), np.float32)
output, mask = dropout_cell(ms.tensor(x), p)
compare_func(x, p, output, mask)
del os.environ["GRAPH_OP_RUN"]