forked from mindspore-Ecosystem/mindspore
add select op
This commit is contained in:
yangzhenzhang
parent
4a8195b805
commit
c2ca2232c5
|
|
@ -234,6 +234,7 @@ using AsinCost = ReLU6Cost;
|
|||
using ErfCost = ReLU6Cost;
|
||||
using ErfcCost = ReLU6Cost;
|
||||
using ActivationInfoCost = ReLU6Cost;
|
||||
using SelectCost = ReLU6Cost;
|
||||
|
||||
class TransposeCost : public CastCost {
|
||||
public:
|
||||
|
|
|
|||
|
|
@ -191,6 +191,7 @@ REGISTER(StackInfo);
|
|||
REGISTER(ConcatInfo);
|
||||
REGISTER(SplitInfo);
|
||||
REGISTER(UniqueInfo);
|
||||
REGISTER(SelectInfo);
|
||||
REGISTER(GatherNdInfo);
|
||||
REGISTER(TopKInfo);
|
||||
REGISTER(ScatterUpdateInfo);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
|
@ -50,6 +50,7 @@
|
|||
#include "frontend/parallel/ops_info/unique_info.h"
|
||||
#include "frontend/parallel/ops_info/uniform_candidate_sampler_info.h"
|
||||
#include "frontend/parallel/ops_info/reluv2_info.h"
|
||||
#include "frontend/parallel/ops_info/select_info.h"
|
||||
#include "frontend/parallel/ops_info/gathernd_info.h"
|
||||
#include "frontend/parallel/ops_info/topk_info.h"
|
||||
#include "frontend/parallel/ops_info/scatter_update_info.h"
|
||||
|
|
|
|||
|
|
@ -189,6 +189,7 @@ constexpr char REDUCE_SCATTER[] = "ReduceScatter";
|
|||
constexpr char HOST_REDUCE_SCATTER[] = "_HostReduceScatter";
|
||||
constexpr char EMBEDDING_LOOKUP[] = "EmbeddingLookup";
|
||||
constexpr char CONCAT[] = "Concat";
|
||||
constexpr char SELECT[] = "Select";
|
||||
constexpr char SOFTMAX_CROSS_ENTROPY_WITH_LOGITS[] = "SoftmaxCrossEntropyWithLogits";
|
||||
constexpr char SIGMOID_CROSS_ENTROPY_WITH_LOGITS[] = "SigmoidCrossEntropyWithLogits";
|
||||
constexpr char MATMUL[] = "MatMul";
|
||||
|
|
|
|||
|
|
@ -0,0 +1,194 @@
|
|||
/**
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "frontend/parallel/ops_info/select_info.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "frontend/parallel/device_matrix.h"
|
||||
#include "frontend/parallel/strategy.h"
|
||||
#include "frontend/parallel/tensor_layout/tensor_redistribution.h"
|
||||
#include "pipeline/jit/resource.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace parallel {
|
||||
Status SelectInfo::CheckStrategy(const StrategyPtr &strategy) {
|
||||
MS_EXCEPTION_IF_NULL(strategy);
|
||||
if (CheckStrategyValue(strategy, inputs_shape_) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Invalid strategy";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
if (inputs_shape_.size() != 3) {
|
||||
MS_LOG(ERROR) << name_ << ": The size of inputs shape must be 3";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
if ((inputs_shape_[0] != inputs_shape_[1]) || (inputs_shape_[1] != inputs_shape_[2])) {
|
||||
MS_LOG(ERROR) << name_ << ": Now we only support the case that all three input shapes are equal";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
std::vector<Dimensions> stra = strategy->GetInputDim();
|
||||
if (stra.size() != 3) {
|
||||
MS_LOG(ERROR) << name_ << ": The size of strategy must be 3";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
if ((stra[0] != stra[1]) || (stra[1] != stra[2])) {
|
||||
MS_LOG(ERROR) << name_ << ": Now we only support the case that all three strategies are equal";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status SelectInfo::InferDevMatrixShape() {
|
||||
MS_EXCEPTION_IF_NULL(strategy_);
|
||||
std::vector<Dimensions> stra = strategy_->GetInputDim();
|
||||
if (stra.empty()) {
|
||||
MS_LOG(ERROR) << name_ << "The strategy is empty";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
dev_matrix_shape_ = stra[0];
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status SelectInfo::InferTensorMap() {
|
||||
TensorMap tensor_map;
|
||||
if (inputs_shape_.empty()) {
|
||||
MS_LOG(ERROR) << name_ << "The inputs shape is empty";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
// cannot use dev_matrix_shape_ replace inputs_shape_[0], because it may not be fully split in all devices.
|
||||
int64_t size = SizeToLong(inputs_shape_[0].size());
|
||||
for (int64_t i = 0; i < size; ++i) {
|
||||
tensor_map.push_back(size - i - 1);
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < inputs_shape_.size(); ++i) {
|
||||
inputs_tensor_map_.push_back(tensor_map);
|
||||
}
|
||||
outputs_tensor_map_.push_back(tensor_map);
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status SelectInfo::InferTensorInfo() {
|
||||
if (inputs_shape_.empty() || outputs_shape_.empty() || inputs_tensor_map_.empty() || outputs_tensor_map_.empty()) {
|
||||
MS_LOG(ERROR) << name_ << ": Invalid args";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
TensorLayout input_layout, output_layout;
|
||||
for (size_t i = 0; i < inputs_shape_.size(); ++i) {
|
||||
// infer tensor layout
|
||||
if (input_layout.InitFromVector(dev_matrix_shape_, inputs_tensor_map_[i], inputs_shape_[i]) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Infer input tensor layout failed.";
|
||||
return FAILED;
|
||||
}
|
||||
TensorInfo input_tensor_info(input_layout);
|
||||
inputs_tensor_info_.push_back(input_tensor_info);
|
||||
}
|
||||
|
||||
if (output_layout.InitFromVector(dev_matrix_shape_, outputs_tensor_map_[0], outputs_shape_[0]) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Infer output tensor layout failed.";
|
||||
return FAILED;
|
||||
}
|
||||
TensorInfo output_tensor_info(output_layout);
|
||||
outputs_tensor_info_.push_back(output_tensor_info);
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
void SelectInfo::ReComputeBatchSplitFlagList() {
|
||||
for (size_t i = 0; i < inputs_shape_.size(); i++) {
|
||||
split_flag_list_[i] = true;
|
||||
}
|
||||
}
|
||||
|
||||
Status SelectInfo::SetCostUnderStrategy(const StrategyPtr &strategy) { return SetCostUnderStrategyBase(strategy); }
|
||||
|
||||
Status SelectInfo::GenerateStrategies(int64_t stage_id) {
|
||||
if (InferAttrs() != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Infer attrs failed";
|
||||
return FAILED;
|
||||
}
|
||||
if (inputs_shape_.empty()) {
|
||||
MS_LOG(ERROR) << name_ << ": The inputs shape is empty";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
// to generate the first input's strategy
|
||||
Shape input_split(inputs_shape_[0].size(), 1);
|
||||
Shapes splittable_input = {input_split};
|
||||
Shapes tmp_inputs_shape = {inputs_shape_[0]};
|
||||
|
||||
std::vector<StrategyPtr> sp_vector;
|
||||
if (GenerateStrategiesForIndependentInputs(stage_id, tmp_inputs_shape, splittable_input, &sp_vector) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Generate strategies failed";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
// the others strategies are equal to the first input's strategy
|
||||
for (auto &sp : sp_vector) {
|
||||
if ((sp == nullptr) || sp->GetInputDim().empty()) {
|
||||
MS_LOG(ERROR) << name_ << ": The strategy is null or empty";
|
||||
return FAILED;
|
||||
}
|
||||
Strategys tmp_strategy;
|
||||
Dimensions first_input_strategy = sp->GetInputDim()[0];
|
||||
for (size_t i = 0; i < inputs_shape_.size(); ++i) {
|
||||
tmp_strategy.push_back(first_input_strategy);
|
||||
}
|
||||
sp->ResetInputs(tmp_strategy);
|
||||
}
|
||||
|
||||
size_t success = 0;
|
||||
for (auto &sp : sp_vector) {
|
||||
PrintStrategy(sp);
|
||||
if (SetCostUnderStrategy(sp) == SUCCESS) {
|
||||
success++;
|
||||
MS_LOG(INFO) << name_ << ": Successfully generated " << success << " strategy.";
|
||||
PrintStrategy(sp);
|
||||
}
|
||||
}
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status SelectInfo::Init(const StrategyPtr &strategy) {
|
||||
if (InitWithAutoRepeatCalc(strategy) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Init failed.";
|
||||
return FAILED;
|
||||
}
|
||||
MS_LOG(INFO) << name_ << ": Init success.";
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status SelectInfo::InitForCostModel(const StrategyPtr &strategy) {
|
||||
if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Init for cost model failed.";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
MS_LOG(INFO) << name_ << ": Init for cost model success.";
|
||||
return SUCCESS;
|
||||
}
|
||||
} // namespace parallel
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
/**
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_FRONTEND_PARALLEL_OPS_INFO_SELECT_INFO_H_
|
||||
#define MINDSPORE_CCSRC_FRONTEND_PARALLEL_OPS_INFO_SELECT_INFO_H_
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "ir/value.h"
|
||||
#include "frontend/parallel/auto_parallel/operator_costmodel.h"
|
||||
#include "frontend/parallel/ops_info/operator_info.h"
|
||||
#include "frontend/parallel/strategy.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace parallel {
|
||||
class SelectInfo : public OperatorInfo {
|
||||
public:
|
||||
SelectInfo(const std::string &operator_name, const Shapes &inputs_shape, const Shapes &outputs_shape,
|
||||
const PrimitiveAttrs &attrs)
|
||||
: OperatorInfo(operator_name, inputs_shape, outputs_shape, attrs, std::make_shared<SelectCost>()) {}
|
||||
~SelectInfo() override = default;
|
||||
|
||||
Status Init(const StrategyPtr &strategy) override;
|
||||
Status InitForCostModel(const StrategyPtr &strategy) override;
|
||||
Status GenerateStrategies(int64_t) override;
|
||||
Status SetCostUnderStrategy(const StrategyPtr &) override;
|
||||
void ReComputeBatchSplitFlagList() override;
|
||||
|
||||
protected:
|
||||
Status GetAttrs() override { return SUCCESS; }
|
||||
Status CheckStrategy(const StrategyPtr &strategy) override;
|
||||
Status InferMirrorOps() override { return SUCCESS; }
|
||||
Status InferForwardCommunication() override { return SUCCESS; }
|
||||
Status InferTensorInfo() override;
|
||||
Status InferDevMatrixShape() override;
|
||||
Status InferTensorMap() override;
|
||||
};
|
||||
|
||||
using SelectInfoPtr = std::shared_ptr<SelectInfo>;
|
||||
} // namespace parallel
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_FRONTEND_PARALLEL_OPS_INFO_CONCAT_INFO_H_
|
||||
|
|
@ -162,7 +162,7 @@ bool IsSplittableOperator(const std::string &op_name) {
|
|||
EXPM1, LOG1P, SIN, SINH, TAN, RSQRT, INV, RECIPROCAL, ROUND, FLOOR, SIGN, ERF, ERFC, ZEROSLIKE, ONESLIKE,
|
||||
BESSELI0E, BESSELI1E, FLOORMOD, ASSIGN, ASSIGN_ADD, ATAN2, DIVNONAN, LOGICALAND, LOGICALOR, ELU, RELU6, RELUV2,
|
||||
SOFTPLUS, SOFTSIGN, GREATEREQUAL, LESSEQUAL, LESS, APPROXIMATEEQUAL, MOD, UNIQUE, UNSORTED_SEGMENT_SUM,
|
||||
UNSORTED_SEGMENT_MIN, REPEAT_ELEMENTS, TENSOR_DOT, RANGE, UNIFORM_CANDIDATE_SAMPLER, SLICE,
|
||||
UNSORTED_SEGMENT_MIN, REPEAT_ELEMENTS, TENSOR_DOT, RANGE, UNIFORM_CANDIDATE_SAMPLER, SLICE, SELECT,
|
||||
UNSORTED_SEGMENT_MAX, GATHER_ND, TOPK, SCATTER_UPDATE};
|
||||
// clang-format on
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,111 @@
|
|||
# Copyright 2021 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import mindspore as ms
|
||||
from mindspore import context, Tensor, Parameter
|
||||
from mindspore.nn import Cell, Momentum
|
||||
from mindspore.ops import operations as P
|
||||
from mindspore.train import Model
|
||||
from tests.dataset_mock import MindData
|
||||
|
||||
|
||||
class Dataset(MindData):
|
||||
def __init__(self, predict, label, length=3):
|
||||
super(Dataset, self).__init__(size=length)
|
||||
self.predict = predict
|
||||
self.label = label
|
||||
self.index = 0
|
||||
self.length = length
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
if self.index >= self.length:
|
||||
raise StopIteration
|
||||
self.index += 1
|
||||
return self.predict, self.label
|
||||
|
||||
def reset(self):
|
||||
self.index = 0
|
||||
|
||||
|
||||
class Net(Cell):
|
||||
def __init__(self, w1, w2, strategy1=None, strategy2=None):
|
||||
super().__init__()
|
||||
self.less = P.Less().shard(strategy1)
|
||||
self.w1 = Parameter(w1, "w1")
|
||||
self.w2 = Parameter(w2, "w2")
|
||||
self.select = P.Select().shard(strategy2)
|
||||
|
||||
def construct(self, x, b):
|
||||
out = self.less(x, b)
|
||||
out = self.select(out, self.w1, self.w2)
|
||||
return out
|
||||
|
||||
|
||||
_x = Tensor(np.ones([16, 64, 32]), dtype=ms.float32)
|
||||
_b = Tensor(np.ones([16, 64, 32]), dtype=ms.float32)
|
||||
_w1 = Tensor(np.ones([128, 64, 32]), dtype=ms.float32)
|
||||
_w2 = Tensor(np.ones([128, 64, 32]), dtype=ms.float32)
|
||||
|
||||
|
||||
def compile_net(net):
|
||||
context.set_context(save_graphs=True)
|
||||
learning_rate = 0.1
|
||||
momentum = 0.9
|
||||
epoch_size = 2
|
||||
dataset = Dataset(_x, _b)
|
||||
opt = Momentum(net.trainable_params(), learning_rate, momentum)
|
||||
model = Model(net, optimizer=opt)
|
||||
model.train(epoch_size, dataset, dataset_sink_mode=False)
|
||||
context.reset_auto_parallel_context()
|
||||
|
||||
|
||||
def test_select_data_parallel():
|
||||
context.set_auto_parallel_context(
|
||||
parallel_mode="semi_auto_parallel", device_num=8, global_rank=0)
|
||||
strategy1 = ((8, 1, 1), (8, 1, 1))
|
||||
strategy2 = ((8, 1, 1), (8, 1, 1), (8, 1, 1))
|
||||
net = Net(_w1, _w2, strategy1, strategy2)
|
||||
compile_net(net)
|
||||
|
||||
|
||||
def test_select_model_parallel():
|
||||
context.set_auto_parallel_context(
|
||||
parallel_mode="semi_auto_parallel", device_num=8, global_rank=0)
|
||||
strategy1 = ((2, 2, 2), (2, 2, 2))
|
||||
strategy2 = ((2, 2, 2), (2, 2, 2), (2, 2, 2))
|
||||
net = Net(_w1, _w2, strategy1, strategy2)
|
||||
compile_net(net)
|
||||
|
||||
|
||||
def test_select_auto_parallel():
|
||||
context.set_auto_parallel_context(
|
||||
parallel_mode="auto_parallel", device_num=8, global_rank=0)
|
||||
net = Net(_w1, _w2)
|
||||
compile_net(net)
|
||||
|
||||
|
||||
def test_select_strategy_error():
|
||||
context.set_auto_parallel_context(
|
||||
parallel_mode="semi_auto_parallel", device_num=8, global_rank=0)
|
||||
strategy1 = ((2, 2, 2), (2, 2, 2))
|
||||
strategy2 = ((8, 1, 1), (2, 2, 2), (2, 2, 2))
|
||||
net = Net(_w1, _w2, strategy1, strategy2)
|
||||
with pytest.raises(RuntimeError):
|
||||
compile_net(net)
|
||||
Loading…
Reference in New Issue