forked from mindspore-Ecosystem/mindspore
add topk op
This commit is contained in:
yangzhenzhang
parent
cb3571a1ca
commit
d070af122f
|
|
@ -204,6 +204,7 @@ using RsqrtCost = SqrtCost;
|
|||
using AsinhCost = SqrtCost;
|
||||
using AcoshCost = SqrtCost;
|
||||
using ReLUV2Cost = SqrtCost;
|
||||
using TopKCost = SqrtCost;
|
||||
|
||||
class ReLU6Cost : public CastCost {
|
||||
public:
|
||||
|
|
|
|||
|
|
@ -192,6 +192,7 @@ REGISTER(ConcatInfo);
|
|||
REGISTER(SplitInfo);
|
||||
REGISTER(UniqueInfo);
|
||||
REGISTER(GatherNdInfo);
|
||||
REGISTER(TopKInfo);
|
||||
} // namespace parallel
|
||||
} // namespace mindspore
|
||||
|
||||
|
|
|
|||
|
|
@ -51,5 +51,6 @@
|
|||
#include "frontend/parallel/ops_info/uniform_candidate_sampler_info.h"
|
||||
#include "frontend/parallel/ops_info/reluv2_info.h"
|
||||
#include "frontend/parallel/ops_info/gathernd_info.h"
|
||||
#include "frontend/parallel/ops_info/topk_info.h"
|
||||
|
||||
#endif // MINDSPORE_CCSRC_FRONTEND_PARALLEL_OPS_INFO_HEAD_FILES_H_
|
||||
|
|
|
|||
|
|
@ -0,0 +1,233 @@
|
|||
/**
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#include "frontend/parallel/ops_info/topk_info.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <memory>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "frontend/parallel/device_matrix.h"
|
||||
#include "frontend/parallel/strategy.h"
|
||||
#include "frontend/parallel/tensor_layout/tensor_redistribution.h"
|
||||
#include "pipeline/jit/resource.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace parallel {
|
||||
Status TopKInfo::CheckStrategy(const StrategyPtr &strategy) {
|
||||
MS_EXCEPTION_IF_NULL(strategy);
|
||||
if (CheckStrategyValue(strategy, inputs_shape_) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Invalid strategy";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
std::vector<Dimensions> stra = strategy->GetInputDim();
|
||||
if (stra.empty()) {
|
||||
MS_LOG(ERROR) << name_ << ": The strategy is empty";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
if (stra[0].back() != 1) {
|
||||
MS_LOG(ERROR) << name_ << ": Now we can not support to split last dimension";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status TopKInfo::InferDevMatrixShape() {
|
||||
MS_EXCEPTION_IF_NULL(strategy_);
|
||||
std::vector<Dimensions> stra = strategy_->GetInputDim();
|
||||
if (stra.empty()) {
|
||||
MS_LOG(ERROR) << name_ << "The strategy is empty";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
dev_matrix_shape_ = stra[0];
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status TopKInfo::InferTensorMap() {
|
||||
TensorMap tensor_map;
|
||||
if (inputs_shape_.empty()) {
|
||||
MS_LOG(ERROR) << name_ << "The inputs shape is empty";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
// cannot use dev_matrix_shape_ replace inputs_shape_[0], because it may not be fully split in all devices.
|
||||
int64_t size = SizeToLong(inputs_shape_[0].size());
|
||||
for (int64_t i = 0; i < size; ++i) {
|
||||
tensor_map.push_back(size - i - 1);
|
||||
}
|
||||
|
||||
for (size_t i = 0; i < inputs_shape_.size(); ++i) {
|
||||
inputs_tensor_map_.push_back(tensor_map);
|
||||
}
|
||||
outputs_tensor_map_.push_back(tensor_map); // values
|
||||
outputs_tensor_map_.push_back(tensor_map); // indices
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status TopKInfo::InferTensorInfo() {
|
||||
if (inputs_shape_.empty() || outputs_shape_.empty() || inputs_tensor_map_.empty() || outputs_tensor_map_.empty()) {
|
||||
MS_LOG(ERROR) << name_ << ": Invalid args";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
TensorLayout input_layout, output_layout;
|
||||
for (size_t i = 0; i < inputs_shape_.size(); ++i) {
|
||||
// infer tensor layout
|
||||
if (input_layout.InitFromVector(dev_matrix_shape_, inputs_tensor_map_[i], inputs_shape_[i]) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Infer input tensor layout failed.";
|
||||
return FAILED;
|
||||
}
|
||||
TensorInfo input_tensor_info(input_layout);
|
||||
inputs_tensor_info_.push_back(input_tensor_info);
|
||||
}
|
||||
|
||||
if (output_layout.InitFromVector(dev_matrix_shape_, outputs_tensor_map_[0], outputs_shape_[0]) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Infer output tensor layout failed.";
|
||||
return FAILED;
|
||||
}
|
||||
TensorInfo output_tensor_info(output_layout);
|
||||
outputs_tensor_info_.push_back(output_tensor_info); // values
|
||||
outputs_tensor_info_.push_back(output_tensor_info); // indices
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status TopKInfo::InferAsLossDivisor() {
|
||||
if (outputs_tensor_map_.empty()) {
|
||||
MS_LOG(ERROR) << name_ << ": The outputs tensor map is empty";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
MS_LOG(INFO) << name_ << " has two outputs, use output[0] to infer";
|
||||
if (outputs_tensor_map_[0].empty()) {
|
||||
as_loss_divisor_ = stage_device_size_;
|
||||
MS_LOG(INFO) << name_ << ": The output is a scalar, use the dev size" << as_loss_divisor_ << " as loss divisor";
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
as_loss_divisor_ = ComputeRepeatDeviceNumByTensorMap(dev_matrix_shape_, outputs_tensor_map_[0]);
|
||||
|
||||
std::string dev_matrix_shape_str = ShapeToString(dev_matrix_shape_);
|
||||
std::string output_tensor_map_str = ShapeToString(outputs_tensor_map_[0]);
|
||||
MS_LOG(INFO) << name_ << ": the dev matrix shape, the output tensor map, and loss divisor is " << dev_matrix_shape_str
|
||||
<< ", " << output_tensor_map_str << ", " << as_loss_divisor_;
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status TopKInfo::InferMirrorOps() {
|
||||
mirror_ops_.clear();
|
||||
if (inputs_shape_.empty()) {
|
||||
MS_LOG(INFO) << name_ << ": The inputs size is empty";
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
if (inputs_tensor_map_.size() != inputs_shape_.size()) {
|
||||
MS_LOG(ERROR) << name_ << ": The size of inputs tensor map is not equal to the size of inputs shape";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
bool group_is_empty = true;
|
||||
for (size_t i = 0; i < inputs_tensor_map_.size(); ++i) {
|
||||
std::vector<Group> group;
|
||||
if (CreateGroupByTensorMap(inputs_tensor_map_[i], &group) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Create group failed, the input index is " << i;
|
||||
mirror_ops_.clear();
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
OperatorVector mirror_op;
|
||||
if (group.empty()) {
|
||||
MS_LOG(INFO) << name_ << ": The mirror group is empty, the input index is " << i;
|
||||
mirror_ops_.push_back(mirror_op);
|
||||
continue;
|
||||
}
|
||||
|
||||
group_is_empty = false;
|
||||
mirror_op = CreateMirrorOps(group[0].name(), group[0].GetDevNum());
|
||||
mirror_ops_.push_back(mirror_op);
|
||||
}
|
||||
|
||||
if (group_is_empty) {
|
||||
mirror_ops_.clear();
|
||||
MS_LOG(INFO) << name_ << ": No need to insert mirror ops";
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
OperatorVector tmp_mirror_op; // tmp mirror op for 'k'
|
||||
mirror_ops_.push_back(tmp_mirror_op);
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status TopKInfo::SetCostUnderStrategy(const StrategyPtr &strategy) { return SetCostUnderStrategyBase(strategy); }
|
||||
|
||||
Status TopKInfo::GenerateStrategies(int64_t stage_id) {
|
||||
if (InferAttrs() != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Infer attrs failed";
|
||||
return FAILED;
|
||||
}
|
||||
if (inputs_shape_.empty()) {
|
||||
MS_LOG(ERROR) << name_ << ": The inputs shape is empty";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
// to generate the first input's strategy
|
||||
Shape input_split(inputs_shape_[0].size(), 1);
|
||||
input_split.back() = 0; // the last dimension can not be split
|
||||
Shapes splittable_input = {input_split};
|
||||
Shapes tmp_inputs_shape = {inputs_shape_[0]};
|
||||
|
||||
std::vector<StrategyPtr> sp_vector;
|
||||
if (GenerateStrategiesForIndependentInputs(stage_id, tmp_inputs_shape, splittable_input, &sp_vector) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Generate strategies failed";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
size_t success = 0;
|
||||
for (auto &sp : sp_vector) {
|
||||
PrintStrategy(sp);
|
||||
if (SetCostUnderStrategy(sp) == SUCCESS) {
|
||||
success++;
|
||||
MS_LOG(INFO) << name_ << ": Successfully generated " << success << " strategy.";
|
||||
PrintStrategy(sp);
|
||||
}
|
||||
}
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status TopKInfo::Init(const StrategyPtr &strategy) {
|
||||
if (InitWithAutoRepeatCalc(strategy) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Init failed.";
|
||||
return FAILED;
|
||||
}
|
||||
MS_LOG(INFO) << name_ << ": Init success.";
|
||||
return SUCCESS;
|
||||
}
|
||||
|
||||
Status TopKInfo::InitForCostModel(const StrategyPtr &strategy) {
|
||||
if (InitForCostModelWithAutoRepeatCalc(strategy) != SUCCESS) {
|
||||
MS_LOG(ERROR) << name_ << ": Init for cost model failed.";
|
||||
return FAILED;
|
||||
}
|
||||
|
||||
MS_LOG(INFO) << name_ << ": Init for cost model success.";
|
||||
return SUCCESS;
|
||||
}
|
||||
} // namespace parallel
|
||||
} // namespace mindspore
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
/**
|
||||
* Copyright 2021 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef MINDSPORE_CCSRC_FRONTEND_PARALLEL_OPS_INFO_TOPK_INFO_H_
|
||||
#define MINDSPORE_CCSRC_FRONTEND_PARALLEL_OPS_INFO_TOPK_INFO_H_
|
||||
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "ir/value.h"
|
||||
#include "frontend/parallel/auto_parallel/operator_costmodel.h"
|
||||
#include "frontend/parallel/ops_info/operator_info.h"
|
||||
#include "frontend/parallel/strategy.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace parallel {
|
||||
// the last dimension of input can not be split, other dimensions can be split
|
||||
class TopKInfo : public OperatorInfo {
|
||||
public:
|
||||
TopKInfo(const std::string &operator_name, const Shapes &inputs_shape, const Shapes &outputs_shape,
|
||||
const PrimitiveAttrs &attrs)
|
||||
: OperatorInfo(operator_name, inputs_shape, outputs_shape, attrs, std::make_shared<TopKCost>()) {}
|
||||
~TopKInfo() override = default;
|
||||
|
||||
Status Init(const StrategyPtr &strategy) override;
|
||||
Status InitForCostModel(const StrategyPtr &strategy) override;
|
||||
Status GenerateStrategies(int64_t) override;
|
||||
Status SetCostUnderStrategy(const StrategyPtr &) override;
|
||||
|
||||
protected:
|
||||
Status GetAttrs() override { return SUCCESS; }
|
||||
Status CheckStrategy(const StrategyPtr &strategy) override;
|
||||
Status InferForwardCommunication() override { return SUCCESS; }
|
||||
Status InferMirrorOps() override; // can not use OperatorInfo::InferMirrorOps(), since the 'k' of topk is scalar
|
||||
Status InferTensorInfo() override;
|
||||
Status InferDevMatrixShape() override;
|
||||
Status InferTensorMap() override;
|
||||
Status InferAsLossDivisor() override;
|
||||
};
|
||||
|
||||
using TopKInfoPtr = std::shared_ptr<TopKInfo>;
|
||||
} // namespace parallel
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // MINDSPORE_CCSRC_FRONTEND_PARALLEL_OPS_INFO_TOPK_INFO_H_
|
||||
|
|
@ -163,8 +163,7 @@ bool IsSplittableOperator(const std::string &op_name) {
|
|||
BESSELI0E, BESSELI1E, FLOORMOD, ASSIGN, ASSIGN_ADD, ATAN2, DIVNONAN, LOGICALAND, LOGICALOR, ELU, RELU6, RELUV2,
|
||||
SOFTPLUS, SOFTSIGN, GREATEREQUAL, LESSEQUAL, LESS, APPROXIMATEEQUAL, MOD, UNIQUE, UNSORTED_SEGMENT_SUM,
|
||||
UNSORTED_SEGMENT_MIN, REPEAT_ELEMENTS, TENSOR_DOT, RANGE, UNIFORM_CANDIDATE_SAMPLER, SLICE,
|
||||
UNSORTED_SEGMENT_MAX, GATHER_ND};
|
||||
|
||||
UNSORTED_SEGMENT_MAX, GATHER_ND, TOPK};
|
||||
// clang-format on
|
||||
|
||||
auto iter = splittable_op.find(op_name);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,109 @@
|
|||
# Copyright 2021 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
import mindspore as ms
|
||||
from mindspore import context, Tensor, Parameter
|
||||
from mindspore.nn import Cell, Momentum
|
||||
from mindspore.ops import operations as P
|
||||
from mindspore.train import Model
|
||||
from tests.dataset_mock import MindData
|
||||
|
||||
|
||||
class Dataset(MindData):
|
||||
def __init__(self, predict, label, length=3):
|
||||
super(Dataset, self).__init__(size=length)
|
||||
self.predict = predict
|
||||
self.label = label
|
||||
self.index = 0
|
||||
self.length = length
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
if self.index >= self.length:
|
||||
raise StopIteration
|
||||
self.index += 1
|
||||
return self.predict, self.label
|
||||
|
||||
def reset(self):
|
||||
self.index = 0
|
||||
|
||||
|
||||
class Net(Cell):
|
||||
def __init__(self, w1, strategy1=None, strategy2=None):
|
||||
super().__init__()
|
||||
self.mul = P.Mul().shard(strategy1)
|
||||
self.w1 = Parameter(w1, "w1")
|
||||
self.topk = P.TopK().shard(strategy2)
|
||||
|
||||
def construct(self, x, b):
|
||||
out = self.mul(x, self.w1)
|
||||
out, _ = self.topk(out, 8)
|
||||
return out
|
||||
|
||||
|
||||
_x = Tensor(np.ones([16, 64]), dtype=ms.float32)
|
||||
_b = Tensor(np.ones([16, 64]), dtype=ms.float32)
|
||||
_w1 = Tensor(np.ones([128, 64]), dtype=ms.float32)
|
||||
|
||||
|
||||
def compile_net(net):
|
||||
context.set_context(save_graphs=True)
|
||||
learning_rate = 0.1
|
||||
momentum = 0.9
|
||||
epoch_size = 2
|
||||
dataset = Dataset(_x, _b)
|
||||
opt = Momentum(net.trainable_params(), learning_rate, momentum)
|
||||
model = Model(net, optimizer=opt)
|
||||
model.train(epoch_size, dataset, dataset_sink_mode=False)
|
||||
context.reset_auto_parallel_context()
|
||||
|
||||
|
||||
def test_topk_data_parallel():
|
||||
context.set_auto_parallel_context(
|
||||
parallel_mode="semi_auto_parallel", device_num=8, global_rank=0)
|
||||
strategy1 = ((8, 1), (8, 1))
|
||||
strategy2 = ((8, 1),)
|
||||
net = Net(_w1, strategy1, strategy2)
|
||||
compile_net(net)
|
||||
|
||||
|
||||
def test_topk_model_parallel():
|
||||
context.set_auto_parallel_context(
|
||||
parallel_mode="semi_auto_parallel", device_num=8, global_rank=0)
|
||||
strategy1 = ((2, 4), (2, 4))
|
||||
strategy2 = ((2, 1),)
|
||||
net = Net(_w1, strategy1, strategy2)
|
||||
compile_net(net)
|
||||
|
||||
|
||||
def test_topk_auto_parallel():
|
||||
context.set_auto_parallel_context(
|
||||
parallel_mode="auto_parallel", device_num=8, global_rank=0)
|
||||
net = Net(_w1)
|
||||
compile_net(net)
|
||||
|
||||
|
||||
def test_topk_strategy_error():
|
||||
context.set_auto_parallel_context(
|
||||
parallel_mode="semi_auto_parallel", device_num=8, global_rank=0)
|
||||
strategy1 = ((8, 1), (8, 1))
|
||||
strategy2 = ((1, 8),)
|
||||
net = Net(_w1, strategy1, strategy2)
|
||||
with pytest.raises(RuntimeError):
|
||||
compile_net(net)
|
||||
Loading…
Reference in New Issue