reg op info from local config file

This commit is contained in:
jjfeing 2020-07-09 17:19:06 +08:00
parent 7b65c5483b
commit 53277f8c02
5 changed files with 454 additions and 21 deletions

383
config/op_info.config Normal file

File diff suppressed because one or more lines are too long

View File

@ -103,13 +103,13 @@ class OpInfo {
partial_flag_ = opinfo.partial_flag_; partial_flag_ = opinfo.partial_flag_;
dynamic_format_ = opinfo.dynamic_format_; dynamic_format_ = opinfo.dynamic_format_;
op_pattern_ = opinfo.op_pattern(); op_pattern_ = opinfo.op_pattern();
for (auto attr : opinfo.attrs_ptr()) { for (const auto &attr : opinfo.attrs_ptr()) {
attrs_ptr_.push_back(std::make_shared<OpAttr>(*attr)); attrs_ptr_.push_back(std::make_shared<OpAttr>(*attr));
} }
for (auto input : opinfo.inputs_ptr()) { for (const auto &input : opinfo.inputs_ptr()) {
inputs_ptr_.push_back(std::make_shared<OpIOInfo>(*input)); inputs_ptr_.push_back(std::make_shared<OpIOInfo>(*input));
} }
for (auto output : opinfo.outputs_ptr()) { for (const auto &output : opinfo.outputs_ptr()) {
outputs_ptr_.push_back(std::make_shared<OpIOInfo>(*output)); outputs_ptr_.push_back(std::make_shared<OpIOInfo>(*output));
} }
ref_infos_ = opinfo.ref_infos(); ref_infos_ = opinfo.ref_infos();

View File

@ -19,6 +19,7 @@
#include <unordered_map> #include <unordered_map>
#include <memory> #include <memory>
#include <map> #include <map>
#include <fstream>
#include "utils/log_adapter.h" #include "utils/log_adapter.h"
#include "utils/overload.h" #include "utils/overload.h"
#include "utils/context/ms_context.h" #include "utils/context/ms_context.h"
@ -59,7 +60,7 @@ constexpr auto kNeedCompile = "need_compile";
constexpr auto kShape = "shape"; constexpr auto kShape = "shape";
std::vector<std::shared_ptr<OpInfo>> OpLib::op_info_; std::vector<std::shared_ptr<OpInfo>> OpLib::op_info_;
std::string ImplTypeToStr(OpImplyType impl_type) { static std::string ImplTypeToStr(OpImplyType impl_type) {
switch (impl_type) { switch (impl_type) {
case kTBE: case kTBE:
return kTbe; return kTbe;
@ -124,6 +125,50 @@ void OpLib::DecodeTBESpecificInfo(const nlohmann::json &obj, const std::shared_p
} }
} }
bool OpLib::RegOpFromLocalInfo() {
MS_LOG(INFO) << "Start";
static bool has_load = false;
if (has_load) {
return true;
}
has_load = true;
std::string dir = common::GetEnv("MINDSPORE_OP_INFO_PATH");
if (dir.empty()) {
MS_LOG(INFO) << "MindSpore op info path does not been setted. use op info from python pass.";
return true;
}
char real_path[PATH_MAX] = {0};
if (dir.size() >= PATH_MAX) {
MS_LOG(ERROR) << "Op info path is invalid: " << dir;
return false;
}
#if defined(_WIN32) || defined(_WIN64)
if (_fullpath(real_path, common::SafeCStr(dir), PATH_MAX) == nullptr) {
MS_LOG(ERROR) << "Op info path is invalid: " << dir;
return false;
}
#else
if (realpath(common::SafeCStr(dir), real_path) == nullptr) {
MS_LOG(ERROR) << "Op info path is invalid: " << dir;
return false;
}
#endif
MS_LOG(INFO) << "Start to read op info from local file.";
std::ifstream file(real_path);
if (!file.is_open()) {
MS_LOG(ERROR) << "Find op info file failed.";
return false;
}
std::string line;
while (getline(file, line)) {
if (!line.empty()) {
(void)OpLib::RegOp(line, "");
}
}
MS_LOG(INFO) << "End";
return true;
}
bool OpLib::DecodeOpInfo(const nlohmann::json &obj, const mindspore::kernel::OpImplyType imply_type, bool OpLib::DecodeOpInfo(const nlohmann::json &obj, const mindspore::kernel::OpImplyType imply_type,
const std::string &impl_path) { const std::string &impl_path) {
std::shared_ptr<OpInfo> op_info = std::make_shared<OpInfo>(); std::shared_ptr<OpInfo> op_info = std::make_shared<OpInfo>();
@ -160,14 +205,16 @@ bool OpLib::DecodeOpInfo(const nlohmann::json &obj, const mindspore::kernel::OpI
return false; return false;
} }
} }
if (CheckRepetition(op_info)) {
MS_LOG(WARNING) << "This op info has been already registed. op name: " << op_info->op_name()
<< ", impl type: " << ImplTypeToStr(op_info->imply_type())
<< ", impl path: " << op_info->impl_path();
return true;
}
if (!GetRefInfo(op_info)) { if (!GetRefInfo(op_info)) {
MS_LOG(ERROR) << "GetRefInfo Failed"; MS_LOG(ERROR) << "GetRefInfo Failed";
return false; return false;
} }
if (!CheckRepetition(op_info)) {
MS_LOG(ERROR) << "CheckRepetition Failed";
return false;
}
op_info_.push_back(op_info); op_info_.push_back(op_info);
return true; return true;
} }
@ -269,6 +316,9 @@ bool OpLib::DecodeInputOutput(const nlohmann::json &obj, const OpImplyType imply
} }
std::shared_ptr<OpInfo> OpLib::FindOp(const std::string &op_name, OpImplyType imply_type) { std::shared_ptr<OpInfo> OpLib::FindOp(const std::string &op_name, OpImplyType imply_type) {
if (!OpLib::RegOpFromLocalInfo()) {
MS_LOG(INFO) << "Warning reg local op info failed.";
}
auto context = MsContext::GetInstance(); auto context = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context); MS_EXCEPTION_IF_NULL(context);
bool is_gpu = (context->device_target() == kGPUDevice); bool is_gpu = (context->device_target() == kGPUDevice);
@ -283,8 +333,8 @@ std::shared_ptr<OpInfo> OpLib::FindOp(const std::string &op_name, OpImplyType im
return op_info; return op_info;
} }
} }
MS_LOG(DEBUG) << "FindOp failed: opname: " << op_name << ", imply_type: " << ImplTypeToStr(imply_type) MS_LOG(INFO) << "FindOp failed: opname: " << op_name << ", imply_type: " << ImplTypeToStr(imply_type)
<< ", current op num: " << op_info_.size(); << ", current op num: " << op_info_.size();
return nullptr; return nullptr;
} }
@ -313,17 +363,19 @@ bool OpLib::GetRefInfo(const std::shared_ptr<OpInfo> &op_info) {
} }
bool OpLib::CheckRepetition(const std::shared_ptr<OpInfo> &op_info) { bool OpLib::CheckRepetition(const std::shared_ptr<OpInfo> &op_info) {
bool has_register = false;
MS_EXCEPTION_IF_NULL(op_info); MS_EXCEPTION_IF_NULL(op_info);
for (const auto &exist_op_info : op_info_) { for (const auto &exist_op_info : op_info_) {
MS_EXCEPTION_IF_NULL(exist_op_info); MS_EXCEPTION_IF_NULL(exist_op_info);
if (exist_op_info->op_name() == op_info->op_name() && exist_op_info->imply_type() == op_info->imply_type() && if (exist_op_info->op_name() == op_info->op_name() && exist_op_info->imply_type() == op_info->imply_type() &&
exist_op_info->impl_path() != op_info->impl_path()) { exist_op_info->impl_path() == op_info->impl_path()) {
MS_LOG(ERROR) << "Op has already exist, please use other name, op name: " << op_info->op_name() MS_LOG(INFO) << "Op has already exist, please use other name, op name: " << op_info->op_name()
<< " op type: " << ImplTypeToStr(op_info->imply_type()); << " op type: " << ImplTypeToStr(op_info->imply_type());
return false; has_register = true;
break;
} }
} }
return true; return has_register;
} }
} // namespace kernel } // namespace kernel
} // namespace mindspore } // namespace mindspore

View File

@ -28,11 +28,8 @@ class OpLib {
public: public:
OpLib() = default; OpLib() = default;
virtual ~OpLib() = default; virtual ~OpLib() = default;
bool RegOp(const std::string &json_string, const std::string &impl_path); static bool RegOp(const std::string &json_string, const std::string &impl_path);
static void RegOpInfo(std::shared_ptr<OpInfo> opinfo) { static void RegOpInfo(const std::shared_ptr<OpInfo> &opinfo) { op_info_.emplace_back(opinfo); }
op_info_.emplace_back(opinfo);
return;
}
static std::shared_ptr<OpInfo> FindOp(const std::string &op_name, OpImplyType imply_type); static std::shared_ptr<OpInfo> FindOp(const std::string &op_name, OpImplyType imply_type);
static const std::vector<std::shared_ptr<OpInfo>> &GetAllOpsInfo() { return op_info_; } static const std::vector<std::shared_ptr<OpInfo>> &GetAllOpsInfo() { return op_info_; }
@ -40,6 +37,7 @@ class OpLib {
static std::vector<std::shared_ptr<OpInfo>> op_info_; static std::vector<std::shared_ptr<OpInfo>> op_info_;
private: private:
static bool RegOpFromLocalInfo();
static bool DecodeOpInfo(const nlohmann::json &obj, const OpImplyType imply_type, const std::string &impl_path); static bool DecodeOpInfo(const nlohmann::json &obj, const OpImplyType imply_type, const std::string &impl_path);
static bool DecodeAttr(const nlohmann::json &obj, const OpImplyType imply_type, static bool DecodeAttr(const nlohmann::json &obj, const OpImplyType imply_type,
const std::shared_ptr<OpInfo> &op_info); const std::shared_ptr<OpInfo> &op_info);

View File

@ -323,7 +323,7 @@ PYBIND11_MODULE(_c_expression, m) {
(void)py::class_<OpLib, std::shared_ptr<OpLib>>(m, "Oplib") (void)py::class_<OpLib, std::shared_ptr<OpLib>>(m, "Oplib")
.def(py::init()) .def(py::init())
.def("reg_op", &OpLib::RegOp, "Register op info."); .def_static("reg_op", &OpLib::RegOp, "Register op info.");
#ifdef ENABLE_GPU_COLLECTIVE #ifdef ENABLE_GPU_COLLECTIVE
(void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::InitCollective, (void)m.def("init_gpu_collective", &mindspore::device::gpu::CollectiveInitializer::InitCollective,
"Init gpu collective communication mode."); "Init gpu collective communication mode.");