p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!26892 ME compiler environment variables normalization 

Merge pull request !26892 from chenfei_mindspore/environment_variable_fix
This commit is contained in:
i-robot 2021-12-06 11:16:57 +00:00 committed by Gitee
parent 9c6fa583f5 c5c2b2f186
commit b4716271bc
13 changed files with 117 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mindspore/_extends/parse/parser.py
View File

@ -134,7 +134,7 @@ def get_bprop_method_of_class(obj, parse_method=None):
# The fallback feature is enabled in default.
# Not support change the flag during the process is alive.
support_fallback_ = os.getenv('ENV_SUPPORT_FALLBACK')
support_fallback_ = os.getenv('DEV_ENV_ENABLE_FALLBACK')
def resolve_symbol(namespace, symbol):
@ -519,7 +519,7 @@ def eval_script(exp_str, params):
obj = eval(exp_str, global_params, local_params)
except Exception as e:
error_info = f"When eval '{exp_str}' by using Fallback feature, an error occurred: " + str(e) + \
". You can try to turn off the Fallback feature by 'export ENV_SUPPORT_FALLBACK=0'."
". You can try to turn off the Fallback feature by 'export DEV_ENV_ENABLE_FALLBACK=0'."
logger.error(error_info)
raise e

2
mindspore/ccsrc/backend/optimizer/common/pass_manager.cc
View File

@ -130,7 +130,7 @@ void PassManager::DumpPassIR(const FuncGraphPtr &func_graph, const std::string &
auto context_ptr = MsContext::GetInstance();
MS_EXCEPTION_IF_NULL(context_ptr);
bool save_graphs = context_ptr->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG);
static const auto enable_dump = (common::GetEnv("ENV_NO_DUMP_BE_PASS_IR") != "1");
static const auto enable_dump = !GetDumpConfig().disable_backend_dump;
if (save_graphs && enable_dump) {
std::ostringstream oss;
oss << "verbose_ir_files"

94
mindspore/ccsrc/debug/anf_ir_dump.cc
View File

@ -585,16 +585,96 @@ void DumpSubgraph(const OrderedMap<FuncGraphPtr, std::shared_ptr<SubGraphIRInfo>
}
}
void GetEnvDumpIrLineLevel(LocDumpMode *dump_location) {
void SetDumpConfigByString(const std::string &str, DumpConfig *dump_config) {
static mindspore::HashMap<std::string, enum LocDumpMode> dump_level_map = {
{std::to_string(kOff), kOff}, {std::to_string(kTopStack), kTopStack}, {std::to_string(kWholeStack), kWholeStack}};
static const auto dump_level_in_env = common::GetEnv("ENV_DUMP_IR_LINE_LEVEL");
auto it = dump_level_map.find(dump_level_in_env);
if (it == dump_level_map.end()) {
{kDumpConfigLineLevel0, kOff}, {kDumpConfigLineLevel1, kTopStack}, {kDumpConfigLineLevel2, kWholeStack}};
auto it = dump_level_map.find(str);
if (it != dump_level_map.end()) {
dump_config->enable_dump_pass_ir = it->second;
return;
}
// Use the env setting instead parameter setting.
*dump_location = it->second;
if (str == kDumpConfigDisableBackend) {
dump_config->disable_backend_dump = true;
return;
}
if (str == kDumpConfigEnablePassIR) {
dump_config->enable_dump_pass_ir = true;
return;
}
}
DumpConfig GetDumpConfig() {
static std::vector<HashSet<std::string>> config_white_list = {
{kDumpConfigLineLevel0, kDumpConfigLineLevel1, kDumpConfigLineLevel2},
{kDumpConfigDisableBackend},
{kDumpConfigEnablePassIR}};
static DumpConfig dump_config;
static bool parsed = false;
if (parsed) {
return dump_config;
}
parsed = true;
// Start parse config.
std::string str(common::GetEnv("DEV_ENV_DUMP_IR_CONFIG"));
std::vector<std::shared_ptr<HashSet<std::string>>> configs = {std::make_shared<HashSet<std::string>>(),
std::make_shared<HashSet<std::string>>(),
std::make_shared<HashSet<std::string>>()};
auto constexpr max_string_len = 100;
if (str.size() > max_string_len) {
MS_LOG(WARNING) << "Dump ir config length exceed max length: " << max_string_len;
return dump_config;
}
if (str.empty()) {
return dump_config;
}
size_t start_pos = 0;
// if '#' is the last char of str, the str is illegal, so we use '<=' but not '<'.
while (start_pos <= str.size()) {
auto pos = str.find('#', start_pos);
if (pos == std::string::npos) {
pos = str.size();
}
auto substr = str.substr(start_pos, pos - start_pos);
start_pos = pos + 1;
bool is_illegal_config = true;
for (size_t i = 0; i < config_white_list.size(); i++) {
if (config_white_list[i].find(substr) != config_white_list[i].end()) {
is_illegal_config = false;
(void)configs[i]->insert(substr);
if (configs[i]->size() > 1) {
std::ostringstream buffer;
std::for_each(configs[i]->begin(), configs[i]->end(), [&buffer](const std::string &config) {
buffer << "\n" << config;
});
MS_LOG(WARNING) << "Dump configs are conflict. Conflict configs: " << buffer.str() << "\n"
<< "Please keep only one of them.";
return dump_config;
}
}
}
if (is_illegal_config) {
std::ostringstream buffer;
buffer << "Support configs:\n"
<< "[0]: " << kDumpConfigLineLevel0 << "\n"
<< "[1]: " << kDumpConfigLineLevel1 << "\n"
<< "[2]: " << kDumpConfigLineLevel2 << "\n"
<< "[3]: " << kDumpConfigDisableBackend << "\n"
<< "[4]: " << kDumpConfigEnablePassIR;
MS_LOG(WARNING) << "Illegal dump config:\n" << substr << "\n" << buffer.str();
return {};
}
}
for (auto &config : configs) {
SetDumpConfigByString(*config->begin(), &dump_config);
}
return dump_config;
}
void GetEnvDumpIrLineLevel(LocDumpMode *dump_location) {
const auto &config = GetDumpConfig();
if (config.dump_line_level != kInValid) {
*dump_location = config.dump_line_level;
}
}
#ifdef ENABLE_DUMP_IR

15
mindspore/ccsrc/debug/anf_ir_dump.h
View File

@ -21,9 +21,21 @@
#include "ir/dtype/type.h"
#include "ir/anf.h"
#include "debug/common.h"
#include "utils/hash_set.h"
namespace mindspore {
enum LocDumpMode { kOff = 0, kTopStack = 1, kWholeStack = 2 };
enum LocDumpMode { kOff = 0, kTopStack = 1, kWholeStack = 2, kInValid = 3 };
auto constexpr kDumpConfigLineLevel0 = "LINE_LEVEL0";
auto constexpr kDumpConfigLineLevel1 = "LINE_LEVEL1";
auto constexpr kDumpConfigLineLevel2 = "LINE_LEVEL2";
auto constexpr kDumpConfigDisableBackend = "DISABLE_BACKEND";
auto constexpr kDumpConfigEnablePassIR = "ENABLE_PASS_IR";
typedef struct {
LocDumpMode dump_line_level = kInValid;
bool disable_backend_dump = false;
bool enable_dump_pass_ir = false;
} DumpConfig;
constexpr char PARALLEL_STRATEGY[] = "strategy";
void DumpIR(const std::string &filename, const FuncGraphPtr &func_graph, bool dump_full_name = false,
LocDumpMode dump_location = kOff, const std::string &target_file = "");
@ -32,6 +44,7 @@ void GatherInputAndOutputInferType(std::ostringstream &buffer, const AnfNodePtr
void DumpIRForRDR(const std::string &filename, const FuncGraphPtr &func_graph, bool dump_full_name = false,
LocDumpMode dump_location = kOff);
const std::string ToShortString(const TypeId &typeId);
DumpConfig GetDumpConfig();
} // namespace mindspore
#endif // MINDSPORE_CCSRC_DEBUG_ANF_IR_DUMP_H_

3
mindspore/ccsrc/frontend/optimizer/ad/grad.cc
View File

@ -80,8 +80,7 @@ FuncGraphPtr Grad(const FuncGraphPtr &func_graph, const pipeline::ResourceBasePt
manager_ptr->AddFuncGraph(func_graph);
FuncGraphPtr grad_fg = func_graph;
lift_fv_before_grad = (common::GetEnv("ENV_DONT_LIFT_FV_BEFORE_GRAD") != "1");
if (lift_fv_before_grad && func_graph->func_graphs_used().size() != 0) {
if (func_graph->func_graphs_used().size() != 0) {
grad_fg = LiftFv(resources, func_graph);
}
auto multi_graph_sink = [&func_graph](const FuncGraphPtr &f) {

6
mindspore/ccsrc/frontend/optimizer/opt.cc
View File

@ -301,7 +301,7 @@ bool SubstitutionList::ApplySubstitutionsToIR(const OptimizerPtr &optimizer, con
changes = changes || change;
loop = loop || change;
#ifdef ENABLE_DUMP_IR
static const auto enable_dump_pass_ir = (common::GetEnv("ENV_DUMP_PASS_IR") == "1");
static const auto enable_dump_pass_ir = GetDumpConfig().enable_dump_pass_ir;
if (enable_dump_pass_ir && MsContext::GetInstance()->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG)) {
auto fg_name = optimizer->name() + "_r" + std::to_string(optimizer->CurPass_.counter) + "_" +
optimizer->CurPass_.name + "_" + substitution->name_;
@ -338,8 +338,8 @@ bool SubstitutionList::operator()(const FuncGraphPtr &func_graph, const Optimize
manager->AddFuncGraph(func_graph);
bool changes = false;
static const auto traverse_mode =
(common::GetEnv("ENV_TRAVERSE_SUBSTITUTIONS_MODE") != "1" ? kOptTraverseFromIRToSubstitutions
: kOptTraverseFromSubstitutionsToIR);
(common::GetEnv("DEV_ENV_TRAVERSE_SUBSTITUTIONS_MODE") != "1" ? kOptTraverseFromIRToSubstitutions
: kOptTraverseFromSubstitutionsToIR);
if (traverse_mode == kOptTraverseFromIRToSubstitutions &&
MsContext::GetInstance()->get_param<int>(MS_CTX_EXECUTION_MODE) != kPynativeMode &&
optimizer->traverse_nodes_first() && !is_once_ && !global_sensitive_) {

2
mindspore/ccsrc/frontend/optimizer/optimizer.h
View File

@ -195,7 +195,7 @@ class Optimizer : public std::enable_shared_from_this<Optimizer> {
};
use_profile ? (WITH(MsProfile::GetProfile()->Step(pass_names_[i])) opt_func) : opt_func();
#ifdef ENABLE_DUMP_IR
static const auto enable_dump_pass_ir = (common::GetEnv("ENV_DUMP_PASS_IR") == "1");
static const auto enable_dump_pass_ir = GetDumpConfig().enable_dump_pass_ir;
if (enable_dump_pass_ir && MsContext::GetInstance()->get_param<bool>(MS_CTX_SAVE_GRAPHS_FLAG)) {
auto fg_name =
"opt_substep_" + name_ + "_r" + std::to_string(counter) + "_" + std::to_string(i) + "_" + pass_names_[i];

2
mindspore/ccsrc/pipeline/jit/parse/data_converter.cc
View File

@ -382,7 +382,7 @@ ValuePtr ConvertOtherObj(const py::object &obj) {
// Start RESOLVE_TYPE_INVALID...
// The fallback feature is enabled in default.
// Not support change the flag during the process is alive.
static const auto support_fallback = common::GetEnv("ENV_SUPPORT_FALLBACK");
static const auto support_fallback = common::GetEnv("DEV_ENV_ENABLE_FALLBACK");
static const auto use_fallback = (support_fallback != "0");
if (use_fallback) {
auto res = std::make_shared<InterpretedObject>(obj, py::str(obj));

4
mindspore/ccsrc/pipeline/jit/parse/parse.cc
View File

@ -92,8 +92,8 @@ AnfNodePtr GetMixedPrecisionCastHelp(const FuncGraphPtr &func_graph, const AnfNo
FuncGraphWeakPtr Parser::top_func_graph_ = FuncGraphWeakPtr();
Parser::Parser(const std::shared_ptr<ParseFunctionAst> &ast) : ast_(ast) {
max_for_loop_count_str_ = common::GetEnv("ENV_FOR_TO_WHILE_LOOP");
support_fallback_ = common::GetEnv("ENV_SUPPORT_FALLBACK");
max_for_loop_count_str_ = common::GetEnv("DEV_ENV_FOR_TO_WHILE_LOOP");
support_fallback_ = common::GetEnv("DEV_ENV_ENABLE_FALLBACK");
errcode_ = PARSE_SUCCESS;
BuildMethodMap();
}

2
mindspore/ccsrc/pipeline/jit/parse/resolve.cc
View File

@ -59,7 +59,7 @@ abstract::AbstractBasePtr ClassType::ToAbstract() {
// The fallback feature is enabled in default.
// Not support change the flag during the process is alive.
static const auto support_fallback = common::GetEnv("ENV_SUPPORT_FALLBACK");
static const auto support_fallback = common::GetEnv("DEV_ENV_ENABLE_FALLBACK");
static const auto use_fallback = (support_fallback != "0");
if (use_fallback && !IsSupportedCreateInstanceType(obj())) {
return abs_scalar;

2
mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.cc
View File

@ -573,7 +573,7 @@ EvalResultPtr AnalysisEngine::ExecuteEvaluators(const std::vector<EvaluatorPtr>
MS_EXCEPTION_IF_NULL(eval);
return eval->Run(shared_from_this(), args_conf_list, out_conf);
}
static bool enable_singleThread = (common::GetEnv("ENV_SINGLE_EVAL") == "1");
static bool enable_singleThread = (common::GetEnv("DEV_ENV_SINGLE_EVAL") == "1");
if (enable_singleThread) {
return ExecuteMultipleEvaluators(evaluators, out_conf, args_conf_list);
} else {

2
mindspore/core/utils/ms_utils.h
View File

@ -45,7 +45,7 @@ inline const char *SafeCStr(const std::string &str) { return str.c_str(); }
const char *SafeCStr(const std::string &&str);
static inline std::string GetEnv(const std::string &envvar) {
const char *value = ::getenv(envvar.c_str());
const char *value = std::getenv(envvar.c_str());
if (value == nullptr) {
return std::string();

4
tests/st/control/test_for_to_while.py
View File

@ -53,7 +53,7 @@ def test_single_for_01():
y = Tensor([5], mstype.int32)
z = Tensor([4], mstype.int32)
os.environ['ENV_FOR_TO_WHILE_LOOP'] = '1'
os.environ['DEV_ENV_FOR_TO_WHILE_LOOP'] = '1'
# graph mode
context.set_context(mode=context.GRAPH_MODE)
for_net = SingleForNet()
@ -67,7 +67,7 @@ def test_single_for_01():
net = GradNet(for_net)
pynative_forward_res = for_net(x, y, z)
pynative_backward_res = net(x, y, z)
os.environ['ENV_FOR_TO_WHILE_LOOP'] = ''
os.environ['DEV_ENV_FOR_TO_WHILE_LOOP'] = ''
assert graph_forward_res == pynative_forward_res
assert graph_backward_res == pynative_backward_res