!9063 Throw exception if using call stack for CNodes in the same func graph.

From: @zh_qh
Reviewed-by: @stsuteng,@chenfei52
Signed-off-by: @stsuteng

mindspore/ccsrc/debug/trace.cc

@@ -73,7 +73,7 @@ std::string GetGraphParamString(const FuncGraphPtr &graph, abstract::AbstractBas
 }
 
 void DumpInferStack(std::ostringstream &oss) {
-  auto &infer_stack = GetCurrenGraphInferStack();
+  auto &infer_stack = GetCurrenGraphEvalStack();
   if (infer_stack.empty()) {
     return;
   }
@@ -104,7 +104,7 @@ void DumpInferStack(std::ostringstream &oss) {
 }
 
 void TraceGraphEval() {
-  auto &infer_stack = GetCurrenGraphInferStack();
+  auto &infer_stack = GetCurrenGraphEvalStack();
   std::ostringstream oss;
   if (infer_stack.empty()) {
     return;
@@ -551,9 +551,10 @@ void TraceEvalCNodeLeave() { cnode_debug_stack.pop_back(); }
 
 std::vector<abstract::AnfNodeConfigPtr> &GetCNodeDebugStack() { return cnode_debug_stack; }
 
-std::stack<std::pair<abstract::EvaluatorPtr, abstract::AnfNodeConfigPtr>> &GetCurrenGraphInferStack() {
+std::stack<std::pair<abstract::EvaluatorPtr, abstract::AnfNodeConfigPtr>> &GetCurrenGraphEvalStack() {
   return graph_infer_stack;
 }
+
 void ClearTraceStack() {
   while (!graph_infer_stack.empty()) {
     graph_infer_stack.pop();

mindspore/ccsrc/debug/trace.h

@@ -40,7 +40,7 @@ void TraceGraphEvalLeave(const abstract::EvaluatorPtr &eval);
 void TraceEvalCNodeEnter(const abstract::AnfNodeConfigPtr &node_cfg);
 void TraceEvalCNodeLeave();
 std::vector<abstract::AnfNodeConfigPtr> &GetCNodeDebugStack();
-std::stack<std::pair<abstract::EvaluatorPtr, abstract::AnfNodeConfigPtr>> &GetCurrenGraphInferStack();
+std::stack<std::pair<abstract::EvaluatorPtr, abstract::AnfNodeConfigPtr>> &GetCurrenGraphEvalStack();
 std::string GetAbstractStr(const abstract::AbstractBasePtr &abs);
 void ClearTraceStack();
 }  // namespace trace

mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.cc

@@ -30,6 +30,7 @@
 #include "pipeline/jit/parse/data_converter.h"
 #include "pipeline/jit/static_analysis/evaluator.h"
 #include "debug/trace.h"
+#include "debug/anf_ir_dump.h"
 
 namespace mindspore {
 namespace abstract {
@@ -181,6 +182,7 @@ EvalResultPtr AnalysisEngine::Eval(const AnfNodeConfigPtr &conf) {
     auto value_node = node->cast<ValueNodePtr>();
     eval_result = std::make_shared<EvalResult>(EvalValueNode(value_node, conf), nullptr);
   } else if (node->isa<CNode>()) {
+    CheckNoStackInSameFuncGraph(conf);
     auto cnode = node->cast<CNodePtr>();
     trace::TraceEvalCNodeEnter(conf);
     eval_result = EvalCNode(cnode, conf);
@@ -202,6 +204,44 @@ EvalResultPtr AnalysisEngine::Eval(const AnfNodeConfigPtr &conf) {
   return eval_result;
 }
 
+void AnalysisEngine::CheckNoStackInSameFuncGraph(const AnfNodeConfigPtr &conf) {
+  auto &list = trace::GetCNodeDebugStack();
+  if (list.empty()) {
+    return;
+  }
+  auto &previous_stack = list.back();
+  auto previous_cnode_fg = previous_stack->node()->func_graph();
+  auto current_cnode_fg = conf->node()->func_graph();
+  if (previous_cnode_fg != current_cnode_fg) {  // Normal.
+    return;
+  }
+  if (forward_count_ != 0) {  // Ignore Forward Config.
+    return;
+  }
+  auto &infer_stack = trace::GetCurrenGraphEvalStack();
+  if (infer_stack.empty()) {
+    return;
+  }
+  auto top_evaluator = infer_stack.top().first;
+  if (!top_evaluator->isa<BaseFuncGraphEvaluator>()) {
+    MS_LOG(EXCEPTION) << "Top evaluator is " << top_evaluator->ToString();
+  }
+  auto top_fg_evaluator = dyn_cast<BaseFuncGraphEvaluator>(top_evaluator);
+  auto top_context_fg = top_fg_evaluator->graph_context()->func_graph();
+  if (current_cnode_fg != top_context_fg) {  // Ignore FV call.
+    return;
+  }
+  MS_LOG(ERROR) << "Should not use call stack in the same function: " << top_context_fg->ToString() << ", for "
+                << conf->node()->DebugString(2);
+  for (size_t i = 0; i < list.size(); ++i) {
+    auto old_conf = list[i];
+    MS_LOG(ERROR) << "  #" << i << ": " << old_conf->node()->DebugString(2) << ", in "
+                  << old_conf->context()->func_graph()->ToString();
+  }
+  DumpIR("use_stack_error.ir", conf->node()->func_graph());
+  MS_LOG(EXCEPTION) << "To check above CNode stack and dumped use_stack_error.ir";
+}
+
 AbstractBasePtr AnalysisEngine::EvalValueNode(const ValueNodePtr &value_node, const AnfNodeConfigPtr &conf) {
   MS_EXCEPTION_IF_NULL(conf);
   MS_EXCEPTION_IF_NULL(value_node);

mindspore/ccsrc/pipeline/jit/static_analysis/static_analysis.h

@@ -187,6 +187,7 @@ class AnalysisEngine : public std::enable_shared_from_this<AnalysisEngine> {
   AnalysisEngine(const PrimEvaluatorMap &prim_evaluator_map, const FuncGraphManagerPtr &func_graph_manager)
       : cache_(AnalysisCache()), prim_constructors_(prim_evaluator_map), func_graph_manager_(func_graph_manager) {
     function_call_depth_ = 0;
+    forward_count_ = 0;
   }
   ~AnalysisEngine() = default;
 
@@ -226,7 +227,10 @@ class AnalysisEngine : public std::enable_shared_from_this<AnalysisEngine> {
     (void)anfnode_config_map_.emplace(orig_conf, new_conf);
     MS_LOG(DEBUG) << "Forward orig_conf: " << orig_conf->node()->DebugString()
                   << ", to new_conf: " << new_conf->node()->DebugString();
-    return GetEvaluatedValue(new_conf);
+    forward_count_++;
+    auto res = GetEvaluatedValue(new_conf);
+    forward_count_--;
+    return res;
   }
   const PrimEvaluatorMap &PrimConstructors() const { return prim_constructors_; }
 
@@ -246,6 +250,8 @@ class AnalysisEngine : public std::enable_shared_from_this<AnalysisEngine> {
 
   uint64_t function_call_depth() { return function_call_depth_; }
 
+  void CheckNoStackInSameFuncGraph(const AnfNodeConfigPtr &conf);
+
  private:
   void SetUndeterminedFlag(const EvaluatorPtr &evaluator);
   EvaluatorPtr HandleNestedRecursion(const std::vector<EvaluatorPtr> &evaluators, const EvaluatorPtr &eval,
@@ -275,6 +281,8 @@ class AnalysisEngine : public std::enable_shared_from_this<AnalysisEngine> {
   // record current depth of function call statck
   uint64_t function_call_depth_;
 
+  uint64_t forward_count_;
+
 #ifdef DEBUG
   std::vector<AnfNodePtr> compute_conf_stack_;
 #endif