Clear backend done_tasks in ms_memory_recycle to free memory

mindspore/ccsrc/backend/session/executor.h

@@ -170,6 +170,7 @@ class Executor {
   bool CreateCommGroup(const std::string &group_name, const std::vector<uint32_t> &ranks);
   bool DestroyCommGroup(const std::string &group_name);
   void OnEvent(const ExecutorEvent &event);
+  void ClearDoneTasks();
 
  private:
   void RunTask(const std::shared_ptr<Task> &task, bool sync, bool long_run = false);
@@ -178,7 +179,6 @@ class Executor {
   void OnClear();
   void OnRunGraphFinished();
   void OnException();
-  void ClearDoneTasks();
 
   uint32_t device_id_;
   std::string device_name_;

mindspore/ccsrc/backend/session/executor_manager.h

@@ -32,6 +32,14 @@ class ExecutorManager {
   std::shared_ptr<Executor> GetExecutor(const std::string &device_name, uint32_t device_id);
   void OnEvent(const ExecutorEvent &event);
   void Clear();
+  void ClearDoneTasks() {
+    for (auto &item : executors_) {
+      auto &executor = item.second;
+      if (executor != nullptr) {
+        executor->ClearDoneTasks();
+      }
+    }
+  }
 
  private:
   ExecutorManager() = default;

mindspore/ccsrc/pipeline/jit/pipeline.cc

@@ -1770,6 +1770,7 @@ void MemoryRecycle() {
   mindspore::RDR::ResetRecorder();
 #endif
   ReclaimOptimizer();
+  session::ExecutorManager::Instance().ClearDoneTasks();
   ad::g_k_prims.clear();
   ad::ClearKPynativeCellStaticRes();
   ad::PrimBpropOptimizer::GetPrimBpropOptimizerInst().Clear();