update signal

This commit is contained in:
vector 2021-07-10 11:48:35 +08:00 committed by YijieChen
parent de635448af
commit ee057e57ec
6 changed files with 38 additions and 22 deletions

View File

@ -19,7 +19,7 @@
#include <memory>
#include <utility>
#include <algorithm>
#include "runtime/device/ascend/signal_util.h"
#include "utils/signal_util.h"
#include "debug/data_dump/e2e_dump.h"
#include "runtime/device/ascend/ascend_device_address.h"
#include "utils/ms_context.h"
@ -32,6 +32,7 @@
#include "runtime/device/ascend/ge_runtime/model_runner.h"
#include "runtime/device/ascend/tasksink/task_generator.h"
#include "backend/session/anf_runtime_algorithm.h"
#include "backend/session/kernel_build_client.h"
#include "runtime/device/ascend/profiling/profiling_utils.h"
#include "runtime/device/ascend/ascend_memory_manager.h"
#include "runtime/device/ascend/ascend_event.h"
@ -105,6 +106,13 @@ std::string GetRankId() {
}
return rank_id_str;
}
void IntHandler(int, siginfo_t *, void *) {
mindspore::kernel::AscendKernelBuildClient::Instance().Close();
int this_pid = getpid();
MS_LOG(WARNING) << "Process " << this_pid << " receive KeyboardInterrupt signal.";
(void)kill(this_pid, SIGTERM);
}
} // namespace
std::vector<rtExceptionInfo> AscendKernelRuntime::task_fail_infoes_ = {};
@ -585,7 +593,7 @@ void AscendKernelRuntime::DumpTaskExceptionInfo(const session::KernelGraph *grap
bool AscendKernelRuntime::Run(session::KernelGraph *const graph, bool is_task_sink) {
const uint64_t kUSecondInSecond = 1000000;
SignalGuard sg;
SignalGuard sg(IntHandler);
MS_EXCEPTION_IF_NULL(graph);
bool ret = false;

View File

@ -27,6 +27,9 @@
#include "utils/log_adapter.h"
#include "utils/convert_utils.h"
#include "utils/ms_context.h"
#if !defined(_WIN32) && !defined(_WIN64)
#include "utils/signal_util.h"
#endif
#include "common/trans.h"
#include "debug/data_dump/dump_json_parser.h"
#ifdef ENABLE_DUMP_IR
@ -402,6 +405,14 @@ bool RunInStepMode(const ActorSet *actor_set, const std::vector<TensorPtr> *inpu
MsException::Instance().CheckException();
return result_future.IsOK();
}
#if !defined(_WIN32) && !defined(_WIN64)
void IntHandler(int, siginfo_t *, void *) {
int this_pid = getpid();
MS_LOG(WARNING) << "Process " << this_pid << " receive KeyboardInterrupt signal.";
(void)kill(this_pid, SIGTERM);
}
#endif
} // namespace
void GraphScheduler::Clear() {
@ -749,7 +760,9 @@ void GraphScheduler::PrepareDataForControlNode(HostQueueDataSourceActor *host_da
bool GraphScheduler::Run(const ActorSet *actor_set, GraphExecutionStrategy strategy,
const std::vector<TensorPtr> *input_tensors) {
MS_EXCEPTION_IF_NULL(actor_set);
#if !defined(_WIN32) && !defined(_WIN64)
SignalGuard sg(IntHandler);
#endif
if (strategy == GraphExecutionStrategy::kStep) {
return RunInStepMode(actor_set, input_tensors);
}

View File

@ -1,4 +1,8 @@
file(GLOB_RECURSE _UTILS_SRC_LIST ./*.cc)
if(CMAKE_SYSTEM_NAME MATCHES "Windows")
file(GLOB_RECURSE _UTILS_SIGNAL_SRC_FILES ./signal_util.cc)
list(REMOVE_ITEM _UTILS_SRC_LIST ${_UTILS_SIGNAL_SRC_FILES})
endif()
if(NOT ENABLE_GE)
file(GLOB_RECURSE _UTILS_GE_SRC_FILES ./callbacks_ge.cc)

View File

@ -14,13 +14,13 @@
* limitations under the License.
*/
#include "runtime/device/ascend/signal_util.h"
#include "utils/signal_util.h"
#include <csignal>
#include "utils/log_adapter.h"
#include "backend/session/kernel_build_client.h"
namespace mindspore {
SignalGuard::SignalGuard() { RegisterHandlers(); }
SignalGuard::SignalGuard(IntHandlerFunc IntHandler) { RegisterHandlers(IntHandler); }
SignalGuard::~SignalGuard() {
if (old_handler != nullptr) {
@ -32,23 +32,16 @@ SignalGuard::~SignalGuard() {
}
}
void SignalGuard::RegisterHandlers() {
void SignalGuard::RegisterHandlers(IntHandlerFunc IntHandler) {
struct sigaction old_int_action;
(void)sigaction(SIGINT, nullptr, &old_int_action);
if (old_int_action.sa_sigaction != nullptr) {
MS_LOG(DEBUG) << "The signal has been registered";
old_handler = old_int_action.sa_sigaction;
}
int_action.sa_sigaction = &IntHandler;
int_action.sa_sigaction = IntHandler;
(void)sigemptyset(&int_action.sa_mask);
int_action.sa_flags = SA_RESTART | SA_SIGINFO;
(void)sigaction(SIGINT, &int_action, nullptr);
}
void SignalGuard::IntHandler(int, siginfo_t *, void *) {
kernel::AscendKernelBuildClient::Instance().Close();
int this_pid = getpid();
MS_LOG(WARNING) << "Process " << this_pid << " receive KeyboardInterrupt signal.";
(void)kill(this_pid, SIGTERM);
}
} // namespace mindspore

View File

@ -13,23 +13,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_SIGNAL_UTIL_H_
#define MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_SIGNAL_UTIL_H_
#ifndef MINDSPORE_CCSRC_UTILS_SIGNAL_UTIL_H_
#define MINDSPORE_CCSRC_UTILS_SIGNAL_UTIL_H_
#include <csignal>
namespace mindspore {
typedef void (*IntHandlerFunc)(int, siginfo_t *, void *);
class SignalGuard {
public:
SignalGuard();
explicit SignalGuard(IntHandlerFunc func);
~SignalGuard();
private:
void RegisterHandlers();
static void IntHandler(int sig_num, siginfo_t *sig_info, void *context);
void RegisterHandlers(IntHandlerFunc func);
void (*old_handler)(int, siginfo_t *, void *) = nullptr;
struct sigaction int_action;
};
} // namespace mindspore
#endif // MINDSPORE_CCSRC_RUNTIME_DEVICE_ASCEND_SIGNAL_UTIL_H_
#endif // MINDSPORE_CCSRC_UTILS_SIGNAL_UTIL_H_

View File

@ -122,7 +122,6 @@ file(GLOB_RECURSE MINDSPORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
"../../../mindspore/ccsrc/runtime/device/ascend/ascend_event.cc"
"../../../mindspore/ccsrc/runtime/device/ascend/kernel_build_ascend.cc"
"../../../mindspore/ccsrc/runtime/device/ascend/ascend_kernel_runtime.cc"
"../../../mindspore/ccsrc/runtime/device/ascend/signal_util.cc"
"../../../mindspore/ccsrc/runtime/device/ascend/ascend_memory_manager.cc"
"../../../mindspore/ccsrc/runtime/device/ascend/ascend_device_address.cc"
"../../../mindspore/ccsrc/runtime/device/ascend/ascend_memory_pool.cc"