diff --git a/CMakeLists.txt b/CMakeLists.txt index 2b4a85b3baf..b724259f147 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -81,9 +81,4 @@ if (ENABLE_TESTCASES) add_subdirectory(tests) endif() -if (ENABLE_SERVING) - add_subdirectory(serving) - add_subdirectory(serving/example/cpp_client) -endif() - include(cmake/package.cmake) diff --git a/build.sh b/build.sh index 2ea42120e5e..219a0590cf9 100755 --- a/build.sh +++ b/build.sh @@ -54,7 +54,6 @@ usage() echo " -V Specify the device version, if -e gpu, default CUDA 10.1, if -e ascend, default Ascend 910" echo " -I Enable compiling mindspore lite for arm64, arm32 or x86_64, default disable mindspore lite compilation" echo " -K Compile with AKG, default on" - echo " -s Enable serving module, default off" echo " -B Enable debugger, default on" echo " -E Enable IBVERBS for parameter server, default off" echo " -l Compile with python dependency, default on" @@ -105,7 +104,6 @@ checkopts() SUPPORT_TRAIN="off" USE_GLOG="on" ENABLE_AKG="on" - ENABLE_SERVING="off" ENABLE_ACL="off" ENABLE_DEBUGGER="on" ENABLE_IBVERBS="off" @@ -123,7 +121,7 @@ checkopts() DEVICE="" ENABLE_NPU="off" # Process the options - while getopts 'drvj:c:t:hsb:a:g:p:ie:m:l:I:LRP:D:zM:V:K:swB:En:T:A:C:o:S:k:W:' opt + while getopts 'drvj:c:t:hsb:a:g:p:ie:m:l:I:LRP:D:zM:V:K:B:En:T:A:C:o:S:k:W:' opt do OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]') case "${opt}" in @@ -273,16 +271,6 @@ checkopts() ENABLE_AKG="on" echo "enable compile with akg" ;; - s) - ENABLE_SERVING="on" - echo "enable serving" - ;; - w) - ENABLE_SERVING="on" - echo "enable serving" - ENABLE_ACL="on" - echo "enable acl" - ;; B) check_on_off $OPTARG B ENABLE_DEBUGGER="$OPTARG" @@ -366,12 +354,10 @@ checkopts() DEVICE_VERSION=910 fi if [[ "X$DEVICE_VERSION" == "X310" ]]; then - ENABLE_SERVING="on" ENABLE_ACL="on" elif [[ "X$DEVICE_VERSION" == "X910" ]]; then ENABLE_D="on" ENABLE_CPU="on" - ENABLE_SERVING="on" else echo "Invalid value ${DEVICE_VERSION} for option -V" usage @@ -467,9 +453,6 @@ build_mindspore() if [[ "X$ENABLE_AKG" = "Xon" ]] && [[ "X$ENABLE_D" = "Xon" || "X$ENABLE_GPU" = "Xon" ]]; then CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_AKG=ON" fi - if [[ "X$ENABLE_SERVING" = "Xon" ]]; then - CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_SERVING=ON" - fi if [[ "X$ENABLE_ACL" = "Xon" ]]; then CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_ACL=ON" fi diff --git a/cmake/mind_expression.cmake b/cmake/mind_expression.cmake index 901ae9f2aa5..c98143d7c4e 100644 --- a/cmake/mind_expression.cmake +++ b/cmake/mind_expression.cmake @@ -88,7 +88,7 @@ if (ENABLE_MINDDATA) include(${CMAKE_SOURCE_DIR}/cmake/external_libs/sentencepiece.cmake) endif() -if (ENABLE_MINDDATA OR ENABLE_SERVING) +if (ENABLE_MINDDATA) include(${CMAKE_SOURCE_DIR}/cmake/external_libs/jpeg_turbo.cmake) endif() diff --git a/cmake/options.cmake b/cmake/options.cmake index 3433c57a0d7..8578c899f2b 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -119,7 +119,7 @@ if(ENABLE_DEBUGGER) add_compile_definitions(ENABLE_DEBUGGER) endif() -if (ENABLE_DEBUGGER OR ENABLE_SERVING OR ENABLE_TESTCASES) +if (ENABLE_DEBUGGER OR ENABLE_TESTCASES) set(MS_BUILD_GRPC ON) endif() if (ENABLE_MINDDATA AND NOT CMAKE_SYSTEM_NAME MATCHES "Windows") diff --git a/cmake/package.cmake b/cmake/package.cmake index c3d9bce0f26..f7e48b5eb97 100644 --- a/cmake/package.cmake +++ b/cmake/package.cmake @@ -202,7 +202,7 @@ if (ENABLE_CPU AND (ENABLE_D OR ENABLE_GPU)) ) endif() -if (ENABLE_SERVING OR ENABLE_TESTCASES) +if (ENABLE_TESTCASES) file(GLOB_RECURSE LIBEVENT_LIB_LIST ${libevent_LIBPATH}/libevent* ${libevent_LIBPATH}/libevent_pthreads* @@ -336,29 +336,3 @@ install( COMPONENT mindspore ) -if (ENABLE_SERVING) - install( - TARGETS ms_serving - DESTINATION ${INSTALL_BASE_DIR} - COMPONENT mindspore - ) - - install( - FILES ${CMAKE_SOURCE_DIR}/build/mindspore/serving/ms_service_pb2.py - ${CMAKE_SOURCE_DIR}/build/mindspore/serving/ms_service_pb2_grpc.py - DESTINATION ${INSTALL_PY_DIR} - COMPONENT mindspore - ) - - install( - TARGETS inference - DESTINATION ${INSTALL_LIB_DIR} - COMPONENT mindspore - ) - - install( - FILES ${LIBEVENT_LIB_LIST} - DESTINATION ${INSTALL_LIB_DIR} - COMPONENT mindspore - ) -endif () diff --git a/mindspore/ccsrc/CMakeLists.txt b/mindspore/ccsrc/CMakeLists.txt index e48e5a7e807..86627eaf331 100644 --- a/mindspore/ccsrc/CMakeLists.txt +++ b/mindspore/ccsrc/CMakeLists.txt @@ -350,44 +350,9 @@ if (ENABLE_MINDDATA) add_subdirectory(minddata/dataset) endif () -# build inference -set(LOAD_MINDIR_SRC - ${CMAKE_SOURCE_DIR}/mindspore/core/load_mindir/load_model.cc - ${CMAKE_SOURCE_DIR}/mindspore/core/load_mindir/anf_model_parser.cc - ) -add_library(inference SHARED - ${CMAKE_CURRENT_SOURCE_DIR}/backend/session/infer_session.cc - ${LOAD_MINDIR_SRC} - ) - -set_target_properties(inference PROPERTIES INSTALL_RPATH ${MINDSPORE_RPATH}) - -if (CMAKE_SYSTEM_NAME MATCHES "Darwin") - target_link_libraries(inference PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY} - -Wl,-force_load mindspore proto_input -Wl,-noall_load mindspore_gvar) -else() - target_link_libraries(inference PRIVATE ${PYTHON_LIBRARIES} ${SECUREC_LIBRARY} - -Wl,--whole-archive mindspore proto_input -Wl,--no-whole-archive mindspore_gvar) -endif() - if (ENABLE_D) find_library(adump_server libadump_server.a ${ASCEND_RUNTIME_PATH} ${ASCEND_TOOLKIT_RUNTIME_PATH}) target_link_libraries(_c_expression PRIVATE ${adump_server}) - target_link_libraries(inference PRIVATE ${adump_server} ms_profile) endif() -if (ENABLE_CPU) - target_link_libraries(inference PRIVATE mindspore::dnnl mindspore::mkldnn) -endif () - -if (USE_GLOG) - target_link_libraries(inference PRIVATE mindspore::glog) -endif() - -if (CMAKE_SYSTEM_NAME MATCHES "Linux") - target_link_options(inference PRIVATE -Wl,-init,common_log_init) -elseif (CMAKE_SYSTEM_NAME MATCHES "Darwin") - set_target_properties(inference PROPERTIES MACOSX_RPATH ON) -endif () - add_subdirectory(cxx_api) diff --git a/mindspore/ccsrc/backend/session/infer_session.cc b/mindspore/ccsrc/backend/session/infer_session.cc deleted file mode 100644 index d653423f895..00000000000 --- a/mindspore/ccsrc/backend/session/infer_session.cc +++ /dev/null @@ -1,341 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "backend/session/infer_session.h" -#include -#include -#include - -#include "include/inference.h" -#include "backend/session/session_basic.h" -#include "backend/session/session_factory.h" -#include "backend/session/executor_manager.h" -#include "base/base_ref_utils.h" -#include "load_mindir/load_model.h" -#include "backend/kernel_compiler/oplib/oplib.h" -#include "utils/context/context_extends.h" -#include "runtime/device/kernel_runtime_manager.h" - -#include "pybind11/pybind11.h" - -#ifdef ENABLE_D -#include "utils/ms_context.h" -#endif - -using std::string; -using std::vector; - -namespace py = pybind11; -namespace mindspore { -namespace inference { -std::shared_ptr InferSession::CreateSession(const std::string &device, uint32_t device_id) { - try { - auto session = std::make_shared(); - Status ret = session->InitEnv(device, device_id); - if (ret != SUCCESS) { - return nullptr; - } - return session; - } catch (std::bad_alloc &e) { - MS_LOG(ERROR) << "Inference CreatSession failed, failed to alloc memory"; - return nullptr; - } -} - -MSInferSession::MSInferSession() = default; -MSInferSession::~MSInferSession() = default; - -Status MSInferSession::LoadModelFromFile(const std::string &file_name, uint32_t &model_id) { - Py_Initialize(); - auto graph = mindspore::LoadMindIR(file_name); - if (graph == nullptr) { - MS_LOG(ERROR) << "Load graph model failed, file name is " << file_name.c_str(); - return FAILED; - } - Status ret = CompileGraph(graph, model_id); - if (ret != SUCCESS) { - MS_LOG(ERROR) << "Compile graph model failed, file name is " << file_name.c_str(); - return FAILED; - } - MS_LOG(INFO) << "Load model from file " << file_name << " success"; - -#ifdef ENABLE_D - // set d context - rtError_t rt_ret = rtCtxGetCurrent(&context_); - if (rt_ret != RT_ERROR_NONE || context_ == nullptr) { - MS_LOG(ERROR) << "the ascend device context is null"; - return FAILED; - } -#endif - - return SUCCESS; -} - -Status MSInferSession::UnloadModel(uint32_t model_id) { return SUCCESS; } - -Status ServingTensor2MSTensor(size_t index, const InferTensorBase &out_tensor, tensor::TensorPtr &ms_tensor) { - std::vector shape = out_tensor.shape(); - TypeId data_type; - const std::map type2id_map{ - {inference::kMSI_Unknown, TypeId::kNumberTypeBegin}, {inference::kMSI_Bool, TypeId::kNumberTypeBool}, - {inference::kMSI_Int8, TypeId::kNumberTypeInt8}, {inference::kMSI_Uint8, TypeId::kNumberTypeUInt8}, - {inference::kMSI_Int16, TypeId::kNumberTypeInt16}, {inference::kMSI_Uint16, TypeId::kNumberTypeUInt16}, - {inference::kMSI_Int32, TypeId::kNumberTypeInt32}, {inference::kMSI_Uint32, TypeId::kNumberTypeUInt32}, - {inference::kMSI_Int64, TypeId::kNumberTypeInt64}, {inference::kMSI_Uint64, TypeId::kNumberTypeUInt64}, - {inference::kMSI_Float16, TypeId::kNumberTypeFloat16}, {inference::kMSI_Float32, TypeId::kNumberTypeFloat32}, - {inference::kMSI_Float64, TypeId::kNumberTypeFloat64}, - }; - auto it = type2id_map.find(out_tensor.data_type()); - if (it == type2id_map.end()) { - MSI_LOG_WARNING << "undefined MSI data type " << out_tensor.data_type(); - return FAILED; - } else { - data_type = it->second; - } - - ms_tensor = std::make_shared(data_type, shape); - if (out_tensor.data_size() == 0 || ms_tensor->Size() != out_tensor.data_size()) { - MSI_LOG_ERROR << "input " << std::to_string(index) - << " data size not match shape and dtype, calculated required size " << ms_tensor->Size() - << ", given " << out_tensor.data_size(); - return INFER_STATUS(INVALID_INPUTS) << "input " << std::to_string(index) - << " data size not match shape and dtype, calculated required size " - << ms_tensor->Size() << ", given " << out_tensor.data_size(); - } - if (out_tensor.data() == nullptr || ms_tensor->data_c() == nullptr) { - MSI_LOG_ERROR << "invalid data buffer"; - return FAILED; - } - auto ret_code = memcpy_s(ms_tensor->data_c(), ms_tensor->Size(), out_tensor.data(), out_tensor.data_size()); - if (ret_code != 0) { - MS_LOG(ERROR) << "Failed to copy data from ms_tensor to out_tensor."; - } - return SUCCESS; -} - -void MSTensor2ServingTensor(tensor::TensorPtr ms_tensor, InferTensorBase &out_tensor) { - vector shape = ms_tensor->shape(); - out_tensor.set_shape(shape); - - const std::map id2type_map{ - {TypeId::kNumberTypeBegin, inference::kMSI_Unknown}, {TypeId::kNumberTypeBool, inference::kMSI_Bool}, - {TypeId::kNumberTypeFloat64, inference::kMSI_Float64}, {TypeId::kNumberTypeInt8, inference::kMSI_Int8}, - {TypeId::kNumberTypeUInt8, inference::kMSI_Uint8}, {TypeId::kNumberTypeInt16, inference::kMSI_Int16}, - {TypeId::kNumberTypeUInt16, inference::kMSI_Uint16}, {TypeId::kNumberTypeInt32, inference::kMSI_Int32}, - {TypeId::kNumberTypeUInt32, inference::kMSI_Uint32}, {TypeId::kNumberTypeInt64, inference::kMSI_Int64}, - {TypeId::kNumberTypeUInt64, inference::kMSI_Uint64}, {TypeId::kNumberTypeFloat16, inference::kMSI_Float16}, - {TypeId::kNumberTypeFloat32, inference::kMSI_Float32}, - }; - auto it = id2type_map.find(ms_tensor->data_type()); - if (it == id2type_map.end()) { - MSI_LOG_WARNING << "undefined MS data type " << ms_tensor->data_type(); - out_tensor.set_data_type(inference::kMSI_Unknown); - } else { - out_tensor.set_data_type(it->second); - } - out_tensor.set_data(ms_tensor->data_c(), ms_tensor->Size()); -} - -Status MSInferSession::ExecuteModel(uint32_t model_id, const RequestBase &request, ReplyBase &reply) { -#ifdef ENABLE_D - if (context_ == nullptr) { - MS_LOG(ERROR) << "rtCtx is nullptr"; - return FAILED; - } - rtError_t rt_ret = rtCtxSetCurrent(context_); - if (rt_ret != RT_ERROR_NONE) { - MS_LOG(ERROR) << "set Ascend rtCtx failed"; - return FAILED; - } -#endif - - vector inputs; - for (size_t i = 0; i < request.size(); i++) { - if (request[i] == nullptr) { - MS_LOG(ERROR) << "Execute Model " << model_id << " Failed， input tensor is null, index " << i; - return FAILED; - } - tensor::TensorPtr input = nullptr; - auto ret = ServingTensor2MSTensor(i, *request[i], input); - if (ret != SUCCESS) { - MS_LOG(ERROR) << "Tensor convert failed"; - return ret; - } - inputs.push_back(input); - } - - auto ret = CheckModelInputs(model_id, inputs); - if (ret != SUCCESS) { - MS_LOG(ERROR) << "Check Model " << model_id << " Inputs Failed"; - return ret; - } - vector outputs = RunGraph(model_id, inputs); - if (outputs.empty()) { - MS_LOG(ERROR) << "Execute Model " << model_id << " Failed"; - return FAILED; - } - reply.clear(); - for (const auto &tensor : outputs) { - auto out_tensor = reply.add(); - if (out_tensor == nullptr) { - MS_LOG(ERROR) << "Execute Model " << model_id << " Failed add output tensor failed"; - return FAILED; - } - MSTensor2ServingTensor(tensor, *out_tensor); - } - return SUCCESS; -} - -Status MSInferSession::FinalizeEnv() { - session::ExecutorManager::Instance().Clear(); - device::KernelRuntimeManager::Instance().ClearRuntimeResource(); - auto ms_context = MsContext::GetInstance(); - if (ms_context == nullptr) { - MS_LOG(ERROR) << "Get Context failed!"; - return FAILED; - } - if (!context::CloseTsd(ms_context)) { - MS_LOG(ERROR) << "Inference CloseTsd failed!"; - return FAILED; - } - return SUCCESS; -} - -void MSInferSession::RegAllOp() { - static std::mutex init_mutex; - static bool Initialized = false; - - std::lock_guard lock(init_mutex); - if (Initialized) { - return; - } - Initialized = true; - MsContext::GetInstance()->set_param(MS_CTX_EXECUTION_MODE, kGraphMode); - Py_Initialize(); - auto c_expression = PyImport_ImportModule("mindspore._c_expression"); - MS_EXCEPTION_IF_NULL(c_expression); - PyObject *c_expression_dict = PyModule_GetDict(c_expression); - MS_EXCEPTION_IF_NULL(c_expression_dict); - - PyObject *op_info_loader_class = PyDict_GetItemString(c_expression_dict, "OpInfoLoaderPy"); - MS_EXCEPTION_IF_NULL(op_info_loader_class); - PyObject *op_info_loader = PyInstanceMethod_New(op_info_loader_class); - MS_EXCEPTION_IF_NULL(op_info_loader); - PyObject *op_info_loader_ins = PyObject_CallObject(op_info_loader, nullptr); - MS_EXCEPTION_IF_NULL(op_info_loader_ins); - auto all_ops_info_vector_addr_ul = PyObject_CallMethod(op_info_loader_ins, "get_all_ops_info", nullptr); - MS_EXCEPTION_IF_NULL(all_ops_info_vector_addr_ul); - auto all_ops_info_vector_addr = PyLong_AsVoidPtr(all_ops_info_vector_addr_ul); - auto all_ops_info = static_cast *>(all_ops_info_vector_addr); - for (auto op_info : *all_ops_info) { - kernel::OpLib::RegOpInfo(std::shared_ptr(op_info)); - } - all_ops_info->clear(); - delete all_ops_info; - Py_DECREF(op_info_loader); - Py_DECREF(op_info_loader_class); - Py_DECREF(c_expression_dict); - Py_DECREF(c_expression); - return; -} - -Status MSInferSession::CompileGraph(std::shared_ptr funcGraphPtr, uint32_t &model_id) { - MS_ASSERT(session_impl_ != nullptr); - try { - auto graph_id = session_impl_->CompileGraph(NOT_NULL(funcGraphPtr)); - py::gil_scoped_release gil_release; - model_id = graph_id; - return SUCCESS; - } catch (std::exception &e) { - MS_LOG(ERROR) << "Inference CompileGraph failed"; - return FAILED; - } -} - -std::vector MSInferSession::RunGraph(uint32_t graph_id, - const std::vector &inputs) { - try { - VectorRef outputs; - session_impl_->RunGraph(graph_id, inputs, &outputs); - return TransformVectorRefToMultiTensor(outputs); - } catch (std::exception &e) { - MS_LOG(ERROR) << "Inference Rungraph failed"; - return std::vector(); - } -} - -string MSInferSession::AjustTargetName(const std::string &device) { - if (device == kAscendDevice) { - return std::string(kAscendDevice) + "Inference"; - } else { - MS_LOG(ERROR) << "Only support device Ascend right now"; - return ""; - } -} - -Status MSInferSession::InitEnv(const std::string &device, uint32_t device_id) { - RegAllOp(); - auto ms_context = MsContext::GetInstance(); - if (ms_context == nullptr) { - MS_LOG(ERROR) << "Get Context failed!"; - return FAILED; - } - ms_context->set_param(MS_CTX_EXECUTION_MODE, kGraphMode); - ms_context->set_param(MS_CTX_DEVICE_ID, device_id); - auto ajust_device = AjustTargetName(device); - if (ajust_device == "") { - return FAILED; - } - ms_context->set_param(MS_CTX_DEVICE_TARGET, device); - if (!context::OpenTsd(ms_context)) { - MS_LOG(ERROR) << "Session init OpenTsd failed!"; - return FAILED; - } - session_impl_ = session::SessionFactory::Get().Create(ajust_device); - if (session_impl_ == nullptr) { - MS_LOG(ERROR) << "Session create failed!, please make sure target device:" << device << " is available."; - return FAILED; - } - session_impl_->Init(device_id); - return SUCCESS; -} - -Status MSInferSession::CheckModelInputs(uint32_t graph_id, const std::vector &inputs) const { - MS_ASSERT(session_impl_ != nullptr); - std::string error_msg; - if (!session_impl_->CheckModelInputs(graph_id, inputs, &error_msg)) { - return INFER_STATUS(INVALID_INPUTS) << error_msg; - } - return SUCCESS; -} - -Status MSInferSession::GetModelInputsInfo(uint32_t model_id, std::vector *tensor_list) const { - vector inputs; - vector input_names; - session_impl_->GetModelInputsInfo(model_id, &inputs, &input_names); - if (inputs.size() == 0) { - MS_LOG(ERROR) << "The model inputs is NULL"; - return FAILED; - } - for (const auto &tensor : inputs) { - InferTensor infer_tensor = InferTensor(); - MSTensor2ServingTensor(tensor, infer_tensor); - tensor_list->push_back(infer_tensor); - } - return SUCCESS; -} -} // namespace inference -} // namespace mindspore diff --git a/mindspore/ccsrc/backend/session/infer_session.h b/mindspore/ccsrc/backend/session/infer_session.h deleted file mode 100644 index 1c8fd2ded12..00000000000 --- a/mindspore/ccsrc/backend/session/infer_session.h +++ /dev/null @@ -1,65 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MINDSPORE_CCSRC_SESSION_SESSION_H -#define MINDSPORE_CCSRC_SESSION_SESSION_H - -#include -#include -#include -#include -#include -#include - -#include "backend/session/session_basic.h" -#include "ir/anf.h" -#include "include/inference.h" - -#ifdef ENABLE_D -#include "runtime/context.h" -#endif - -namespace mindspore { -namespace inference { -class MSInferSession : public InferSession { - public: - MSInferSession(); - ~MSInferSession(); - - Status InitEnv(const std::string &device_type, uint32_t device_id) override; - Status FinalizeEnv() override; - Status LoadModelFromFile(const std::string &file_name, uint32_t &model_id) override; - Status UnloadModel(uint32_t model_id) override; - Status ExecuteModel(uint32_t model_id, const RequestBase &inputs, ReplyBase &outputs) override; - Status GetModelInputsInfo(uint32_t graph_id, std::vector *tensor_list) const override; - - private: - std::shared_ptr session_impl_ = nullptr; - std::vector graph_id_; - std::string device_type_; - int32_t device_id_ = 0; -#ifdef ENABLE_D - rtContext_t context_ = nullptr; -#endif - - static void RegAllOp(); - string AjustTargetName(const std::string &device); - Status CompileGraph(std::shared_ptr funcGraphPtr, uint32_t &model_id); - Status CheckModelInputs(uint32_t graph_id, const std::vector &inputs) const; - std::vector RunGraph(uint32_t graph_id, const std::vector &inputs); -}; -} // namespace inference -} // namespace mindspore -#endif // MINDSPORE_CCSRC_SESSION_SESSION_BASIC_H diff --git a/serving/CMakeLists.txt b/serving/CMakeLists.txt deleted file mode 100644 index 6a9e26eda42..00000000000 --- a/serving/CMakeLists.txt +++ /dev/null @@ -1,125 +0,0 @@ -find_package(Threads REQUIRED) - -# This branch assumes that gRPC and all its dependencies are already installed -# on this system, so they can be located by find_package(). - -# Find Protobuf installation -# Looks for protobuf-config.cmake file installed by Protobuf's cmake installation. - -#set(protobuf_MODULE_COMPATIBLE TRUE) -#find_package(Protobuf CONFIG REQUIRED) -#message(STATUS "Using protobuf ${protobuf_VERSION}") -add_library(protobuf::libprotobuf ALIAS protobuf::protobuf) -add_executable(protobuf::libprotoc ALIAS protobuf::protoc) - -set(_PROTOBUF_LIBPROTOBUF protobuf::libprotobuf) -if (CMAKE_CROSSCOMPILING) - find_program(_PROTOBUF_PROTOC protoc) -else () - set(_PROTOBUF_PROTOC $) -endif () - -# Find gRPC installation -# Looks for gRPCConfig.cmake file installed by gRPC's cmake installation. -if (EXISTS ${grpc_ROOT}/lib64) - set(gRPC_DIR "${grpc_ROOT}/lib64/cmake/grpc") -else () - set(gRPC_DIR "${grpc_ROOT}/lib/cmake/grpc") -endif () -message("serving using grpc_DIR : " ${gPRC_DIR}) - -find_package(gRPC CONFIG REQUIRED) -message(STATUS "Using gRPC ${gRPC_VERSION}") - -set(_GRPC_GRPCPP gRPC::grpc++) -set(_REFLECTION gRPC::grpc++_reflection) - -if (CMAKE_CROSSCOMPILING) - find_program(_GRPC_CPP_PLUGIN_EXECUTABLE grpc_cpp_plugin) - find_program(_GRPC_PYTHON_PLUGIN_EXECUTABLE grpc_python_plugin) -else () - set(_GRPC_CPP_PLUGIN_EXECUTABLE $) - set(_GRPC_PYTHON_PLUGIN_EXECUTABLE $) -endif () - -# Proto file -get_filename_component(hw_proto "ms_service.proto" ABSOLUTE) -get_filename_component(hw_proto_path "${hw_proto}" PATH) -# Generated sources -set(hw_proto_srcs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.pb.cc") -set(hw_proto_hdrs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.pb.h") -set(hw_grpc_srcs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.grpc.pb.cc") -set(hw_grpc_hdrs "${CMAKE_CURRENT_BINARY_DIR}/ms_service.grpc.pb.h") -set(hw_py_pb2 "${CMAKE_CURRENT_BINARY_DIR}/ms_service_pb2.py") -set(hw_py_pb2_grpc "${CMAKE_CURRENT_BINARY_DIR}/ms_service_pb2_grpc.py") -add_custom_command( - OUTPUT "${hw_proto_srcs}" "${hw_proto_hdrs}" "${hw_grpc_srcs}" "${hw_grpc_hdrs}" "${hw_py_pb2}" "${hw_py_pb2_grpc}" - COMMAND ${_PROTOBUF_PROTOC} - ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" - --cpp_out "${CMAKE_CURRENT_BINARY_DIR}" - -I "${hw_proto_path}" - --plugin=protoc-gen-grpc="${_GRPC_CPP_PLUGIN_EXECUTABLE}" - "${hw_proto}" - COMMAND ${_PROTOBUF_PROTOC} - ARGS --grpc_out "${CMAKE_CURRENT_BINARY_DIR}" - --python_out "${CMAKE_CURRENT_BINARY_DIR}" - -I "${hw_proto_path}" - --plugin=protoc-gen-grpc="${_GRPC_PYTHON_PLUGIN_EXECUTABLE}" - "${hw_proto}" - DEPENDS "${hw_proto}") - -# Include generated *.pb.h files -include_directories("${CMAKE_CURRENT_BINARY_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}" "${CMAKE_CURRENT_SOURCE_DIR}/core" - "${PROJECT_SOURCE_DIR}/mindspore/ccsrc" "${PROJECT_SOURCE_DIR}/mindspore/core") -file(GLOB_RECURSE CORE_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} - "core/*.cc" "core/util/*.cc" "core/version_control/*.cc") - -list(APPEND SERVING_SRC "main.cc" ${hw_proto_srcs} ${hw_grpc_srcs} ${CORE_SRC_LIST}) - -if (ENABLE_ACL) - if (DEFINED ENV{ASCEND_CUSTOM_PATH}) - set(ASCEND_PATH $ENV{ASCEND_CUSTOM_PATH}) - else () - set(ASCEND_PATH /usr/local/Ascend) - endif () - set(ACL_LIB_DIR ${ASCEND_PATH}/acllib/) - set(ATLAS_ACL_LIB_DIR ${ASCEND_PATH}/ascend-toolkit/latest/acllib) - MESSAGE("hisi acl lib dir " ${ACL_LIB_DIR} " ,atlas acl lib dir " ${ATLAS_ACL_LIB_DIR}) - - include_directories(${ACL_LIB_DIR}/include/) - include_directories(${ATLAS_ACL_LIB_DIR}/include/) - file(GLOB_RECURSE ACL_SESSION_SRC_LIST RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "acl/*.cc") - list(APPEND SERVING_SRC ${ACL_SESSION_SRC_LIST}) -endif () - -include_directories(${CMAKE_BINARY_DIR}) - -add_executable(ms_serving ${SERVING_SRC}) -#libevent -target_link_libraries(ms_serving mindspore::event mindspore::event_pthreads) - -target_link_libraries(ms_serving ${_REFLECTION} ${_GRPC_GRPCPP} ${_PROTOBUF_LIBPROTOBUF} pthread) - -include(CheckPIESupported) -check_pie_supported() -set_property(TARGET ms_serving PROPERTY POSITION_INDEPENDENT_CODE TRUE) - -if (ENABLE_D) - add_compile_definitions(ENABLE_D) - target_link_libraries(ms_serving ${RUNTIME_LIB}) -endif () - -if (ENABLE_ACL) - add_compile_definitions(ENABLE_ACL) - add_compile_definitions(ENABLE_DVPP_INTERFACE) - find_library(acl libascendcl.so ${ACL_LIB_DIR}/lib64 ${ATLAS_ACL_LIB_DIR}/lib64) - find_library(acl_retr libacl_retr.so ${ACL_LIB_DIR}/lib64 ${ATLAS_ACL_LIB_DIR}/lib64) - find_library(acl_cblas libacl_cblas.so ${ACL_LIB_DIR}/lib64 ${ATLAS_ACL_LIB_DIR}/lib64) - find_library(acl_dvpp libacl_dvpp.so ${ACL_LIB_DIR}/lib64 ${ATLAS_ACL_LIB_DIR}/lib64) - find_library(acl_runtime libruntime.so ${ACL_LIB_DIR}/lib64 ${ATLAS_ACL_LIB_DIR}/lib64) - - target_link_libraries(ms_serving ${acl} ${acl_retr} ${acl_cblas} ${acl_dvpp} ${acl_runtime}) - target_link_libraries(ms_serving jpeg_turbo::jpeg securec) -else () - target_link_libraries(ms_serving inference mindspore_gvar) -endif () diff --git a/serving/README.md b/serving/README.md deleted file mode 100644 index 176cb39bb20..00000000000 --- a/serving/README.md +++ /dev/null @@ -1,150 +0,0 @@ -# MindSpore-based Inference Service Deployment - - - - -- [MindSpore-based Inference Service Deployment](#mindspore-based-inference-service-deployment) - - [Overview](#overview) - - [Starting Serving](#starting-serving) - - [Application Example](#application-example) - - [Exporting Model](#exporting-model) - - [Starting Serving Inference](#starting-serving-inference) - - [Client Samples](#client-samples) - - [Python Client Sample](#python-client-sample) - - [C++ Client Sample](#cpp-client-sample) - - - - - -## Overview - -MindSpore Serving is a lightweight and high-performance service module that helps MindSpore developers efficiently deploy online inference services in the production environment. After completing model training using MindSpore, you can export the MindSpore model and use MindSpore Serving to create an inference service for the model. Currently, only Ascend 910 is supported. - - -## Starting Serving -After MindSpore is installed using `pip`, the Serving executable program is stored in `/{your python path}/lib/python3.7/site-packages/mindspore/ms_serving`. -Run the following command to start Serving: -```bash -ms_serving [--help] [--model_path ] [--model_name ] - [--port ] [--device_id ] -``` -Parameters are described as follows: - -|Parameter|Attribute|Function|Parameter Type|Default Value|Value Range| -|---|---|---|---|---|---| -|`--help`|Optional|Displays the help information about the startup command. |-|-|-| -|`--model_path=`|Mandatory|Path for storing the model to be loaded. |String|Null|-| -|`--model_name=`|Mandatory|Name of the model file to be loaded. |String|Null|-| -|`--=port `|Optional|Specifies the external Serving port number. |Integer|5500|1–65535| -|`--device_id=`|Optional|Specifies device ID to be used.|Integer|0|0 to 7| - - > Before running the startup command, add the path `/{your python path}/lib:/{your python path}/lib/python3.7/site-packages/mindspore/lib` to the environment variable `LD_LIBRARY_PATH`. - -## Application Example -The following uses a simple network as an example to describe how to use MindSpore Serving. - -### Exporting Model -Use [add_model.py](https://gitee.com/mindspore/mindspore/blob/master/serving/example/export_model/add_model.py) to build a network with only the Add operator and export the MindSpore inference deployment model. - -```python -python add_model.py -``` -Execute the script to generate the `tensor_add.mindir` file. The input of the model is two one-dimensional tensors with shape [2,2], and the output is the sum of the two input tensors. - -### Starting Serving Inference -```bash -ms_serving --model_path={model directory} --model_name=tensor_add.mindir -``` -If the server prints the `MS Serving Listening on 0.0.0.0:5500` log, the Serving has loaded the inference model. - -### Client Samples -#### Python Client Sample -Obtain [ms_client.py](https://gitee.com/mindspore/mindspore/blob/master/serving/example/python_client/ms_client.py) and start the Python client. -```bash -python ms_client.py -``` - -If the following information is displayed, the Serving has correctly executed the inference of the Add network. -``` -ms client received: -[[2. 2.] - [2. 2.]] -``` - -#### C++ Client Sample -1. Obtain an executable client sample program. - - Download the [MindSpore source code](https://gitee.com/mindspore/mindspore). You can use either of the following methods to compile and obtain the client sample program: - + When MindSpore is compiled using the source code, the Serving C++ client sample program is generated. You can find the `ms_client` executable program in the `build/mindspore/serving/example/cpp_client` directory. - + Independent compilation - - Preinstall [gRPC](https://gRPC.io). - - Run the following command in the MindSpore source code path to compile a client sample program: - ```bash - cd mindspore/serving/example/cpp_client - mkdir build && cd build - cmake -D GRPC_PATH={grpc_install_dir} .. - make - ``` - In the preceding command, `{grpc_install_dir}` indicates the gRPC installation path. Replace it with the actual gRPC installation path. - -2. Start the client. - - Execute `ms_client` to send an inference request to the Serving. - ```bash - ./ms_client --target=localhost:5500 - ``` - If the following information is displayed, the Serving has correctly executed the inference of the Add network. - ``` - Compute [[1, 2], [3, 4]] + [[1, 2], [3, 4]] - Add result is 2 4 6 8 - client received: RPC OK - ``` - -The client code consists of the following parts: - -1. Implement the client based on MSService::Stub and create a client instance. - ``` - class MSClient { - public: - explicit MSClient(std::shared_ptr channel) : stub_(MSService::NewStub(channel)) {} - private: - std::unique_ptr stub_; - };MSClient client(grpc::CreateChannel(target_str, grpc::InsecureChannelCredentials())); - - MSClient client(grpc::CreateChannel(target_str, grpc::InsecureChannelCredentials())); - - ``` -2. Build the request input parameter `Request`, output parameter `Reply`, and gRPC client `Context` based on the actual network input. - ``` - PredictRequest request; - PredictReply reply; - ClientContext context; - - //construct tensor - Tensor data; - - //set shape - TensorShape shape; - shape.add_dims(4); - *data.mutable_tensor_shape() = shape; - - //set type - data.set_tensor_type(ms_serving::MS_FLOAT32); - std::vector input_data{1, 2, 3, 4}; - - //set datas - data.set_data(input_data.data(), input_data.size()); - - //add tensor to request - *request.add_data() = data; - *request.add_data() = data; - ``` -3. Call the gRPC API to communicate with the Serving that has been started, and obtain the return value. - ``` - Status status = stub_->Predict(&context, request, &reply); - ``` - -For details about the complete code, see [ms_client](https://gitee.com/mindspore/mindspore/blob/master/serving/example/cpp_client/ms_client.cc). diff --git a/serving/README_CN.md b/serving/README_CN.md deleted file mode 100644 index f61b6b1f64a..00000000000 --- a/serving/README_CN.md +++ /dev/null @@ -1,151 +0,0 @@ -# 基于MindSpore部署推理服务 - - - - -- [基于MindSpore部署推理服务](#基于mindspore部署推理服务) - - [概述](#概述) - - [启动Serving服务](#启动serving服务) - - [应用示例](#应用示例) - - [导出模型](#导出模型) - - [启动Serving推理服务](#启动serving推理服务) - - [客户端示例](#客户端示例) - - [Python客户端示例](#python客户端示例) - - [C++客户端示例](#cpp客户端示例) - - - - - -## 概述 - -MindSpore Serving是一个轻量级、高性能的服务模块，旨在帮助MindSpore开发者在生产环境中高效部署在线推理服务。当用户使用MindSpore完成模型训练后，导出MindSpore模型，即可使用MindSpore Serving创建该模型的推理服务。当前Serving仅支持Ascend 910。 - - -## 启动Serving服务 -通过pip安装MindSpore后，Serving可执行程序位于`/{your python path}/lib/python3.7/site-packages/mindspore/ms_serving` 。 -启动Serving服务命令如下 -```bash -ms_serving [--help] [--model_path ] [--model_name ] - [--port ] [--device_id ] -``` -参数含义如下 - -|参数名|属性|功能描述|参数类型|默认值|取值范围| -|---|---|---|---|---|---| -|`--help`|可选|显示启动命令的帮助信息。|-|-|-| -|`--model_path=`|必选|指定待加载模型的存放路径。|String|空|-| -|`--model_name=`|必选|指定待加载模型的文件名。|String|空|-| -|`--port=`|可选|指定Serving对外的端口号。|Integer|5500|1~65535| -|`--device_id=`|可选|指定使用的设备号|Integer|0|0~7| - - > 执行启动命令前，需将`/{your python path}/lib:/{your python path}/lib/python3.7/site-packages/mindspore/lib`对应的路径加入到环境变量LD_LIBRARY_PATH中 。 - -## 应用示例 -下面以一个简单的网络为例，演示MindSpore Serving如何使用。 - -### 导出模型 -使用[add_model.py](https://gitee.com/mindspore/mindspore/blob/master/serving/example/export_model/add_model.py)，构造一个只有Add算子的网络，并导出MindSpore推理部署模型。 - -```python -python add_model.py -``` -执行脚本，生成`tensor_add.mindir`文件，该模型的输入为两个shape为[2,2]的二维Tensor，输出结果是两个输入Tensor之和。 - -### 启动Serving推理服务 -```bash -ms_serving --model_path={model directory} --model_name=tensor_add.mindir -``` -当服务端打印日志`MS Serving Listening on 0.0.0.0:5500`时，表示Serving服务已加载推理模型完毕。 - -### 客户端示例 -#### Python客户端示例 -获取[ms_client.py](https://gitee.com/mindspore/mindspore/blob/master/serving/example/python_client/ms_client.py)，启动Python客户端。 -```bash -python ms_client.py -``` - -显示如下返回值说明Serving服务已正确执行Add网络的推理。 -``` -ms client received: -[[2. 2.] - [2. 2.]] -``` - -#### C++客户端示例 -1. 获取客户端示例执行程序 - - 首先需要下载[MindSpore源码](https://gitee.com/mindspore/mindspore)。有两种方式编译并获取客户端示例程序： - + 从源码编译MindSpore时候，将会编译产生Serving C++客户端示例程序，可在`build/mindspore/serving/example/cpp_client`目录下找到`ms_client`可执行程序。 - + 独立编译： - - 需要先预装[gRPC](https://gRPC.io)。 - - 然后，在MindSpore源码路径中执行如下命令，编译一个客户端示例程序。 - ```bash - cd mindspore/serving/example/cpp_client - mkdir build && cd build - cmake -D GRPC_PATH={grpc_install_dir} .. - make - ``` - 其中`{grpc_install_dir}`为gRPC安装时的路径，请替换为实际gRPC安装路径。 - -2. 启动客户端 - - 执行ms_client，向Serving服务发送推理请求： - ```bash - ./ms_client --target=localhost:5500 - ``` - 显示如下返回值说明Serving服务已正确执行Add网络的推理。 - ``` - Compute [[1, 2], [3, 4]] + [[1, 2], [3, 4]] - Add result is 2 4 6 8 - client received: RPC OK - ``` - -客户端代码主要包含以下几个部分： - -1. 基于MSService::Stub实现Client，并创建Client实例。 - ``` - class MSClient { - public: - explicit MSClient(std::shared_ptr channel) : stub_(MSService::NewStub(channel)) {} - private: - std::unique_ptr stub_; - };MSClient client(grpc::CreateChannel(target_str, grpc::InsecureChannelCredentials())); - - MSClient client(grpc::CreateChannel(target_str, grpc::InsecureChannelCredentials())); - - ``` -2. 根据网络的实际输入构造请求的入参Request、出参Reply和gRPC的客户端Context。 - ``` - PredictRequest request; - PredictReply reply; - ClientContext context; - - //construct tensor - Tensor data; - - //set shape - TensorShape shape; - shape.add_dims(4); - *data.mutable_tensor_shape() = shape; - - //set type - data.set_tensor_type(ms_serving::MS_FLOAT32); - std::vector input_data{1, 2, 3, 4}; - - //set datas - data.set_data(input_data.data(), input_data.size()); - - //add tensor to request - *request.add_data() = data; - *request.add_data() = data; - ``` -3. 调用gRPC接口和已经启动的Serving服务通信，并取回返回值。 - ``` - Status status = stub_->Predict(&context, request, &reply); - ``` - -完整代码参考[ms_client](https://gitee.com/mindspore/mindspore/blob/master/serving/example/cpp_client/ms_client.cc)。 - diff --git a/serving/acl/acl_session.cc b/serving/acl/acl_session.cc deleted file mode 100644 index 92444924eb1..00000000000 --- a/serving/acl/acl_session.cc +++ /dev/null @@ -1,243 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include "serving/acl/acl_session.h" -#include "include/infer_log.h" - -namespace mindspore::inference { - -std::shared_ptr InferSession::CreateSession(const std::string &device, uint32_t device_id) { - try { - auto session = std::make_shared(); - auto ret = session->InitEnv(device, device_id); - if (ret != SUCCESS) { - return nullptr; - } - return session; - } catch (std::exception &e) { - MSI_LOG_ERROR << "Inference CreatSession failed"; - return nullptr; - } -} - -Status AclSession::LoadModelFromFile(const std::string &file_name, uint32_t &model_id) { - Status ret = model_process_.LoadModelFromFile(file_name, model_id); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "Load model from file failed, model file " << file_name; - return FAILED; - } - std::string dvpp_config_file; - auto index = file_name.rfind("."); - if (index == std::string::npos) { - dvpp_config_file = file_name; - } else { - dvpp_config_file = file_name.substr(0, index); - } - dvpp_config_file += "_dvpp_config.json"; - std::ifstream fp(dvpp_config_file); - if (!fp.is_open()) { - MSI_LOG_INFO << "Dvpp config file not exist, model will execute with tensors as inputs, dvpp config file " - << dvpp_config_file; - return SUCCESS; - } - fp.close(); - if (dvpp_process_.InitWithJsonConfig(dvpp_config_file) != SUCCESS) { - MSI_LOG_ERROR << "Dvpp config file parse error, dvpp config file " << dvpp_config_file; - return FAILED; - } - execute_with_dvpp_ = true; - MSI_LOG_INFO << "Dvpp config success"; - return SUCCESS; -} - -Status AclSession::UnloadModel(uint32_t /*model_id*/) { - model_process_.UnLoad(); - return SUCCESS; -} - -Status AclSession::ExecuteModel(uint32_t /*model_id*/, const RequestBase &request, - ReplyBase &reply) { // set d context - aclError rt_ret = aclrtSetCurrentContext(context_); - if (rt_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "set the ascend device context failed"; - return FAILED; - } - return model_process_.Execute(request, reply); -} - -Status AclSession::PreProcess(uint32_t /*model_id*/, const InferImagesBase *images_input, - ImagesDvppOutput &dvpp_output) { - if (images_input == nullptr) { - MSI_LOG_ERROR << "images input is nullptr"; - return FAILED; - } - auto batch_size = images_input->batch_size(); - if (batch_size <= 0) { - MSI_LOG_ERROR << "invalid batch size " << images_input->batch_size(); - return FAILED; - } - std::vector pic_buffer_list; - std::vector pic_size_list; - for (size_t i = 0; i < batch_size; i++) { - const void *pic_buffer = nullptr; - uint32_t pic_size = 0; - if (!images_input->get(i, pic_buffer, pic_size) || pic_buffer == nullptr || pic_size == 0) { - MSI_LOG_ERROR << "Get request " << 0 << "th buffer failed"; - return FAILED; - } - pic_buffer_list.push_back(pic_buffer); - pic_size_list.push_back(pic_size); - } - auto ret = dvpp_process_.Process(pic_buffer_list, pic_size_list, dvpp_output.buffer_device, dvpp_output.buffer_size); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "dvpp process failed"; - return ret; - } - return SUCCESS; -} - -Status AclSession::ExecuteModel(uint32_t model_id, const ImagesRequestBase &images_inputs, // images for preprocess - const RequestBase &request, ReplyBase &reply) { - if (!execute_with_dvpp_) { - MSI_LOG_ERROR << "Unexpected images as inputs, DVPP not config"; - return INFER_STATUS(INVALID_INPUTS) << "Unexpected images as inputs, DVPP not config"; - } - aclError rt_ret = aclrtSetCurrentContext(context_); - if (rt_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "set the ascend device context failed"; - return FAILED; - } - if (images_inputs.size() != 1) { - MSI_LOG_ERROR << "Only support one input to do DVPP preprocess"; - return INFER_STATUS(INVALID_INPUTS) << "Only support one input to do DVPP preprocess"; - } - if (images_inputs[0] == nullptr) { - MSI_LOG_ERROR << "Get first images input failed"; - return FAILED; - } - if (images_inputs[0]->batch_size() != model_process_.GetBatchSize()) { - MSI_LOG_ERROR << "Input batch size " << images_inputs[0]->batch_size() << " not match Model batch size " - << model_process_.GetBatchSize(); - return INFER_STATUS(INVALID_INPUTS) << "Input batch size " << images_inputs[0]->batch_size() - << " not match Model batch size " << model_process_.GetBatchSize(); - } - if (request.size() != 0) { - MSI_LOG_ERROR << "only support one input, images input size is 1, tensor inputs is not 0 " << request.size(); - return INFER_STATUS(INVALID_INPUTS) << "only support one input, images input size is 1, tensor inputs is not 0 " - << request.size(); - } - ImagesDvppOutput dvpp_output; - Status ret = PreProcess(model_id, images_inputs[0], dvpp_output); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "DVPP preprocess failed"; - return ret; - } - ret = model_process_.Execute(dvpp_output.buffer_device, dvpp_output.buffer_size, reply); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "Execute model failed"; - return ret; - } - return SUCCESS; -} - -Status AclSession::InitEnv(const std::string &device_type, uint32_t device_id) { - device_type_ = device_type; - device_id_ = device_id; - auto ret = aclInit(nullptr); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Execute aclInit Failed"; - return FAILED; - } - MSI_LOG_INFO << "acl init success"; - - ret = aclrtSetDevice(device_id_); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "acl open device " << device_id_ << " failed"; - return FAILED; - } - MSI_LOG_INFO << "open device " << device_id_ << " success"; - - ret = aclrtCreateContext(&context_, device_id_); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "acl create context failed"; - return FAILED; - } - MSI_LOG_INFO << "create context success"; - - ret = aclrtCreateStream(&stream_); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "acl create stream failed"; - return FAILED; - } - MSI_LOG_INFO << "create stream success"; - - aclrtRunMode run_mode; - ret = aclrtGetRunMode(&run_mode); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "acl get run mode failed"; - return FAILED; - } - bool is_device = (run_mode == ACL_DEVICE); - model_process_.SetIsDevice(is_device); - MSI_LOG_INFO << "get run mode success is device input/output " << is_device; - - if (dvpp_process_.InitResource(stream_) != SUCCESS) { - MSI_LOG_ERROR << "dvpp init resource failed"; - return FAILED; - } - MSI_LOG_INFO << "Init acl success, device id " << device_id_; - return SUCCESS; -} - -Status AclSession::FinalizeEnv() { - dvpp_process_.Finalize(); - aclError ret; - if (stream_ != nullptr) { - ret = aclrtDestroyStream(stream_); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "destroy stream failed"; - } - stream_ = nullptr; - } - MSI_LOG_INFO << "end to destroy stream"; - if (context_ != nullptr) { - ret = aclrtDestroyContext(context_); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "destroy context failed"; - } - context_ = nullptr; - } - MSI_LOG_INFO << "end to destroy context"; - - ret = aclrtResetDevice(device_id_); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "reset devie " << device_id_ << " failed"; - } - MSI_LOG_INFO << "end to reset device " << device_id_; - - ret = aclFinalize(); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "finalize acl failed"; - } - MSI_LOG_INFO << "end to finalize acl"; - return SUCCESS; -} - -AclSession::AclSession() = default; -} // namespace mindspore::inference diff --git a/serving/acl/acl_session.h b/serving/acl/acl_session.h deleted file mode 100644 index b60b9a60189..00000000000 --- a/serving/acl/acl_session.h +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MINDSPORE_SERVING_ACL_SESSION_H -#define MINDSPORE_SERVING_ACL_SESSION_H - -#include -#include -#include -#include -#include -#include - -#include "include/inference.h" -#include "serving/acl/model_process.h" -#include "serving/acl/dvpp_process.h" - -namespace mindspore { -namespace inference { -class AclSession : public InferSession { - public: - AclSession(); - - Status InitEnv(const std::string &device_type, uint32_t device_id) override; - Status FinalizeEnv() override; - Status LoadModelFromFile(const std::string &file_name, uint32_t &model_id) override; - Status UnloadModel(uint32_t model_id) override; - Status ExecuteModel(uint32_t model_id, const RequestBase &request, ReplyBase &reply) override; - Status ExecuteModel(uint32_t model_id, const ImagesRequestBase &images_inputs, // images for preprocess - const RequestBase &request, ReplyBase &reply) override; - - private: - std::string device_type_; - int32_t device_id_; - aclrtStream stream_ = nullptr; - aclrtContext context_ = nullptr; - ModelProcess model_process_; - bool execute_with_dvpp_ = false; - DvppProcess dvpp_process_; - - Status PreProcess(uint32_t model_id, const InferImagesBase *images_input, ImagesDvppOutput &dvpp_output); -}; -} // namespace inference -} // namespace mindspore -#endif // MINDSPORE_SERVING_ACL_SESSION_H diff --git a/serving/acl/dvpp_process.cc b/serving/acl/dvpp_process.cc deleted file mode 100644 index f97eecaa1c0..00000000000 --- a/serving/acl/dvpp_process.cc +++ /dev/null @@ -1,1136 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "serving/acl/dvpp_process.h" -#include -#include -#include -#include -#include "include/infer_log.h" - -namespace mindspore { -namespace inference { -DvppProcess::DvppProcess() {} - -DvppProcess::~DvppProcess() {} - -static uint32_t ToEven(uint32_t num) { return (num + 1) / 2 * 2; } -static uint32_t ToOdd(uint32_t num) { - if (num == 0) { - return 1; - } - return (num + 1) / 2 * 2 - 1; -} - -class DvppJsonConfigParser { - public: - DvppJsonConfigParser() = default; - ~DvppJsonConfigParser() = default; - - Status InitWithJsonConfig(const std::string &json_config); - DvppDecodePara GetDecodePara() const { return decode_para_; } - DvppResizePara GetResizePara() const { return resize_para_; } - DvppCropPara GetCropPara() const { return crop_para_; } - DvppCropAndPastePara GetCropAndPastePara() const { return crop_and_paste_para_; } - bool HasResizeConfig() const { return resize_flag_; } - bool HasCropConfig() const { return crop_flag_; } - bool HasCropAndPasteConfig() const { return crop_and_paste_flag_; } - - private: - DvppDecodePara decode_para_; - DvppResizePara resize_para_; - DvppCropPara crop_para_; - DvppCropAndPastePara crop_and_paste_para_; - bool resize_flag_ = false; - bool crop_flag_ = false; - bool crop_and_paste_flag_ = false; - - Status GetStringValue(const nlohmann::json &json_item, const std::string &key, std::string &val); - Status GetIntValue(const nlohmann::json &json_item, const std::string &key, uint32_t &val); - Status ParseInputPara(const nlohmann::json &preprocess_item); - Status ParseDecodePara(const nlohmann::json &preprocess_item); - Status ParseResizePara(const nlohmann::json &json_item); - Status ParseCropPara(const nlohmann::json &json_item); - Status ParseCropAndPastePara(const nlohmann::json &json_item); - Status InitWithJsonConfigImp(const std::string &json_config); -}; - -Status DvppProcess::InitResource(aclrtStream stream) { - stream_ = stream; - aclError acl_ret; - dvpp_channel_desc_ = acldvppCreateChannelDesc(); - if (dvpp_channel_desc_ == nullptr) { - MSI_LOG_ERROR << "acldvppCreateChannelDesc failed"; - return FAILED; - } - acl_ret = acldvppCreateChannel(dvpp_channel_desc_); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "acldvppCreateChannel failed, acl return " << acl_ret; - return FAILED; - } - MSI_LOG_INFO << "End init dvpp process resource"; - return SUCCESS; -} - -void DvppProcess::DestroyResource() { - if (dvpp_channel_desc_ != nullptr) { - auto acl_ret = acldvppDestroyChannel(dvpp_channel_desc_); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "acldvppDestroyChannel failed, acl return " << acl_ret; - } - acl_ret = acldvppDestroyChannelDesc(dvpp_channel_desc_); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "acldvppDestroyChannelDesc failed, acl return " << acl_ret; - } - dvpp_channel_desc_ = nullptr; - } -} - -void DvppProcess::Finalize() { - DestroyDecodeDesc(); - DestroyVpcOutputDesc(); - DestroyResource(); - if (resize_config_ != nullptr) { - acldvppDestroyResizeConfig(resize_config_); - resize_config_ = nullptr; - } - if (crop_area_ != nullptr) { - acldvppDestroyRoiConfig(crop_area_); - crop_area_ = nullptr; - } - if (paste_area_ != nullptr) { - acldvppDestroyRoiConfig(paste_area_); - paste_area_ = nullptr; - } - if (input_pic_dev_buffer_ != nullptr) { - acldvppFree(input_pic_dev_buffer_); - } - input_pic_buffer_size_ = 0; - MSI_LOG_INFO << "End dvpp process finalize"; -} - -Status DvppProcess::InitJpegDecodePara(const DvppDecodePara &decode_para) { - decode_para_ = decode_para; - MSI_LOG_INFO << "Init decode para, pixel_format " << decode_para_.pixel_format; - return SUCCESS; -} - -Status DvppProcess::InitResizePara(const DvppResizePara &resize_para) { - resize_para_ = resize_para; - MSI_LOG_INFO << "Init resize para, " - << "output_width " << resize_para_.output_width << ", output_height " << resize_para_.output_height; - to_resize_flag_ = true; - to_crop_flag_ = false; - to_crop_and_paste_flag_ = false; - Status ret = InitResizeOutputDesc(); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "InitResizeOutputDesc failed"; - } - return ret; -} - -Status DvppProcess::InitCommonCropPara(DvppCropInfo &crop_info, uint32_t output_width, uint32_t output_height) { - if (crop_info.crop_type == kDvppCropTypeOffset) { - if (CheckAndAdjustRoiArea(crop_info.crop_area) != SUCCESS) { - MSI_LOG_ERROR << "Check and adjust crop area failed"; - return FAILED; - } - MSI_LOG_INFO << "Init common crop para, crop type offset " - << ", left " << crop_info.crop_area.left << ", right " << crop_info.crop_area.right << ", top " - << crop_info.crop_area.top << ", bottom " << crop_info.crop_area.bottom << ", output_width " - << output_width << ", output_height " << output_height; - } else { - crop_info.crop_width = ToEven(crop_info.crop_width); - crop_info.crop_height = ToEven(crop_info.crop_height); - if (CheckRoiAreaWidthHeight(crop_info.crop_width, crop_info.crop_height) != SUCCESS) { - MSI_LOG_ERROR << "Check crop area width and height failed, actually width " << crop_info.crop_width << " height " - << crop_info.crop_height; - return FAILED; - } - MSI_LOG_INFO << "Init common crop para, crop type centre " - << ", crop_width " << crop_info.crop_width << ", crop_height " << crop_info.crop_height - << ", output_width " << output_width << ", output_height " << output_height; - } - return SUCCESS; -} - -Status DvppProcess::InitCropPara(const DvppCropPara &crop_para) { - crop_para_ = crop_para; - if (InitCommonCropPara(crop_para_.crop_info, crop_para_.output_width, crop_para_.output_height) != SUCCESS) { - MSI_LOG_ERROR << "Init common crop para failed in InitCropPara"; - return FAILED; - } - to_crop_flag_ = true; - to_resize_flag_ = false; - to_crop_and_paste_flag_ = false; - Status ret = InitCropOutputDesc(); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "InitCropOutputDesc failed"; - } - return ret; -} - -Status DvppProcess::InitCropAndPastePara(const DvppCropAndPastePara &crop_and_paste_para) { - crop_and_paste_para_ = crop_and_paste_para; - if (InitCommonCropPara(crop_and_paste_para_.crop_info, crop_and_paste_para_.output_width, - crop_and_paste_para_.output_height) != SUCCESS) { - MSI_LOG_ERROR << "Init common crop para failed in InitCropAndPastePara"; - return FAILED; - } - auto &paste_area = crop_and_paste_para_.paste_area; - if (CheckAndAdjustRoiArea(paste_area) != SUCCESS) { - MSI_LOG_ERROR << "Check and adjust paste area failed"; - return FAILED; - } - MSI_LOG_INFO << "Init crop and paste para, paste info: " - << ", left " << paste_area.left << ", right " << paste_area.right << ", top " << paste_area.top - << ", bottom " << paste_area.bottom; - - to_crop_and_paste_flag_ = true; - to_crop_flag_ = false; - to_resize_flag_ = false; - Status ret = InitCropAndPasteOutputDesc(); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "InitCropAndPasteOutputDesc failed"; - } - return ret; -} - -Status DvppProcess::InputInputBuffer(const void *pic_buffer, size_t pic_buffer_size) { - aclError acl_ret; - if (pic_buffer_size != input_pic_buffer_size_) { - acldvppFree(input_pic_dev_buffer_); - input_pic_buffer_size_ = 0; - acl_ret = acldvppMalloc(&input_pic_dev_buffer_, pic_buffer_size); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "acldvppMalloc input picture buffer on device failed, buffer size " << pic_buffer_size; - return FAILED; - } - input_pic_buffer_size_ = pic_buffer_size; - } - acl_ret = - aclrtMemcpy(input_pic_dev_buffer_, input_pic_buffer_size_, pic_buffer, pic_buffer_size, ACL_MEMCPY_HOST_TO_DEVICE); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "aclrtMemcpy input picture buffer to device, buffer size " << pic_buffer_size; - return FAILED; - } - return SUCCESS; -} - -static void JpegErrorExitCustom(j_common_ptr cinfo) { - char jpeg_last_error_msg[JMSG_LENGTH_MAX] = {0}; - if (cinfo != nullptr && cinfo->err != nullptr && cinfo->err->format_message != nullptr) { - (*(cinfo->err->format_message))(cinfo, jpeg_last_error_msg); - } - throw std::runtime_error(jpeg_last_error_msg); -} - -Status DvppProcess::GetJpegWidthHeight(const void *pic_buffer, size_t pic_buffer_size, uint32_t &image_width, - uint32_t &image_height) { - struct jpeg_decompress_struct jpeg_header; - struct jpeg_error_mgr jpeg_error; - jpeg_header.err = jpeg_std_error(&jpeg_error); - jpeg_error.error_exit = JpegErrorExitCustom; - try { - jpeg_create_decompress(&jpeg_header); - jpeg_mem_src(&jpeg_header, reinterpret_cast(pic_buffer), pic_buffer_size); - (void)jpeg_read_header(&jpeg_header, TRUE); - } catch (std::runtime_error &e) { - jpeg_destroy_decompress(&jpeg_header); - MSI_LOG_ERROR << "jpeg images read failed, " << e.what(); - return INFER_STATUS(INVALID_INPUTS) << "jpeg images decode failed"; - } - image_width = jpeg_header.image_width; - image_height = jpeg_header.image_height; - - if (jpeg_header.jpeg_color_space != JCS_YCbCr) { - MSI_LOG_ERROR << "Expect color space YUV(YCbCr), current " << jpeg_header.jpeg_color_space; - jpeg_destroy_decompress(&jpeg_header); - return INFER_STATUS(INVALID_INPUTS) << "Expect color space YUV(YCbCr), current " << jpeg_header.jpeg_color_space; - } - if (jpeg_header.dc_huff_tbl_ptrs[0] == nullptr) { - MSI_LOG_ERROR << "Only support Huffman code"; - jpeg_destroy_decompress(&jpeg_header); - return INFER_STATUS(INVALID_INPUTS) << "Only support Huffman code"; - } - jpeg_destroy_decompress(&jpeg_header); - - const uint32_t min_width = 32; - const uint32_t max_width = 8192; - const uint32_t min_height = 32; - const uint32_t max_height = 8192; - if (image_width < min_width || image_width > max_width) { - MSI_LOG_ERROR << "expect image width [" << min_width << ", " << max_width << "], the real image width is " - << image_width; - return INFER_STATUS(INVALID_INPUTS) << "expect image width [" << min_width << ", " << max_width - << "], the real image width is " << image_width; - } - if (image_height < min_height || image_height > max_height) { - MSI_LOG_ERROR << "expect image height [" << min_height << ", " << max_height << "], the real image height is " - << image_height; - return INFER_STATUS(INVALID_INPUTS) << "expect image height [" << min_height << ", " << max_height - << "], the real image height is " << image_height; - } - return SUCCESS; -} - -Status DvppProcess::Process(const void *pic_buffer, size_t pic_buffer_size, void *&output_device_buffer, - size_t &output_size) { - if (dvpp_channel_desc_ == nullptr) { - MSI_LOG_ERROR << "Process failed, dvpp not inited"; - return FAILED; - } - uint32_t image_width = 0; - uint32_t image_height = 0; - Status ret = GetJpegWidthHeight(pic_buffer, pic_buffer_size, image_width, image_height); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "Get jpeg image height and width failed"; - return ret; - } - MSI_LOG_INFO << "Get jpeg width " << image_width << ", height " << image_height; - ret = InitDecodeOutputDesc(image_width, image_height); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "InitDecodeOutputDesc failed"; - return FAILED; - } - ret = UpdateCropArea(image_width, image_height); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "Update crop area failed"; - return ret; - } - ret = CheckResizeImageInfo(image_width, image_height); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "Check resize para failed"; - return ret; - } - if (InputInputBuffer(pic_buffer, pic_buffer_size) != SUCCESS) { - MSI_LOG_ERROR << "InputInputBuffer failed"; - return FAILED; - } - if (ProcessDecode() != SUCCESS) { - MSI_LOG_ERROR << "Process Decode failed"; - return INFER_STATUS(INVALID_INPUTS) << "Decode image failed"; - } - MSI_LOG_INFO << "Process Decode success"; - if (to_resize_flag_) { - if (ProcessResize() != SUCCESS) { - MSI_LOG_ERROR << "Process Resize failed"; - return INFER_STATUS(FAILED) << "Resize image failed"; - } - MSI_LOG_INFO << "Process Resize success"; - } else if (to_crop_flag_) { - if (ProcessCrop() != SUCCESS) { - MSI_LOG_ERROR << "Process Crop failed"; - return INFER_STATUS(FAILED) << "Crop image failed"; - } - MSI_LOG_INFO << "Process Crop success"; - } else if (to_crop_and_paste_flag_) { - if (ProcessCropAndPaste() != SUCCESS) { - MSI_LOG_ERROR << "Process Crop And Paste failed"; - return INFER_STATUS(FAILED) << "Crop And Paste image failed"; - } - MSI_LOG_INFO << "Process Crop And Paste success"; - } - if (vpc_output_buffer_dev_ == nullptr) { - output_device_buffer = decode_output_buffer_dev_; - output_size = decode_output_buffer_size_; - } else { - output_device_buffer = vpc_output_buffer_dev_; - output_size = vpc_output_buffer_size_; - } - MSI_LOG_INFO << "Process dvpp success"; - return SUCCESS; -} - -Status DvppProcess::Process(const std::vector &pic_buffer_list, - const std::vector &pic_buffer_size_list, void *&output_device_buffer, - size_t &output_size) { - auto batch_size = pic_buffer_list.size(); - if (batch_size == 0 || batch_size != pic_buffer_size_list.size()) { - MSI_LOG_ERROR << "invalid batch size " << batch_size << ", pic size count" << pic_buffer_size_list.size(); - return FAILED; - } - MSI_LOG_INFO << "Begin dvpp process, batch size " << batch_size; - if (batch_size == 1) { - return Process(pic_buffer_list[0], pic_buffer_size_list[0], output_device_buffer, output_size); - } - size_t total_buffer_size = vpc_output_buffer_size_ * batch_size; - if (batch_size_ != batch_size) { - if (batch_vpc_output_buffer_dev_ != nullptr) { - acldvppFree(batch_vpc_output_buffer_dev_); - batch_vpc_output_buffer_dev_ = nullptr; - } - batch_size_ = batch_size; - auto acl_rt = acldvppMalloc(&batch_vpc_output_buffer_dev_, total_buffer_size); - if (acl_rt != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "acldvppMalloc failed, buffer size " << total_buffer_size; - return FAILED; - } - } - for (size_t i = 0; i < batch_size; i++) { - const void *pic_buffer = pic_buffer_list[i]; - uint32_t pic_size = pic_buffer_size_list[i]; - if (pic_buffer == nullptr || pic_size == 0) { - MSI_LOG_ERROR << "Get " << 0 << "th images failed"; - return FAILED; - } - void *output_dev_buffer_tmp = nullptr; - size_t output_buffer_size_tmp = 0; - Status ret = Process(pic_buffer, pic_size, output_dev_buffer_tmp, output_buffer_size_tmp); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "dvpp process failed"; - return ret; - } - aclrtMemcpy(static_cast(batch_vpc_output_buffer_dev_) + vpc_output_buffer_size_ * i, - total_buffer_size - vpc_output_buffer_size_ * i, output_dev_buffer_tmp, vpc_output_buffer_size_, - ACL_MEMCPY_DEVICE_TO_DEVICE); - - MSI_LOG_INFO << "Dvpp process " << i << " th images success, input pic size " << pic_size << " output buffer size " - << output_buffer_size_tmp; - } - output_device_buffer = batch_vpc_output_buffer_dev_; - output_size = total_buffer_size; - MSI_LOG_INFO << "End dvpp process, batch size " << batch_size << ", output size " << output_size; - return SUCCESS; -} - -uint32_t DvppProcess::AlignmentHelper(uint32_t org_size, uint32_t alignment) const { - if (alignment == 0) { - return 0; - } - return (org_size + alignment - 1) / alignment * alignment; -} - -uint32_t DvppProcess::GetImageBufferSize(uint32_t stride_width, uint32_t stride_height, - acldvppPixelFormat pixel_format) const { - if (stride_height == 0 || stride_width == 0) { - MSI_LOG_ERROR << "invalid stride height or width, stride_width " << stride_width << " stride_height " - << stride_height; - return 0; - } - if (UINT32_MAX / 3 < stride_height || UINT32_MAX / (3 * stride_height) < stride_width) { - MSI_LOG_ERROR << "invalid stride height or width, stride_width " << stride_width << " stride_height " - << stride_height; - return 0; - } - if (pixel_format == PIXEL_FORMAT_YUV_SEMIPLANAR_420 || pixel_format == PIXEL_FORMAT_YVU_SEMIPLANAR_420) { - return stride_width * stride_height * 3 / 2; // 420 - } else if (pixel_format == PIXEL_FORMAT_YUV_SEMIPLANAR_422 || pixel_format == PIXEL_FORMAT_YVU_SEMIPLANAR_422) { - return stride_width * stride_height * 2; // 422 - } else if (pixel_format == PIXEL_FORMAT_YUV_SEMIPLANAR_444 || pixel_format == PIXEL_FORMAT_YVU_SEMIPLANAR_444) { - return stride_width * stride_height * 3; // 444 - } - MSI_LOG_ERROR << "Not support pixel format " << pixel_format; - return 0; -} - -Status DvppProcess::GetPicDescStride(uint32_t width, uint32_t height, uint32_t &stride_width, uint32_t &stride_height) { - const uint32_t width_alignment = 16; - const uint32_t height_alignment = 2; - const uint32_t stride_width_minimum = 32; - const uint32_t stride_width_maximum = 4096; - const uint32_t stride_height_minimum = 6; - const uint32_t stride_height_maximum = 4096; - - stride_width = AlignmentHelper(width, width_alignment); - stride_height = AlignmentHelper(height, height_alignment); - if (stride_width == 0 || stride_height == 0) { - MSI_LOG_ERROR << "Init VPC output desc failed, get stride width or height failed"; - return FAILED; - } - if (stride_width < stride_width_minimum || stride_width > stride_width_maximum) { - MSI_LOG_ERROR << "Expect stride width [" << stride_width_minimum << ", " << stride_width_maximum - << "], current stride width " << stride_width << " given width " << width; - return FAILED; - } - if (stride_height < stride_height_minimum || stride_height > stride_height_maximum) { - MSI_LOG_ERROR << "Expect stride height [" << stride_height_minimum << ", " << stride_height_maximum - << "], current stride height " << stride_height << " given height " << height; - return FAILED; - } - return SUCCESS; -} - -Status DvppProcess::GetPicDescStrideDecode(uint32_t width, uint32_t height, uint32_t &stride_width, - uint32_t &stride_height) { - const uint32_t width_alignment = 128; - const uint32_t height_alignment = 16; - const uint32_t width_minimum = 32; - const uint32_t width_maximum = 4096; // decode support 8192, dvpp(resize/crop/crop&paste) support 4096 - const uint32_t height_minimum = 32; - const uint32_t height_maximum = 4096; // decode support 8192, dvpp(resize/crop/crop&paste) support 4096 - if (width < width_minimum || width > width_maximum) { - MSI_LOG_ERROR << "Expect width [" << width_minimum << ", " << width_maximum << "], current width " << width; - return INFER_STATUS(INVALID_INPUTS) << "Expect width [" << width_minimum << ", " << width_maximum - << "], current width " << width; - } - if (height < height_minimum || height > height_maximum) { - MSI_LOG_ERROR << "Expect height [" << height_minimum << ", " << height_maximum << "], current height " << height; - return INFER_STATUS(INVALID_INPUTS) << "Expect height [" << height_minimum << ", " << height_maximum - << "], current height " << height; - } - stride_width = AlignmentHelper(width, width_alignment); - stride_height = AlignmentHelper(height, height_alignment); - if (stride_width == 0 || stride_height == 0) { - MSI_LOG_ERROR << "Init decode output desc failed, get stride width or height failed"; - return FAILED; - } - return SUCCESS; -} - -Status DvppProcess::InitVpcOutputDesc(uint32_t output_width, uint32_t output_height, acldvppPixelFormat pixel_format) { - DestroyVpcOutputDesc(); - uint32_t vpc_stride_width = 0; - uint32_t vpc_stride_height = 0; - if (GetPicDescStride(output_width, output_height, vpc_stride_width, vpc_stride_height) != SUCCESS) { - MSI_LOG_ERROR << "Init VPC output desc failed, get VPC output stride width/height failed"; - return FAILED; - } - vpc_output_buffer_size_ = GetImageBufferSize(vpc_stride_width, vpc_stride_height, pixel_format); - if (vpc_output_buffer_size_ == 0) { - MSI_LOG_ERROR << "Init VPC output desc failed, get image buffer size failed"; - return FAILED; - } - auto acl_ret = acldvppMalloc(&vpc_output_buffer_dev_, vpc_output_buffer_size_); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Init VPC output desc failed, malloc dvpp memory failed"; - return FAILED; - } - vpc_output_desc_ = acldvppCreatePicDesc(); - if (vpc_output_desc_ == nullptr) { - MSI_LOG_ERROR << "Init VPC output desc failed, create pic desc failed"; - return FAILED; - } - acldvppSetPicDescData(vpc_output_desc_, vpc_output_buffer_dev_); - acldvppSetPicDescSize(vpc_output_desc_, vpc_output_buffer_size_); - acldvppSetPicDescFormat(vpc_output_desc_, pixel_format); - acldvppSetPicDescWidth(vpc_output_desc_, output_width); - acldvppSetPicDescHeight(vpc_output_desc_, output_height); - acldvppSetPicDescWidthStride(vpc_output_desc_, vpc_stride_width); - acldvppSetPicDescHeightStride(vpc_output_desc_, vpc_stride_height); - MSI_LOG_INFO << "Init VPC output desc success"; - return SUCCESS; -} - -void DvppProcess::DestroyVpcOutputDesc() { - if (vpc_output_desc_ != nullptr) { - acldvppDestroyPicDesc(vpc_output_desc_); - vpc_output_desc_ = nullptr; - } - if (vpc_output_buffer_dev_ != nullptr) { - acldvppFree(vpc_output_buffer_dev_); - vpc_output_buffer_dev_ = nullptr; - } - if (batch_vpc_output_buffer_dev_ != nullptr) { - acldvppFree(batch_vpc_output_buffer_dev_); - batch_vpc_output_buffer_dev_ = nullptr; - } - vpc_output_buffer_size_ = 0; - MSI_LOG_INFO << "End destroy vpc desc"; -} - -Status DvppProcess::InitDecodeOutputDesc(uint32_t image_width, uint32_t image_height) { - if (decode_output_buffer_dev_ != nullptr && image_width == pic_width_ && image_height == pic_height_) { - return SUCCESS; - } - DestroyDecodeDesc(); - - pic_width_ = image_width; - pic_height_ = image_height; - - uint32_t stride_width = 0; - uint32_t stride_height = 0; - Status ret = GetPicDescStrideDecode(pic_width_, pic_height_, stride_width, stride_height); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "Init VPC output desc failed, get VPC output stride width/height failed"; - return ret; - } - - decode_output_buffer_size_ = GetImageBufferSize(stride_width, stride_height, decode_para_.pixel_format); - if (decode_output_buffer_size_ == 0) { - MSI_LOG_ERROR << "Init decode output desc failed, get image buffer size failed"; - return FAILED; - } - auto acl_ret = acldvppMalloc(&decode_output_buffer_dev_, decode_output_buffer_size_); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Init decode output desc failed, malloc dvpp memory failed"; - return FAILED; - } - decode_output_desc_ = acldvppCreatePicDesc(); - if (decode_output_desc_ == nullptr) { - MSI_LOG_ERROR << "Init decode output desc failed, create pic desc failed"; - return FAILED; - } - acldvppSetPicDescData(decode_output_desc_, decode_output_buffer_dev_); - acldvppSetPicDescSize(decode_output_desc_, decode_output_buffer_size_); - acldvppSetPicDescFormat(decode_output_desc_, decode_para_.pixel_format); - acldvppSetPicDescWidth(decode_output_desc_, pic_width_); - acldvppSetPicDescHeight(decode_output_desc_, pic_height_); - acldvppSetPicDescWidthStride(decode_output_desc_, stride_width); - acldvppSetPicDescHeightStride(decode_output_desc_, stride_height); - MSI_LOG_INFO << "Init decode output desc success"; - return SUCCESS; -} - -Status DvppProcess::CheckRoiAreaWidthHeight(uint32_t width, uint32_t height) { - const uint32_t min_crop_width = 10; - const uint32_t max_crop_width = 4096; - const uint32_t min_crop_height = 6; - const uint32_t max_crop_height = 4096; - - if (width < min_crop_width || width > max_crop_width) { - MSI_LOG_ERROR << "Expect roi area width in [" << min_crop_width << ", " << max_crop_width << "], actually " - << width; - return FAILED; - } - if (height < min_crop_height || height > max_crop_height) { - MSI_LOG_ERROR << "Expect roi area height in [" << min_crop_height << ", " << max_crop_height << "], actually " - << height; - return FAILED; - } - return SUCCESS; -} - -Status DvppProcess::CheckAndAdjustRoiArea(DvppRoiArea &area) { - if (area.right < area.left) { - MSI_LOG_ERROR << "check roi area failed, left " << area.left << ", right " << area.right; - return FAILED; - } - if (area.bottom < area.top) { - MSI_LOG_ERROR << "check roi area failed, top " << area.top << ", bottom " << area.bottom; - return FAILED; - } - - area.left = ToEven(area.left); - area.top = ToEven(area.top); - area.right = ToOdd(area.right); - area.bottom = ToOdd(area.bottom); - - auto width = area.right - area.left + 1; - auto height = area.bottom - area.top + 1; - if (CheckRoiAreaWidthHeight(width, height) != SUCCESS) { - MSI_LOG_ERROR << "Check roi area width and height failed," - << " actually width " << width << " left " << area.left << ", right " << area.right - << " actually height " << height << " top " << area.top << ", bottom " << area.bottom; - return FAILED; - } - return SUCCESS; -} - -Status DvppProcess::UpdateCropArea(uint32_t image_width, uint32_t image_height) { - DvppCropInfo *crop_info = nullptr; - if (to_crop_flag_) { - crop_info = &crop_para_.crop_info; - } else if (to_crop_and_paste_flag_) { - crop_info = &crop_and_paste_para_.crop_info; - } else { - return SUCCESS; - } - if (crop_info->crop_type != kDvppCropTypeCentre) { - return SUCCESS; - } - if (image_width < crop_info->crop_width) { - MSI_LOG_ERROR << "Image width " << image_width << "smaller than crop width " << crop_info->crop_width; - return INFER_STATUS(INVALID_INPUTS) << "Image width " << image_width << "smaller than crop width " - << crop_info->crop_width; - } - if (image_height < crop_info->crop_height) { - MSI_LOG_ERROR << "Image height " << image_height << "smaller than crop height " << crop_info->crop_height; - return INFER_STATUS(INVALID_INPUTS) << "Image width " << image_width << "smaller than crop width " - << crop_info->crop_width; - } - uint32_t left = ToEven((image_width - crop_info->crop_width) / 2); - uint32_t top = ToEven((image_height - crop_info->crop_height) / 2); - uint32_t right = ToOdd(left + crop_info->crop_width); - uint32_t bottom = ToOdd(top + crop_info->crop_height); - - auto acl_ret = acldvppSetRoiConfig(crop_area_, left, right, top, bottom); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Update Crop Area failed"; - return FAILED; - } - MSI_LOG_INFO << "Update crop area, crop type centre, crop info: " - << ", left " << left << ", right " << right << ", top " << top << ", bottom " << bottom; - return SUCCESS; -} - -Status DvppProcess::CheckResizeImageInfo(uint32_t image_width, uint32_t image_height) const { - if (!to_resize_flag_) { - return SUCCESS; - } - // resize ratio required [1/32, 16] - auto check_resize_ratio = [](uint32_t before_resize, uint32_t after_resize) { - if (before_resize == 0 || after_resize == 0) { - return false; - } - if (before_resize / after_resize > 32) { - return false; - } - if (after_resize / before_resize > 16) { - return false; - } - return true; - }; - if (!check_resize_ratio(image_width, resize_para_.output_width)) { - MSI_LOG_ERROR << "Resize ratio required [1/32, 16], current width resize from " << image_width << " to " - << resize_para_.output_width; - return INFER_STATUS(INVALID_INPUTS) << "Resize ratio required [1/32, 16], current width resize from " << image_width - << " to " << resize_para_.output_width; - } - if (!check_resize_ratio(image_height, resize_para_.output_height)) { - MSI_LOG_ERROR << "Resize ratio required [1/32, 16], current height resize from " << image_height << " to " - << resize_para_.output_height; - return INFER_STATUS(INVALID_INPUTS) << "Resize ratio required [1/32, 16], current height resize from " - << image_height << " to " << resize_para_.output_height; - } - return SUCCESS; -} - -void DvppProcess::DestroyDecodeDesc() { - if (decode_output_desc_ != nullptr) { - acldvppDestroyPicDesc(decode_output_desc_); - decode_output_desc_ = nullptr; - } - if (decode_output_buffer_dev_ != nullptr) { - acldvppFree(decode_output_buffer_dev_); - decode_output_buffer_dev_ = nullptr; - } - decode_output_buffer_size_ = 0; - MSI_LOG_INFO << "End destroy decode desc"; -} - -Status DvppProcess::InitResizeOutputDesc() { - if (InitVpcOutputDesc(resize_para_.output_width, resize_para_.output_height, decode_para_.pixel_format) != SUCCESS) { - MSI_LOG_ERROR << "Init VPC output desc failed"; - return FAILED; - } - if (resize_config_ == nullptr) { - resize_config_ = acldvppCreateResizeConfig(); - if (resize_config_ == nullptr) { - MSI_LOG_ERROR << "Create Resize config failed"; - return FAILED; - } - } - return SUCCESS; -} - -Status DvppProcess::InitRoiAreaConfig(acldvppRoiConfig *&roi_area, const DvppRoiArea &init_para) { - if (roi_area == nullptr) { - roi_area = acldvppCreateRoiConfig(init_para.left, init_para.right, init_para.top, init_para.bottom); - if (roi_area == nullptr) { - MSI_LOG_ERROR << "Create Roi config failed"; - return FAILED; - } - } else { - auto acl_ret = acldvppSetRoiConfig(roi_area, init_para.left, init_para.right, init_para.top, init_para.bottom); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Set Roi config failed"; - return FAILED; - } - } - return SUCCESS; -} - -Status DvppProcess::InitCropOutputDesc() { - if (InitVpcOutputDesc(crop_para_.output_width, crop_para_.output_height, decode_para_.pixel_format) != SUCCESS) { - MSI_LOG_ERROR << "Init VPC output desc failed"; - return FAILED; - } - if (InitRoiAreaConfig(crop_area_, crop_para_.crop_info.crop_area) != SUCCESS) { - MSI_LOG_ERROR << "Init crop area failed"; - return FAILED; - } - return SUCCESS; -} - -Status DvppProcess::InitCropAndPasteOutputDesc() { - if (InitVpcOutputDesc(crop_and_paste_para_.output_width, crop_and_paste_para_.output_height, - decode_para_.pixel_format) != SUCCESS) { - MSI_LOG_ERROR << "Init VPC output desc failed"; - return FAILED; - } - if (InitRoiAreaConfig(crop_area_, crop_and_paste_para_.crop_info.crop_area) != SUCCESS) { - MSI_LOG_ERROR << "Init crop area failed"; - return FAILED; - } - if (InitRoiAreaConfig(paste_area_, crop_and_paste_para_.paste_area) != SUCCESS) { - MSI_LOG_ERROR << "Init paste area failed"; - return FAILED; - } - return SUCCESS; -} - -Status DvppProcess::ProcessDecode() { - aclError acl_ret; - acl_ret = acldvppJpegDecodeAsync(dvpp_channel_desc_, input_pic_dev_buffer_, input_pic_buffer_size_, - decode_output_desc_, stream_); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "acldvppJpegDecodeAsync failed, acl return " << acl_ret; - return FAILED; - } - acl_ret = aclrtSynchronizeStream(stream_); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "aclrtSynchronizeStream failed, acl return " << acl_ret; - return FAILED; - } - return SUCCESS; -} - -Status DvppProcess::ProcessResize() { - aclError acl_ret; - acl_ret = acldvppVpcResizeAsync(dvpp_channel_desc_, decode_output_desc_, vpc_output_desc_, resize_config_, stream_); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "acldvppVpcResizeAsync failed, acl return " << acl_ret; - return FAILED; - } - acl_ret = aclrtSynchronizeStream(stream_); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "aclrtSynchronizeStream failed, acl return " << acl_ret; - return FAILED; - } - return SUCCESS; -} - -Status DvppProcess::ProcessCrop() { - aclError acl_ret; - acl_ret = acldvppVpcCropAsync(dvpp_channel_desc_, decode_output_desc_, vpc_output_desc_, crop_area_, stream_); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "acldvppVpcCropAsync failed, acl return " << acl_ret; - return FAILED; - } - acl_ret = aclrtSynchronizeStream(stream_); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "aclrtSynchronizeStream failed, acl return " << acl_ret; - return FAILED; - } - return SUCCESS; -} - -Status DvppProcess::ProcessCropAndPaste() { - aclError acl_ret; - acl_ret = acldvppVpcCropAndPasteAsync(dvpp_channel_desc_, decode_output_desc_, vpc_output_desc_, crop_area_, - paste_area_, stream_); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "acldvppVpcCropAndPasteAsync failed, acl return " << acl_ret; - return FAILED; - } - acl_ret = aclrtSynchronizeStream(stream_); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "aclrtSynchronizeStream failed, acl return " << acl_ret; - return FAILED; - } - return SUCCESS; -} - -Status DvppJsonConfigParser::GetStringValue(const nlohmann::json &json_item, const std::string &key, std::string &val) { - auto it = json_item.find(key); - if (it == json_item.end()) { - MSI_LOG_ERROR << "get string item " << key << " failed"; - return FAILED; - } - if (!it->is_string()) { - MSI_LOG_ERROR << "item " << key << " value is not string type"; - return FAILED; - } - val = it->get(); - return SUCCESS; -} - -Status DvppJsonConfigParser::GetIntValue(const nlohmann::json &json_item, const std::string &key, uint32_t &val) { - auto it = json_item.find(key); - if (it == json_item.end()) { - MSI_LOG_ERROR << "get string item " << key << " failed"; - return FAILED; - } - if (!it->is_number_integer()) { - MSI_LOG_ERROR << "item " << key << " value is not integer type"; - return FAILED; - } - val = it->get(); - return SUCCESS; -} - -Status DvppJsonConfigParser::ParseInputPara(const nlohmann::json &preprocess_item) { - auto input = preprocess_item.find("input"); - if (input == preprocess_item.end()) { - MSI_LOG_ERROR << "get input failed"; - return FAILED; - } - if (!input->is_object()) { - MSI_LOG_ERROR << "input is not object"; - return FAILED; - } - return SUCCESS; -} - -Status DvppJsonConfigParser::ParseDecodePara(const nlohmann::json &preprocess_item) { - auto decode_para = preprocess_item.find("decode_para"); - if (decode_para == preprocess_item.end()) { - MSI_LOG_ERROR << "get input failed"; - return FAILED; - } - if (!decode_para->is_object()) { - MSI_LOG_ERROR << "input is not object"; - return FAILED; - } - const std::unordered_map pixel_format_map = { - {"YUV420SP", PIXEL_FORMAT_YUV_SEMIPLANAR_420}, {"YVU420SP", PIXEL_FORMAT_YVU_SEMIPLANAR_420}, - {"YUV422SP", PIXEL_FORMAT_YUV_SEMIPLANAR_422}, {"YVU422SP", PIXEL_FORMAT_YVU_SEMIPLANAR_422}, - {"YUV444SP", PIXEL_FORMAT_YUV_SEMIPLANAR_444}, {"YVU444SP", PIXEL_FORMAT_YVU_SEMIPLANAR_444}, - }; - std::string pixel_format; - if (GetStringValue(*decode_para, "out_pixel_format", pixel_format) != SUCCESS) { - MSI_LOG_ERROR << "get op out_pixel_format failed"; - return FAILED; - } - auto format = pixel_format_map.find(pixel_format); - if (format == pixel_format_map.end()) { - MSI_LOG_ERROR << "unsupported out_pixel_format " << pixel_format; - return FAILED; - } - decode_para_.pixel_format = format->second; - return SUCCESS; -} - -Status DvppJsonConfigParser::ParseResizePara(const nlohmann::json &json_item) { - if (GetIntValue(json_item, "out_width", resize_para_.output_width) != SUCCESS) { - return FAILED; - } - if (GetIntValue(json_item, "out_height", resize_para_.output_height) != SUCCESS) { - return FAILED; - } - resize_flag_ = true; - return SUCCESS; -} - -Status DvppJsonConfigParser::ParseCropPara(const nlohmann::json &json_item) { - if (GetIntValue(json_item, "out_width", crop_para_.output_width) != SUCCESS) { - return FAILED; - } - if (GetIntValue(json_item, "out_height", crop_para_.output_height) != SUCCESS) { - return FAILED; - } - auto &crop_info = crop_para_.crop_info; - std::string crop_type = "crop_type"; - if (GetStringValue(json_item, "crop_type", crop_type) != SUCCESS) { - return FAILED; - } - if (crop_type == "offset") { - MSI_LOG_INFO << "Crop type is 'offset'"; - crop_info.crop_type = kDvppCropTypeOffset; - auto &crop_area = crop_info.crop_area; - if (GetIntValue(json_item, "crop_left", crop_area.left) != SUCCESS) { - return FAILED; - } - if (GetIntValue(json_item, "crop_top", crop_area.top) != SUCCESS) { - return FAILED; - } - if (GetIntValue(json_item, "crop_right", crop_area.right) != SUCCESS) { - return FAILED; - } - if (GetIntValue(json_item, "crop_bottom", crop_area.bottom) != SUCCESS) { - return FAILED; - } - } else if (crop_type == "centre") { - MSI_LOG_INFO << "Crop type is 'centre'"; - if (GetIntValue(json_item, "crop_width", crop_info.crop_width) != SUCCESS) { - return FAILED; - } - if (GetIntValue(json_item, "crop_height", crop_info.crop_height) != SUCCESS) { - return FAILED; - } - crop_info.crop_type = kDvppCropTypeCentre; - } else { - MSI_LOG_ERROR << "Invalid crop type " << crop_type << ", expect offset or centre"; - return FAILED; - } - crop_flag_ = true; - return SUCCESS; -} - -Status DvppJsonConfigParser::ParseCropAndPastePara(const nlohmann::json &json_item) { - // crop info - if (GetIntValue(json_item, "out_width", crop_and_paste_para_.output_width) != SUCCESS) { - return FAILED; - } - if (GetIntValue(json_item, "out_height", crop_and_paste_para_.output_height) != SUCCESS) { - return FAILED; - } - auto &crop_info = crop_and_paste_para_.crop_info; - std::string crop_type = "crop_type"; - if (GetStringValue(json_item, "crop_type", crop_type) != SUCCESS) { - return FAILED; - } - if (crop_type == "offset") { - MSI_LOG_INFO << "Crop type is 'offset'"; - crop_info.crop_type = kDvppCropTypeOffset; - auto &crop_area = crop_info.crop_area; - if (GetIntValue(json_item, "crop_left", crop_area.left) != SUCCESS) { - return FAILED; - } - if (GetIntValue(json_item, "crop_top", crop_area.top) != SUCCESS) { - return FAILED; - } - if (GetIntValue(json_item, "crop_right", crop_area.right) != SUCCESS) { - return FAILED; - } - if (GetIntValue(json_item, "crop_bottom", crop_area.bottom) != SUCCESS) { - return FAILED; - } - } else if (crop_type == "centre") { - MSI_LOG_INFO << "Crop type is 'centre'"; - if (GetIntValue(json_item, "crop_width", crop_info.crop_width) != SUCCESS) { - return FAILED; - } - if (GetIntValue(json_item, "crop_height", crop_info.crop_height) != SUCCESS) { - return FAILED; - } - crop_info.crop_type = kDvppCropTypeCentre; - } else { - MSI_LOG_ERROR << "Invalid crop type " << crop_type << ", expect offset or centre"; - return FAILED; - } - // paste info - auto &paste_area = crop_and_paste_para_.paste_area; - if (GetIntValue(json_item, "paste_left", paste_area.left) != SUCCESS) { - return FAILED; - } - if (GetIntValue(json_item, "paste_top", paste_area.top) != SUCCESS) { - return FAILED; - } - if (GetIntValue(json_item, "paste_right", paste_area.right) != SUCCESS) { - return FAILED; - } - if (GetIntValue(json_item, "paste_bottom", paste_area.bottom) != SUCCESS) { - return FAILED; - } - crop_and_paste_flag_ = true; - return SUCCESS; -} - -Status DvppJsonConfigParser::InitWithJsonConfigImp(const std::string &json_config) { - std::ifstream fp(json_config); - if (!fp.is_open()) { - MSI_LOG_ERROR << "read json config file failed"; - return FAILED; - } - const auto &model_info = nlohmann::json::parse(fp); - auto preprocess_list = model_info.find("preprocess"); - if (preprocess_list == model_info.end()) { - MSI_LOG_ERROR << "get preprocess failed"; - return FAILED; - } - if (!preprocess_list->is_array()) { - MSI_LOG_ERROR << "preprocess is not array"; - return FAILED; - } - if (preprocess_list->empty()) { - MSI_LOG_ERROR << "preprocess size is 0"; - return FAILED; - } - auto &preprocess = preprocess_list->at(0); - // input - if (ParseInputPara(preprocess) != SUCCESS) { - MSI_LOG_ERROR << "parse input failed"; - return FAILED; - } - // decode para - if (ParseDecodePara(preprocess) != SUCCESS) { - MSI_LOG_ERROR << "parse decode failed"; - return FAILED; - } - // ops - auto dvpp_process = preprocess.find("dvpp_process"); - if (dvpp_process == preprocess.end()) { - MSI_LOG_ERROR << "get dvpp_process failed"; - return FAILED; - } - if (!dvpp_process->is_object()) { - MSI_LOG_ERROR << "dvpp_process is not array"; - return FAILED; - } - const auto &item = *dvpp_process; - std::string op_name; - if (GetStringValue(item, "op_name", op_name) != SUCCESS) { - return FAILED; - } - if (op_name == "resize") { - if (ParseResizePara(item) != SUCCESS) { - MSI_LOG_ERROR << "Parse resize para failed"; - return FAILED; - } - } else if (op_name == "crop") { - if (ParseCropPara(item) != SUCCESS) { - MSI_LOG_ERROR << "Parse crop para failed"; - return FAILED; - } - } else if (op_name == "crop_and_paste") { - if (ParseCropAndPastePara(item) != SUCCESS) { - MSI_LOG_ERROR << "Parse decode para failed"; - return FAILED; - } - } else { - MSI_LOG_ERROR << "Unsupported op name " << op_name << ", expect resize, crop or crop_and_paste"; - return FAILED; - } - return SUCCESS; -} - -Status DvppJsonConfigParser::InitWithJsonConfig(const std::string &json_config) { - try { - auto ret = InitWithJsonConfigImp(json_config); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "init dvpp with json config failed, json config " << json_config; - return FAILED; - } - } catch (nlohmann::json::exception &e) { - MSI_LOG_ERROR << "init dvpp with json config failed, json config " << json_config << ", error: " << e.what(); - return FAILED; - } - MSI_LOG_INFO << "Init with json config " << json_config << " success"; - return SUCCESS; -} - -Status DvppProcess::InitWithJsonConfig(const std::string &json_config) { - DvppJsonConfigParser parser; - if (parser.InitWithJsonConfig(json_config) != SUCCESS) { - MSI_LOG_ERROR << "init json config failed"; - return FAILED; - } - if (InitJpegDecodePara(parser.GetDecodePara()) != SUCCESS) { - MSI_LOG_ERROR << "init decode para failed"; - return FAILED; - } - if (parser.HasResizeConfig()) { - if (InitResizePara(parser.GetResizePara()) != SUCCESS) { - MSI_LOG_ERROR << "init resize para failed"; - return FAILED; - } - } else if (parser.HasCropConfig()) { - if (InitCropPara(parser.GetCropPara()) != SUCCESS) { - MSI_LOG_ERROR << "init crop para failed"; - return FAILED; - } - } else if (parser.HasCropAndPasteConfig()) { - if (InitCropAndPastePara(parser.GetCropAndPastePara()) != SUCCESS) { - MSI_LOG_ERROR << "init crop and paste para failed"; - return FAILED; - } - } - return SUCCESS; -} -} // namespace inference -} // namespace mindspore diff --git a/serving/acl/dvpp_process.h b/serving/acl/dvpp_process.h deleted file mode 100644 index da0275118d8..00000000000 --- a/serving/acl/dvpp_process.h +++ /dev/null @@ -1,159 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef INC_DVPP_PROCESS_ACL -#define INC_DVPP_PROCESS_ACL -#include -#include -#include "acl/acl.h" -#include "acl/acl_mdl.h" -#include "acl/acl_rt.h" -#include "acl/ops/acl_dvpp.h" -#include "include/inference.h" - -namespace mindspore::inference { - -struct DvppDecodePara { - acldvppPixelFormat pixel_format = PIXEL_FORMAT_YUV_SEMIPLANAR_420; -}; - -struct DvppResizePara { - uint32_t output_width = 0; - uint32_t output_height = 0; -}; - -enum DvppCropType { - // crop left,top,right,bottom is given in config - kDvppCropTypeOffset = 0, - // crop left,top,right,bottom is calculated by image width/height and output crop width/height - kDvppCropTypeCentre = 1, -}; - -struct DvppRoiArea { - uint32_t left = 0; - uint32_t top = 0; - uint32_t right = 0; - uint32_t bottom = 0; -}; - -struct DvppCropInfo { - DvppCropType crop_type = kDvppCropTypeOffset; - DvppRoiArea crop_area; // when kDvppCropTypeOffset - uint32_t crop_width = 0; // when kDvppCropTypeCentre - uint32_t crop_height = 0; // when kDvppCropTypeCentre -}; - -struct DvppCropPara { - DvppCropInfo crop_info; - uint32_t output_width = 0; - uint32_t output_height = 0; -}; - -struct DvppCropAndPastePara { - DvppCropInfo crop_info; - DvppRoiArea paste_area; - uint32_t output_width = 0; - uint32_t output_height = 0; -}; - -class DvppProcess { - public: - DvppProcess(); - ~DvppProcess(); - - Status InitResource(aclrtStream stream); - void Finalize(); - Status InitJpegDecodePara(const DvppDecodePara &decode_para); // jpeg decode + (resize | crop) - Status InitResizePara(const DvppResizePara &resize_para); // jpeg decode + resize - Status InitCropPara(const DvppCropPara &crop_para); // jpeg decode + crop - Status InitCropAndPastePara(const DvppCropAndPastePara &crop_and_paste_para); // jpeg decode + crop&paste - - Status InitWithJsonConfig(const std::string &json_config); - - // output device buffer will be destroy by DvppProcess itself. - Status Process(const void *pic_buffer, size_t pic_buffer_size, void *&output_device_buffer, size_t &output_size); - Status Process(const std::vector &pic_buffer_list, const std::vector &pic_buffer_size_list, - void *&output_device_buffer, size_t &output_size); - - private: - uint32_t pic_width_ = 0; - uint32_t pic_height_ = 0; - - DvppDecodePara decode_para_; - DvppResizePara resize_para_; - DvppCropPara crop_para_; - DvppCropAndPastePara crop_and_paste_para_; - // only one of the resize or crop flag can be true - bool to_resize_flag_ = false; - bool to_crop_flag_ = false; - bool to_crop_and_paste_flag_ = false; - - void *input_pic_dev_buffer_ = nullptr; - uint32_t input_pic_buffer_size_ = 0; - - uint32_t decode_output_buffer_size_ = 0; - void *decode_output_buffer_dev_ = nullptr; - acldvppPicDesc *decode_output_desc_ = nullptr; - - acldvppResizeConfig *resize_config_ = nullptr; - acldvppRoiConfig *crop_area_ = nullptr; - acldvppRoiConfig *paste_area_ = nullptr; - - acldvppPicDesc *vpc_output_desc_ = nullptr; - void *vpc_output_buffer_dev_ = nullptr; // vpc_output_buffer_size_ length - uint32_t vpc_output_buffer_size_ = 0; - - void *batch_vpc_output_buffer_dev_ = nullptr; // batch_size_ * vpc_output_buffer_size_ length - uint32_t batch_size_ = 0; - - aclrtStream stream_ = nullptr; - acldvppChannelDesc *dvpp_channel_desc_ = nullptr; - - uint32_t AlignmentHelper(uint32_t org_size, uint32_t alignment) const; - uint32_t GetImageBufferSize(uint32_t stride_width, uint32_t stride_height, acldvppPixelFormat pixel_format) const; - Status GetPicDescStride(uint32_t width, uint32_t height, uint32_t &stride_width, uint32_t &stride_height); - Status GetPicDescStrideDecode(uint32_t width, uint32_t height, uint32_t &stride_width, uint32_t &stride_height); - Status InputInputBuffer(const void *pic_buffer, size_t pic_buffer_size); - Status InitDecodeOutputDesc(uint32_t image_width, - uint32_t image_height); // decode_output_desc_, decode_output_buffer_dev_ - Status CheckRoiAreaWidthHeight(uint32_t width, uint32_t height); - Status CheckAndAdjustRoiArea(DvppRoiArea &area); - Status UpdateCropArea(uint32_t image_width, uint32_t image_height); - Status CheckResizeImageInfo(uint32_t image_width, uint32_t image_height) const; - void DestroyDecodeDesc(); - - Status InitVpcOutputDesc(uint32_t output_width, uint32_t output_height, - acldvppPixelFormat pixel_format); // vpc_output_desc_, vpc_output_buffer_dev_batch_ - Status InitRoiAreaConfig(acldvppRoiConfig *&roi_area, const DvppRoiArea &init_para); - Status InitCommonCropPara(DvppCropInfo &crop_info, uint32_t out_width, uint32_t out_height); - Status InitResizeOutputDesc(); // vpc_output_desc_, vpc_output_buffer_dev_, resize_config - Status InitCropOutputDesc(); // vpc_output_desc_, vpc_output_buffer_dev_, crop_area_ - Status InitCropAndPasteOutputDesc(); // vpc_output_desc_, vpc_output_buffer_dev_, crop_area_, paste_area_ - void DestroyVpcOutputDesc(); - - Status ProcessDecode(); - Status ProcessResize(); - Status ProcessCrop(); - Status ProcessCropAndPaste(); - void DestroyResource(); - - Status GetJpegWidthHeight(const void *pic_buffer, size_t pic_buffer_size, uint32_t &image_width, - uint32_t &image_height); -}; - -} // namespace mindspore::inference - -#endif // INC_DVPP_PROCESS_ACL diff --git a/serving/acl/model_info_example.json b/serving/acl/model_info_example.json deleted file mode 100644 index e6d37048d04..00000000000 --- a/serving/acl/model_info_example.json +++ /dev/null @@ -1,68 +0,0 @@ -{ - "preprocess": [ - { - "input": { - "index": 0 - }, - "decode_para": { - "out_pixel_format": "YUV420SP" - }, - "dvpp_process": { - "op_name": "resize", - "out_width": 224, - "out_height": 224 - }, - "sample of dvpp_process content": [ - { - "op_name": "resize", - "out_width": 224, - "out_height": 224 - }, - { - "op_name": "crop", - "crop_type": "offset", - "crop_left": 10, - "crop_top": 10, - "crop_right": 100, - "crop_bottom": 200, - "out_width": 224, - "out_height": 224 - }, - { - "op_name": "crop", - "crop_type": "centre", - "crop_width": 100, - "crop_height": 100, - "out_width": 224, - "out_height": 224 - }, - { - "op_name": "crop_and_paste", - "crop_type": "offset", - "crop_left": 10, - "crop_top": 10, - "crop_right": 100, - "crop_bottom": 200, - "paste_left": 10, - "paste_top": 10, - "paste_right": 100, - "paste_bottom": 200, - "out_width": 224, - "out_height": 224 - }, - { - "op_name": "crop_and_paste", - "crop_type": "centre", - "crop_width": 100, - "crop_height": 100, - "paste_left": 10, - "paste_top": 10, - "paste_right": 100, - "paste_bottom": 200, - "out_width": 224, - "out_height": 224 - } - ] - } - ] -} \ No newline at end of file diff --git a/serving/acl/model_process.cc b/serving/acl/model_process.cc deleted file mode 100644 index 0c0debdb0b7..00000000000 --- a/serving/acl/model_process.cc +++ /dev/null @@ -1,431 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "serving/acl/model_process.h" -#include -#include - -#include "include/infer_log.h" - -namespace mindspore { -namespace inference { -Status ModelProcess::PreInitModelResource() { - model_desc_ = aclmdlCreateDesc(); - aclError acl_ret = aclmdlGetDesc(model_desc_, model_id_); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Read model desc failed"; - return FAILED; - } - Status ret = InitInputsBuffer(); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "Create input buffer failed"; - return FAILED; - } - ret = InitOutputsBuffer(); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "Create output buffer failed"; - return FAILED; - } - return SUCCESS; -} - -Status ModelProcess::LoadModelFromFile(const std::string &file_name, uint32_t &model_id) { - aclError acl_ret = aclmdlLoadFromFile(file_name.c_str(), &model_id); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Read model file failed, file name is " << file_name; - return FAILED; - } - MSI_LOG_INFO << "Load model success " << file_name; - model_id_ = model_id; - if (PreInitModelResource() != SUCCESS) { - aclmdlUnload(model_id_); - MSI_LOG_ERROR << "Pre init model resource failed, file name is " << file_name; - return FAILED; - } - return SUCCESS; -} - -Status ModelProcess::InitInputsBuffer() { - aclError ret; - size_t input_size = aclmdlGetNumInputs(model_desc_); - - for (size_t i = 0; i < input_size; ++i) { - auto buffer_size = aclmdlGetInputSizeByIndex(model_desc_, i); - void *data_mem_buffer = nullptr; - if (!is_run_on_device_) { // need to copy input/output to/from device - ret = aclrtMalloc(&data_mem_buffer, buffer_size, ACL_MEM_MALLOC_NORMAL_ONLY); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Malloc device input buffer faild , input size " << buffer_size; - return FAILED; - } - } - - aclmdlIODims dims; - ret = aclmdlGetInputDims(model_desc_, i, &dims); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Get input shape failed"; - if (!is_run_on_device_) { - aclrtFree(data_mem_buffer); - } - return FAILED; - } - aclDataType data_type = aclmdlGetInputDataType(model_desc_, i); - std::vector shape(dims.dims, dims.dims + dims.dimCount); - input_infos_.emplace_back(AclTensorInfo{data_mem_buffer, buffer_size, data_type, shape}); - } - MSI_LOG_INFO << "Create model inputs success"; - return SUCCESS; -} - -Status ModelProcess::CreateDataBuffer(void *&data_mem_buffer, size_t buffer_size, aclmdlDataset *dataset) { - aclError ret; - auto free_data_buffer = [this](void *dataMemBuffer) { - if (!is_run_on_device_) { - aclrtFree(dataMemBuffer); - } else { - aclrtFreeHost(dataMemBuffer); - } - }; - if (!is_run_on_device_) { - ret = aclrtMalloc(&data_mem_buffer, buffer_size, ACL_MEM_MALLOC_NORMAL_ONLY); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Malloc device buffer faild , buffer size " << buffer_size; - return FAILED; - } - } else { - ret = aclrtMallocHost(&data_mem_buffer, buffer_size); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Malloc device buffer faild , buffer size " << buffer_size; - return FAILED; - } - } - - auto data_buffer = aclCreateDataBuffer(data_mem_buffer, buffer_size); - if (data_buffer == nullptr) { - MSI_LOG_ERROR << "Create Data Buffer failed"; - free_data_buffer(data_mem_buffer); - return FAILED; - } - ret = aclmdlAddDatasetBuffer(dataset, data_buffer); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "add data buffer failed"; - free_data_buffer(data_mem_buffer); - aclDestroyDataBuffer(data_buffer); - return FAILED; - } - return SUCCESS; -} - -Status ModelProcess::InitOutputsBuffer() { - aclError ret; - outputs_ = aclmdlCreateDataset(); - if (outputs_ == nullptr) { - MSI_LOG_ERROR << "Create input dataset failed"; - return FAILED; - } - size_t output_size = aclmdlGetNumOutputs(model_desc_); - for (size_t i = 0; i < output_size; ++i) { - auto buffer_size = aclmdlGetOutputSizeByIndex(model_desc_, i); - - void *data_mem_buffer = nullptr; - if (CreateDataBuffer(data_mem_buffer, buffer_size, outputs_) != SUCCESS) { - MSI_LOG_ERROR << "add output data buffer failed, buffer size " << buffer_size; - return FAILED; - } - aclmdlIODims dims; - ret = aclmdlGetOutputDims(model_desc_, i, &dims); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Get input shape failed"; - if (!is_run_on_device_) { - aclrtFree(data_mem_buffer); - } else { - aclrtFreeHost(data_mem_buffer); - } - return FAILED; - } - aclDataType data_type = aclmdlGetOutputDataType(model_desc_, i); - std::vector shape(dims.dims, dims.dims + dims.dimCount); - output_infos_.emplace_back(AclTensorInfo{data_mem_buffer, buffer_size, data_type, shape}); - } - MSI_LOG_INFO << "Create model output success"; - return SUCCESS; -} - -void ModelProcess::DestroyInputsDataset() { - if (inputs_ == nullptr) { - return; - } - for (size_t i = 0; i < aclmdlGetDatasetNumBuffers(inputs_); i++) { - auto dataBuffer = aclmdlGetDatasetBuffer(inputs_, i); - aclDestroyDataBuffer(dataBuffer); - } - aclmdlDestroyDataset(inputs_); - inputs_ = nullptr; -} - -void ModelProcess::DestroyInputsDataMem() { - if (!is_run_on_device_) { - for (const auto &item : input_infos_) { - aclrtFree(item.device_data); - } - } - input_infos_.clear(); -} - -void ModelProcess::DestroyInputsBuffer() { - DestroyInputsDataMem(); - DestroyInputsDataset(); -} - -void ModelProcess::DestroyOutputsBuffer() { - for (const auto &item : output_infos_) { - if (!is_run_on_device_) { - aclrtFree(item.device_data); - } else { - aclrtFreeHost(item.device_data); - } - } - output_infos_.clear(); - - if (outputs_ == nullptr) { - return; - } - for (size_t i = 0; i < aclmdlGetDatasetNumBuffers(outputs_); i++) { - auto dataBuffer = aclmdlGetDatasetBuffer(outputs_, i); - aclDestroyDataBuffer(dataBuffer); - } - aclmdlDestroyDataset(outputs_); - outputs_ = nullptr; -} - -void ModelProcess::UnLoad() { - auto ret = aclmdlUnload(model_id_); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Unload model failed"; - } - if (model_desc_ != nullptr) { - aclmdlDestroyDesc(model_desc_); - model_desc_ = nullptr; - } - DestroyInputsBuffer(); - DestroyOutputsBuffer(); - MSI_LOG_INFO << "End unload model " << model_id_; -} - -Status ModelProcess::CheckAndInitInput(const RequestBase &request) { - aclError ret; - inputs_ = aclmdlCreateDataset(); - // check inputs - if (request.size() != input_infos_.size()) { - MSI_LOG_ERROR << "inputs count not match, required count " << input_infos_.size() << ", given count " - << request.size(); - return INFER_STATUS(INVALID_INPUTS) << "inputs count not match, required count " << input_infos_.size() - << ", given count " << request.size(); - } - for (size_t i = 0; i < input_infos_.size(); i++) { - if (request[i] == nullptr) { - MSI_LOG_ERROR << "input " << i << " cannot be null"; - return FAILED; - } - if (request[i]->data_size() != input_infos_[i].buffer_size) { - MSI_LOG_ERROR << "input " << i << " data size not match, required size " << input_infos_[i].buffer_size - << ", given count " << request[i]->data_size(); - return INFER_STATUS(INVALID_INPUTS) << "input " << i << " data size not match, required size " - << input_infos_[i].buffer_size << ", given count " << request[i]->data_size(); - } - } - // copy inputs - for (size_t i = 0; i < input_infos_.size(); i++) { - void *input_buffer = nullptr; - auto &info = input_infos_[i]; - const void *data = request[i]->data(); - if (!is_run_on_device_) { - ret = aclrtMemcpy(info.device_data, info.buffer_size, data, request[i]->data_size(), ACL_MEMCPY_HOST_TO_DEVICE); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "memcpy input " << i << " data to device failed, buffer size " << request[i]->data_size(); - return FAILED; - } - input_buffer = info.device_data; - } else { - input_buffer = const_cast(data); - } - auto data_buffer = aclCreateDataBuffer(input_buffer, info.buffer_size); - if (data_buffer == nullptr) { - MSI_LOG_ERROR << "Create Data Buffer failed"; - return FAILED; - } - ret = aclmdlAddDatasetBuffer(inputs_, data_buffer); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "add data buffer failed"; - aclDestroyDataBuffer(data_buffer); - return FAILED; - } - } - return SUCCESS; -} - -Status ModelProcess::CheckAndInitDvppInput(const void *dvpp_outputs_buffer_dev, size_t dvpp_outputs_buffer_size, - size_t input_index) { - aclError ret; - inputs_ = aclmdlCreateDataset(); - // check inputs - if (input_index >= input_infos_.size()) { - MSI_LOG_ERROR << "inputs count not match, required count " << input_infos_.size() << ", given index " - << input_index; - return INFER_STATUS(INVALID_INPUTS) << "inputs count not match, required count " << input_infos_.size() - << ", given index " << input_index; - } - if (dvpp_outputs_buffer_dev == nullptr) { - MSI_LOG_ERROR << "input " << 0 << " cannot be null"; - return FAILED; - } - if (dvpp_outputs_buffer_size != input_infos_[input_index].buffer_size) { - MSI_LOG_ERROR << "input " << 0 << " data size not match, required size " << input_infos_[input_index].buffer_size - << ", given count " << dvpp_outputs_buffer_size; - return INFER_STATUS(INVALID_INPUTS) << "input " << 0 << " data size not match, required size " - << input_infos_[input_index].buffer_size << ", given count " - << dvpp_outputs_buffer_size; - } - // copy inputs - auto &info = input_infos_[input_index]; - auto data_buffer = aclCreateDataBuffer(const_cast(dvpp_outputs_buffer_dev), info.buffer_size); - if (data_buffer == nullptr) { - MSI_LOG_ERROR << "Create Data Buffer failed"; - return FAILED; - } - ret = aclmdlAddDatasetBuffer(inputs_, data_buffer); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "add data buffer failed"; - aclDestroyDataBuffer(data_buffer); - return FAILED; - } - return SUCCESS; -} - -Status ModelProcess::BuildOutputs(ReplyBase &reply) { - aclError ret; - // copy outputs - reply.clear(); - - std::unordered_map data_type_map = { - {ACL_FLOAT16, inference::kMSI_Float16}, {ACL_FLOAT, inference::kMSI_Float32}, {ACL_DOUBLE, inference::kMSI_Float64}, - {ACL_INT8, inference::kMSI_Int8}, {ACL_INT16, inference::kMSI_Int16}, {ACL_INT32, inference::kMSI_Int32}, - {ACL_INT64, inference::kMSI_Int64}, {ACL_UINT8, inference::kMSI_Uint8}, {ACL_UINT16, inference::kMSI_Uint16}, - {ACL_UINT32, inference::kMSI_Uint32}, {ACL_UINT64, inference::kMSI_Uint64}, {ACL_BOOL, inference::kMSI_Bool}, - }; - auto trans_to_serving_type = [&data_type_map](aclDataType data_type) { - auto it = data_type_map.find(data_type); - if (it == data_type_map.end()) { - return inference::kMSI_Unknown; - } else { - return it->second; - } - }; - for (size_t i = 0; i < output_infos_.size(); i++) { - auto &info = output_infos_[i]; - auto output = reply.add(); - if (output == nullptr) { - MSI_LOG_ERROR << "add new output failed"; - return FAILED; - } - output->set_data_type(trans_to_serving_type(info.data_type)); - output->set_shape(info.dims); - if (!output->resize_data(info.buffer_size)) { - MSI_LOG_ERROR << "new output data buffer failed, data size " << info.buffer_size; - return FAILED; - } - if (!is_run_on_device_) { - ret = aclrtMemcpy(output->mutable_data(), output->data_size(), info.device_data, info.buffer_size, - ACL_MEMCPY_DEVICE_TO_HOST); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Memcpy output " << i << " to host failed, memory size " << info.buffer_size; - return FAILED; - } - } else { - ret = aclrtMemcpy(output->mutable_data(), output->data_size(), info.device_data, info.buffer_size, - ACL_MEMCPY_HOST_TO_HOST); - if (ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Memcpy output " << i << " to host failed, memory size " << info.buffer_size; - return FAILED; - } - } - } - return SUCCESS; -} - -Status ModelProcess::Execute(const RequestBase &request, ReplyBase &reply) { - aclError acl_ret; - Status ret = CheckAndInitInput(request); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "check or init input failed"; - DestroyInputsDataset(); - return ret; // forward status error - } - acl_ret = aclmdlExecute(model_id_, inputs_, outputs_); - DestroyInputsDataset(); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Execute Model Failed"; - return FAILED; - } - ret = BuildOutputs(reply); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "Build outputs faield"; - return FAILED; - } - MSI_LOG_INFO << "excute model success"; - return SUCCESS; -} - -Status ModelProcess::Execute(const void *dvpp_outputs_buffer_dev, size_t dvpp_outputs_buffer_size, ReplyBase &reply) { - aclError acl_ret; - if (input_infos_.size() != 1) { - MSI_LOG_ERROR << "can only support input size 1, now model inputs size is " << input_infos_.size(); - return INFER_STATUS(INVALID_INPUTS) << "can only support input size 1, now model inputs size is " - << input_infos_.size(); - } - Status ret = CheckAndInitDvppInput(dvpp_outputs_buffer_dev, dvpp_outputs_buffer_size, 0); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "check or init input failed"; - DestroyInputsDataset(); - return ret; // forward status msg - } - acl_ret = aclmdlExecute(model_id_, inputs_, outputs_); - DestroyInputsDataset(); - if (acl_ret != ACL_ERROR_NONE) { - MSI_LOG_ERROR << "Execute Model Failed"; - return INFER_STATUS(FAILED) << "Execute Model Failed"; - } - ret = BuildOutputs(reply); - if (ret != SUCCESS) { - MSI_LOG_ERROR << "Build outputs faield"; - return FAILED; - } - MSI_LOG_INFO << "excute model success"; - return SUCCESS; -} - -size_t ModelProcess::GetBatchSize() const { - if (input_infos_.empty()) { - MSI_LOG_ERROR << "Model is not loaded"; - return 0; - } - if (input_infos_[0].dims.empty()) { - return 1; - } - return static_cast(input_infos_[0].dims[0]); -} -} // namespace inference -} // namespace mindspore diff --git a/serving/acl/model_process.h b/serving/acl/model_process.h deleted file mode 100644 index c9995035aaf..00000000000 --- a/serving/acl/model_process.h +++ /dev/null @@ -1,83 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef INC_MODEL_PROCESS_ACL -#define INC_MODEL_PROCESS_ACL -#include -#include -#include "acl/acl.h" -#include "acl/acl_mdl.h" -#include "acl/acl_rt.h" -#include "include/inference.h" - -namespace mindspore { -namespace inference { -struct AclTensorInfo { - void *device_data; - size_t buffer_size; - aclDataType data_type; - std::vector dims; -}; - -struct ImagesDvppOutput { - void *buffer_device = nullptr; - size_t buffer_size = 0; - size_t input_index = 0; -}; - -class ModelProcess { - public: - ModelProcess() {} - ~ModelProcess() {} - - Status LoadModelFromFile(const std::string &file_name, uint32_t &model_id); - void UnLoad(); - - // override this method to avoid request/reply data copy - Status Execute(const RequestBase &request, ReplyBase &reply); - Status Execute(const void *dvpp_outputs_buffer_dev, size_t dvpp_outputs_buffer_size, ReplyBase &reply); - void SetIsDevice(bool is_device) { is_run_on_device_ = is_device; } - - size_t GetBatchSize() const; - - private: - uint32_t model_id_ = 0xffffffff; - // if run one device(AICPU), there is no need to alloc device memory and copy inputs to(/outputs from) device - bool is_run_on_device_ = false; - aclmdlDesc *model_desc_ = nullptr; - aclmdlDataset *inputs_ = nullptr; - aclmdlDataset *outputs_ = nullptr; - std::vector input_infos_; - std::vector output_infos_; - - Status PreInitModelResource(); - Status CreateDataBuffer(void *&data_mem_buffer, size_t buffer_size, aclmdlDataset *dataset); - Status CheckAndInitInput(const RequestBase &request); - Status CheckAndInitDvppInput(const void *dvpp_outputs_buffer_dev, size_t dvpp_outputs_buffer_size, - size_t input_index); - Status BuildOutputs(ReplyBase &reply); - - Status InitInputsBuffer(); - Status InitOutputsBuffer(); - void DestroyInputsDataset(); - void DestroyInputsDataMem(); - void DestroyInputsBuffer(); - void DestroyOutputsBuffer(); -}; -} // namespace inference -} // namespace mindspore - -#endif diff --git a/serving/core/http_process.cc b/serving/core/http_process.cc deleted file mode 100644 index f8a082d2b2a..00000000000 --- a/serving/core/http_process.cc +++ /dev/null @@ -1,561 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include -#include "serving/ms_service.pb.h" -#include "util/status.h" -#include "core/session.h" -#include "core/http_process.h" -#include "core/serving_tensor.h" - -using ms_serving::MSService; -using ms_serving::PredictReply; -using ms_serving::PredictRequest; -using nlohmann::json; - -namespace mindspore { -namespace serving { -const int BUF_MAX = 0x7FFFFFFF; -static constexpr char HTTP_DATA[] = "data"; -static constexpr char HTTP_TENSOR[] = "tensor"; -enum HTTP_TYPE { TYPE_DATA = 0, TYPE_TENSOR }; -enum HTTP_DATA_TYPE { HTTP_DATA_NONE, HTTP_DATA_INT, HTTP_DATA_FLOAT }; - -static const std::map infer_type2_http_type{ - {inference::DataType::kMSI_Int32, HTTP_DATA_INT}, {inference::DataType::kMSI_Float32, HTTP_DATA_FLOAT}}; - -Status GetPostMessage(struct evhttp_request *const req, std::string *const buf) { - Status status(SUCCESS); - size_t post_size = evbuffer_get_length(req->input_buffer); - if (post_size == 0) { - ERROR_INFER_STATUS(status, INVALID_INPUTS, "http message invalid"); - return status; - } else if (post_size > BUF_MAX) { - ERROR_INFER_STATUS(status, INVALID_INPUTS, "http message is bigger than 0x7FFFFFFF."); - return status; - } else { - buf->resize(post_size); - auto src_data = evbuffer_pullup(req->input_buffer, -1); - if (src_data == nullptr) { - ERROR_INFER_STATUS(status, FAILED, "get http message failed."); - return status; - } - if (memcpy_s(buf->data(), post_size, src_data, post_size) != EOK) { - ERROR_INFER_STATUS(status, FAILED, "copy http message failed."); - return status; - } - return status; - } -} - -Status CheckRequestValid(const struct evhttp_request *const http_request) { - Status status(SUCCESS); - switch (evhttp_request_get_command(http_request)) { - case EVHTTP_REQ_POST: - return status; - default: - ERROR_INFER_STATUS(status, INVALID_INPUTS, "http message only support POST right now"); - return status; - } -} - -void ErrorMessage(struct evhttp_request *const req, Status status) { - json error_json = {{"error_message", status.StatusMessage()}}; - std::string out_error_str = error_json.dump(); - struct evbuffer *retbuff = evbuffer_new(); - evbuffer_add(retbuff, out_error_str.data(), out_error_str.size()); - evhttp_send_reply(req, HTTP_OK, "Client", retbuff); - evbuffer_free(retbuff); -} - -Status CheckMessageValid(const json &message_info, HTTP_TYPE *const type) { - Status status(SUCCESS); - int count = 0; - if (message_info.find(HTTP_DATA) != message_info.end()) { - *type = TYPE_DATA; - count++; - } - if (message_info.find(HTTP_TENSOR) != message_info.end()) { - *type = TYPE_TENSOR; - count++; - } - if (count != 1) { - ERROR_INFER_STATUS(status, INVALID_INPUTS, "http message must have only one type of (data, tensor)"); - return status; - } - return status; -} - -std::vector GetJsonArrayShape(const json &json_array) { - std::vector json_shape; - const json *tmp_json = &json_array; - while (tmp_json->is_array()) { - if (tmp_json->empty()) { - break; - } - json_shape.push_back(tmp_json->size()); - tmp_json = &tmp_json->at(0); - } - return json_shape; -} - -Status GetScalarDataFromJson(const json &json_data_array, ServingTensor *const request_tensor, HTTP_DATA_TYPE type) { - Status status(SUCCESS); - auto type_name = [](const json &json_data) -> std::string { - if (json_data.is_number_integer()) { - return "integer"; - } else if (json_data.is_number_float()) { - return "float"; - } - return json_data.type_name(); - }; - const json *json_data = &json_data_array; - if (json_data_array.is_array()) { - if (json_data_array.size() != 1 || json_data_array[0].is_array()) { - status = INFER_STATUS(INVALID_INPUTS) << "get data failed, expected scalar data is scalar or shape(1) array, " - "now array shape is " - << GetJsonArrayShape(json_data_array); - MSI_LOG_ERROR << status.StatusMessage(); - return status; - } - json_data = &json_data_array.at(0); - } - if (type == HTTP_DATA_INT) { - auto data = reinterpret_cast(request_tensor->mutable_data()); - if (!json_data->is_number_integer()) { - status = INFER_STATUS(INVALID_INPUTS) << "get data failed, expected integer, given " << type_name(*json_data); - MSI_LOG_ERROR << status.StatusMessage(); - return status; - } - data[0] = json_data->get(); - } else if (type == HTTP_DATA_FLOAT) { - auto data = reinterpret_cast(request_tensor->mutable_data()); - if (!json_data->is_number_float()) { - status = INFER_STATUS(INVALID_INPUTS) << "get data failed, expected float, given " << type_name(*json_data); - MSI_LOG_ERROR << status.StatusMessage(); - return status; - } - data[0] = json_data->get(); - } - return SUCCESS; -} - -Status GetDataFromJson(const json &json_data_array, ServingTensor *const request_tensor, size_t data_index, - HTTP_DATA_TYPE type) { - Status status(SUCCESS); - auto type_name = [](const json &json_data) -> std::string { - if (json_data.is_number_integer()) { - return "integer"; - } else if (json_data.is_number_float()) { - return "float"; - } - return json_data.type_name(); - }; - size_t array_size = json_data_array.size(); - if (type == HTTP_DATA_INT) { - auto data = reinterpret_cast(request_tensor->mutable_data()) + data_index; - for (size_t k = 0; k < array_size; k++) { - auto &json_data = json_data_array[k]; - if (!json_data.is_number_integer()) { - status = INFER_STATUS(INVALID_INPUTS) << "get data failed, expected integer, given " << type_name(json_data); - MSI_LOG_ERROR << status.StatusMessage(); - return status; - } - data[k] = json_data.get(); - } - } else if (type == HTTP_DATA_FLOAT) { - auto data = reinterpret_cast(request_tensor->mutable_data()) + data_index; - for (size_t k = 0; k < array_size; k++) { - auto &json_data = json_data_array[k]; - if (!json_data.is_number_float()) { - status = INFER_STATUS(INVALID_INPUTS) << "get data failed, expected float, given " << type_name(json_data); - MSI_LOG_ERROR << status.StatusMessage(); - return status; - } - data[k] = json_data.get(); - } - } - return SUCCESS; -} - -Status RecusiveGetTensor(const json &json_data, size_t depth, ServingTensor *const request_tensor, size_t data_index, - HTTP_DATA_TYPE type) { - Status status(SUCCESS); - std::vector required_shape = request_tensor->shape(); - if (depth >= required_shape.size()) { - status = INFER_STATUS(INVALID_INPUTS) - << "input tensor shape dims is more than required dims " << required_shape.size(); - MSI_LOG_ERROR << status.StatusMessage(); - return status; - } - if (!json_data.is_array()) { - ERROR_INFER_STATUS(status, INVALID_INPUTS, "the tensor is constructed illegally"); - return status; - } - if (json_data.size() != static_cast(required_shape[depth])) { - status = INFER_STATUS(INVALID_INPUTS) - << "tensor format request is constructed illegally, input tensor shape dim " << depth - << " not match, required " << required_shape[depth] << ", given " << json_data.size(); - MSI_LOG_ERROR << status.StatusMessage(); - return status; - } - if (depth + 1 < required_shape.size()) { - size_t sub_element_cnt = - std::accumulate(required_shape.begin() + depth + 1, required_shape.end(), 1LL, std::multiplies()); - for (size_t k = 0; k < json_data.size(); k++) { - status = RecusiveGetTensor(json_data[k], depth + 1, request_tensor, data_index + sub_element_cnt * k, type); - if (status != SUCCESS) { - return status; - } - } - } else { - status = GetDataFromJson(json_data, request_tensor, data_index, type); - if (status != SUCCESS) { - return status; - } - } - return status; -} - -Status TransDataToPredictRequest(const json &message_info, PredictRequest *const request) { - Status status = SUCCESS; - auto tensors = message_info.find(HTTP_DATA); - if (tensors == message_info.end()) { - ERROR_INFER_STATUS(status, INVALID_INPUTS, "http message do not have data type"); - return status; - } - if (!tensors->is_array()) { - ERROR_INFER_STATUS(status, INVALID_INPUTS, "the input tensor list is not array"); - return status; - } - auto const &json_shape = GetJsonArrayShape(*tensors); - if (json_shape.size() != 2) { // 2 is data format list deep - status = INFER_STATUS(INVALID_INPUTS) - << "the data format request is constructed illegally, expected list nesting depth 2, given " - << json_shape.size(); - MSI_LOG_ERROR << status.StatusMessage(); - return status; - } - if (tensors->size() != static_cast(request->data_size())) { - status = INFER_STATUS(INVALID_INPUTS) - << "model input count not match, model required " << request->data_size() << ", given " << tensors->size(); - MSI_LOG_ERROR << status.StatusMessage(); - return status; - } - for (size_t i = 0; i < tensors->size(); i++) { - const auto &tensor = tensors->at(i); - ServingTensor request_tensor(*(request->mutable_data(i))); - auto iter = infer_type2_http_type.find(request_tensor.data_type()); - if (iter == infer_type2_http_type.end()) { - ERROR_INFER_STATUS(status, FAILED, "the model input type is not supported right now"); - return status; - } - HTTP_DATA_TYPE type = iter->second; - if (!tensor.is_array()) { - ERROR_INFER_STATUS(status, INVALID_INPUTS, "the tensor is constructed illegally"); - return status; - } - if (tensor.empty()) { - ERROR_INFER_STATUS(status, INVALID_INPUTS, "the input tensor is null"); - return status; - } - if (tensor.size() != static_cast(request_tensor.ElementNum())) { - status = INFER_STATUS(INVALID_INPUTS) << "input " << i << " element count not match, model required " - << request_tensor.ElementNum() << ", given " << tensor.size(); - MSI_LOG_ERROR << status.StatusMessage(); - return status; - } - status = GetDataFromJson(tensor, &request_tensor, 0, type); - if (status != SUCCESS) { - return status; - } - } - return SUCCESS; -} - -Status TransTensorToPredictRequest(const json &message_info, PredictRequest *const request) { - Status status(SUCCESS); - auto tensors = message_info.find(HTTP_TENSOR); - if (tensors == message_info.end()) { - ERROR_INFER_STATUS(status, INVALID_INPUTS, "http message do not have tensor type"); - return status; - } - if (!tensors->is_array()) { - ERROR_INFER_STATUS(status, INVALID_INPUTS, "the input tensor list is not array"); - return status; - } - if (tensors->size() != static_cast(request->data_size())) { - status = - INFER_STATUS(INVALID_INPUTS) - << "model input count not match or json tensor request is constructed illegally, model input count required " - << request->data_size() << ", given " << tensors->size(); - MSI_LOG_ERROR << status.StatusMessage(); - return status; - } - - for (size_t i = 0; i < tensors->size(); i++) { - const auto &tensor = tensors->at(i); - ServingTensor request_tensor(*(request->mutable_data(i))); - - auto iter = infer_type2_http_type.find(request_tensor.data_type()); - if (iter == infer_type2_http_type.end()) { - ERROR_INFER_STATUS(status, FAILED, "the model input type is not supported right now"); - return status; - } - HTTP_DATA_TYPE type = iter->second; - // check data shape - auto const &json_shape = GetJsonArrayShape(tensor); - auto is_scalar_shape = [](const std::vector &shape) { - return shape.empty() || (shape.size() == 1 && shape[0] == 1); - }; - if (is_scalar_shape(request_tensor.shape())) { - return GetScalarDataFromJson(tensor, &request_tensor, type); - } else { - if (json_shape != request_tensor.shape()) { // data shape not match - status = INFER_STATUS(INVALID_INPUTS) << "input " << i << " shape is invalid, expected " - << request_tensor.shape() << ", given " << json_shape; - MSI_LOG_ERROR << status.StatusMessage(); - return status; - } - size_t depth = 0; - size_t data_index = 0; - status = RecusiveGetTensor(tensor, depth, &request_tensor, data_index, type); - if (status != SUCCESS) { - MSI_LOG_ERROR << "Transfer tensor to predict request failed"; - return status; - } - } - } - return status; -} - -Status TransHTTPMsgToPredictRequest(struct evhttp_request *const http_request, PredictRequest *const request, - HTTP_TYPE *const type) { - Status status = CheckRequestValid(http_request); - if (status != SUCCESS) { - return status; - } - std::string post_message; - status = GetPostMessage(http_request, &post_message); - if (status != SUCCESS) { - return status; - } - - // get model required shape - std::vector tensor_list; - status = Session::Instance().GetModelInputsInfo(tensor_list); - if (status != SUCCESS) { - ERROR_INFER_STATUS(status, FAILED, "get model inputs info failed"); - return status; - } - for (auto &item : tensor_list) { - auto input = request->add_data(); - ServingTensor tensor(*input); - tensor.set_shape(item.shape()); - tensor.set_data_type(item.data_type()); - int64_t element_num = tensor.ElementNum(); - int64_t data_type_size = tensor.GetTypeSize(tensor.data_type()); - if (element_num <= 0 || INT64_MAX / element_num < data_type_size) { - ERROR_INFER_STATUS(status, FAILED, "model shape invalid"); - return status; - } - tensor.resize_data(element_num * data_type_size); - } - MSI_TIME_STAMP_START(ParseJson) - json message_info; - try { - message_info = nlohmann::json::parse(post_message); - } catch (nlohmann::json::exception &e) { - std::string json_exception = e.what(); - std::string error_message = "Illegal JSON format." + json_exception; - ERROR_INFER_STATUS(status, INVALID_INPUTS, error_message); - return status; - } - MSI_TIME_STAMP_END(ParseJson) - - status = CheckMessageValid(message_info, type); - if (status != SUCCESS) { - return status; - } - switch (*type) { - case TYPE_DATA: - status = TransDataToPredictRequest(message_info, request); - break; - case TYPE_TENSOR: - status = TransTensorToPredictRequest(message_info, request); - break; - default: - ERROR_INFER_STATUS(status, INVALID_INPUTS, "http message must have only one type of (data, tensor)"); - return status; - } - return status; -} - -Status GetJsonFromTensor(const ms_serving::Tensor &tensor, int len, int *const pos, json *const out_json) { - Status status(SUCCESS); - switch (tensor.tensor_type()) { - case ms_serving::MS_INT32: { - auto data = reinterpret_cast(tensor.data().data()) + *pos; - std::vector result_tensor(len); - memcpy_s(result_tensor.data(), result_tensor.size() * sizeof(int32_t), data, len * sizeof(int32_t)); - *out_json = std::move(result_tensor); - *pos += len; - break; - } - case ms_serving::MS_FLOAT32: { - auto data = reinterpret_cast(tensor.data().data()) + *pos; - std::vector result_tensor(len); - (void)memcpy_s(result_tensor.data(), result_tensor.size() * sizeof(float), data, len * sizeof(float)); - *out_json = std::move(result_tensor); - *pos += len; - break; - } - default: - MSI_LOG(ERROR) << "the result type is not supported in restful api, type is " << tensor.tensor_type(); - ERROR_INFER_STATUS(status, FAILED, "reply have unsupported type"); - } - return status; -} - -Status TransPredictReplyToData(const PredictReply &reply, json *const out_json) { - Status status(SUCCESS); - for (int i = 0; i < reply.result_size(); i++) { - (*out_json)["data"].push_back(json()); - json &tensor_json = (*out_json)["data"].back(); - int num = 1; - for (auto j = 0; j < reply.result(i).tensor_shape().dims_size(); j++) { - num *= reply.result(i).tensor_shape().dims(j); - } - int pos = 0; - status = GetJsonFromTensor(reply.result(i), num, &pos, &tensor_json); - if (status != SUCCESS) { - return status; - } - } - return status; -} - -Status RecusiveGetJson(const ms_serving::Tensor &tensor, int depth, int *const pos, json *const out_json) { - Status status(SUCCESS); - if (depth >= 10) { - ERROR_INFER_STATUS(status, FAILED, "result tensor shape dims is larger than 10"); - return status; - } - if (depth == tensor.tensor_shape().dims_size() - 1) { - status = GetJsonFromTensor(tensor, tensor.tensor_shape().dims(depth), pos, out_json); - if (status != SUCCESS) { - return status; - } - } else { - for (int i = 0; i < tensor.tensor_shape().dims(depth); i++) { - out_json->push_back(json()); - json &tensor_json = out_json->back(); - status = RecusiveGetJson(tensor, depth + 1, pos, &tensor_json); - if (status != SUCCESS) { - return status; - } - } - } - return status; -} - -Status TransPredictReplyToTensor(const PredictReply &reply, json *const out_json) { - Status status(SUCCESS); - for (int i = 0; i < reply.result_size(); i++) { - (*out_json)["tensor"].push_back(json()); - json &tensor_json = (*out_json)["tensor"].back(); - int pos = 0; - status = RecusiveGetJson(reply.result(i), 0, &pos, &tensor_json); - if (status != SUCCESS) { - return status; - } - } - return status; -} - -Status TransPredictReplyToHTTPMsg(const PredictReply &reply, const HTTP_TYPE &type, struct evbuffer *const buf) { - Status status(SUCCESS); - json out_json; - switch (type) { - case TYPE_DATA: - status = TransPredictReplyToData(reply, &out_json); - break; - case TYPE_TENSOR: - status = TransPredictReplyToTensor(reply, &out_json); - break; - default: - ERROR_INFER_STATUS(status, FAILED, "http message must have only one type of (data, tensor)"); - return status; - } - - const std::string &out_str = out_json.dump(); - evbuffer_add(buf, out_str.data(), out_str.size()); - return status; -} - -Status HttpHandleMsgDetail(struct evhttp_request *const req, void *const arg, struct evbuffer *const retbuff) { - PredictRequest request; - PredictReply reply; - HTTP_TYPE type; - MSI_TIME_STAMP_START(ParseRequest) - auto status = TransHTTPMsgToPredictRequest(req, &request, &type); - MSI_TIME_STAMP_END(ParseRequest) - if (status != SUCCESS) { - MSI_LOG(ERROR) << "restful trans to request failed"; - return status; - } - MSI_TIME_STAMP_START(Predict) - status = Session::Instance().Predict(request, reply); - MSI_TIME_STAMP_END(Predict) - if (status != SUCCESS) { - MSI_LOG(ERROR) << "restful predict failed"; - return status; - } - MSI_TIME_STAMP_START(CreateReplyJson) - status = TransPredictReplyToHTTPMsg(reply, type, retbuff); - MSI_TIME_STAMP_END(CreateReplyJson) - if (status != SUCCESS) { - MSI_LOG(ERROR) << "restful trans to reply failed"; - return status; - } - return SUCCESS; -} - -void http_handler_msg(struct evhttp_request *const req, void *const arg) { - MSI_TIME_STAMP_START(TotalRestfulPredict) - struct evbuffer *retbuff = evbuffer_new(); - if (retbuff == nullptr) { - MSI_LOG_ERROR << "Create event buffer failed"; - return; - } - auto status = HttpHandleMsgDetail(req, arg, retbuff); - if (status != SUCCESS) { - ErrorMessage(req, status); - evbuffer_free(retbuff); - return; - } - MSI_TIME_STAMP_START(ReplyJson) - evhttp_send_reply(req, HTTP_OK, "Client", retbuff); - MSI_TIME_STAMP_END(ReplyJson) - evbuffer_free(retbuff); - MSI_TIME_STAMP_END(TotalRestfulPredict) -} -} // namespace serving -} // namespace mindspore diff --git a/serving/core/http_process.h b/serving/core/http_process.h deleted file mode 100644 index a4dfc374a7b..00000000000 --- a/serving/core/http_process.h +++ /dev/null @@ -1,29 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MINDSPORE_SERVING_HTTP_PROCESS_H -#define MINDSPORE_SERVING_HTTP_PROCESS_H - -#include -#include -#include -#include - -namespace mindspore { -namespace serving { -void http_handler_msg(struct evhttp_request *req, void *arg); -} // namespace serving -} // namespace mindspore -#endif // MINDSPORE_SERVER_H diff --git a/serving/core/server.cc b/serving/core/server.cc deleted file mode 100644 index 58a6de0952c..00000000000 --- a/serving/core/server.cc +++ /dev/null @@ -1,273 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "core/server.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "include/infer_log.h" -#include "serving/ms_service.grpc.pb.h" -#include "core/util/option_parser.h" -#include "core/version_control/version_controller.h" -#include "core/session.h" -#include "core/serving_tensor.h" -#include "core/http_process.h" - -using ms_serving::MSService; -using ms_serving::PredictReply; -using ms_serving::PredictRequest; - -namespace mindspore { -namespace serving { -namespace { -static const uint32_t uint32max = 0x7FFFFFFF; -std::promise exit_requested; -std::atomic_flag has_exited = ATOMIC_FLAG_INIT; - -static const char kServerHttpIp[] = "0.0.0.0"; - -void ClearEnv() { Session::Instance().Clear(); } -void HandleSignal(int sig) { - if (!has_exited.test_and_set()) { - exit_requested.set_value(); - } -} - -grpc::Status CreatGRPCStatus(const Status &status) { - switch (status.StatusCode()) { - case SUCCESS: - return grpc::Status::OK; - case FAILED: - return grpc::Status::CANCELLED; - case INVALID_INPUTS: { - auto status_msg = status.StatusMessage(); - if (status_msg.empty()) { - status_msg = "The Predict Inputs do not match the Model Request!"; - } - return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, status_msg); - } - default: - return grpc::Status::CANCELLED; - } -} -} // namespace - -// Service Implement -class MSServiceImpl final : public MSService::Service { - grpc::Status Predict(grpc::ServerContext *context, const PredictRequest *request, PredictReply *reply) override { - std::lock_guard lock(mutex_); - MSI_TIME_STAMP_START(Predict) - auto res = Session::Instance().Predict(*request, *reply); - MSI_TIME_STAMP_END(Predict) - if (res != inference::SUCCESS) { - return CreatGRPCStatus(res); - } - MSI_LOG(INFO) << "Finish call service Eval"; - return grpc::Status::OK; - } - - grpc::Status Test(grpc::ServerContext *context, const PredictRequest *request, PredictReply *reply) override { - MSI_LOG(INFO) << "TestService call"; - return grpc::Status::OK; - } - std::mutex mutex_; -}; - -static std::pair NewHttpServer() { - auto option_args = Options::Instance().GetArgs(); - int32_t http_port = option_args->rest_api_port; - // init http server - event_init(); - evthread_use_pthreads(); - struct event_base *eb = event_base_new(); - if (eb == nullptr) { - MSI_LOG(ERROR) << "Serving Error: RESTful server start failed, new http event failed"; - std::cout << "Serving Error: RESTful server start failed, new http event failed" << std::endl; - return std::make_pair(nullptr, nullptr); - } - struct evhttp *http_server = evhttp_new(eb); - if (http_server == nullptr) { - MSI_LOG(ERROR) << "Serving Error: RESTful server start failed, create http server faild"; - std::cout << "Serving Error: RESTful server start failed, create http server faild" << std::endl; - event_base_free(eb); - return std::make_pair(nullptr, nullptr); - } - - struct sockaddr_in sin = {}; - sin.sin_family = AF_INET; - sin.sin_port = htons(http_port); - auto listener = - evconnlistener_new_bind(eb, nullptr, nullptr, LEV_OPT_REUSEABLE | LEV_OPT_CLOSE_ON_EXEC | LEV_OPT_CLOSE_ON_FREE, -1, - reinterpret_cast(&sin), sizeof(sin)); - if (listener == nullptr) { - MSI_LOG_ERROR << "Serving Error: RESTful server start failed, create http listener faild, port " << http_port; - std::cout << "Serving Error: RESTful server start failed, create http listener faild, port " << http_port - << std::endl; - event_base_free(eb); - evhttp_free(http_server); - return std::make_pair(nullptr, nullptr); - } - auto bound = evhttp_bind_listener(http_server, listener); - if (bound == nullptr) { - MSI_LOG_ERROR << "Serving Error: RESTful server start failed, bind http listener to server faild, port " - << http_port; - std::cout << "Serving Error: RESTful server start failed, bind http listener to server faild, port " << http_port - << std::endl; - evconnlistener_free(listener); - event_base_free(eb); - evhttp_free(http_server); - return std::make_pair(nullptr, nullptr); - } - return std::make_pair(http_server, eb); -} - -Status BuildAndStartModelInner() { - Status res; - auto option_args = Options::Instance().GetArgs(); - std::string model_path = option_args->model_path; - std::string model_name = option_args->model_name; - std::string device_type = option_args->device_type; - auto device_id = option_args->device_id; - res = Session::Instance().CreatDeviceSession(device_type, device_id); - if (res != SUCCESS) { - MSI_LOG(ERROR) << "Serving Error: create inference session failed, device type " << device_type << " device id " - << device_id; - std::cout << "Serving Error: create inference session failed, device type " << device_type << " device id " - << device_id << std::endl; - return res; - } - VersionController version_controller(option_args->poll_model_wait_seconds, model_path, model_name); - res = version_controller.Run(); - if (res != SUCCESS) { - MSI_LOG(ERROR) << "Serving Error: load model failed, model directory " << option_args->model_path << " model name " - << option_args->model_name; - std::cout << "Serving Error: load model failed, model directory " << option_args->model_path << " model name " - << option_args->model_name << std::endl; - return res; - } - return SUCCESS; -} - -Status BuildAndStartModel() { - try { - auto status = BuildAndStartModelInner(); - return status; - } catch (const std::bad_alloc &ex) { - MSI_LOG(ERROR) << "Serving Error: malloc memory failed"; - std::cout << "Serving Error: malloc memory failed" << std::endl; - } catch (const std::runtime_error &ex) { - MSI_LOG(ERROR) << "Serving Error: runtime error occurred: " << ex.what(); - std::cout << "Serving Error: runtime error occurred: " << ex.what() << std::endl; - } catch (const std::exception &ex) { - MSI_LOG(ERROR) << "Serving Error: exception occurred: " << ex.what(); - std::cout << "Serving Error: exception occurred: " << ex.what() << std::endl; - } catch (...) { - MSI_LOG(ERROR) << "Serving Error: exception occurred"; - std::cout << "Serving Error: exception occurred"; - } - return FAILED; -} - -Status Server::BuildAndStart() { - // handle exit signal - signal(SIGINT, HandleSignal); - signal(SIGTERM, HandleSignal); - Status res = BuildAndStartModel(); - if (res != SUCCESS) { - ClearEnv(); - return res; - } - auto option_args = Options::Instance().GetArgs(); - std::string server_address = std::string(kServerHttpIp) + ":" + std::to_string(option_args->grpc_port); - - auto http_server_new_ret = NewHttpServer(); - struct evhttp *http_server = http_server_new_ret.first; - struct event_base *eb = http_server_new_ret.second; - if (http_server == nullptr || eb == nullptr) { - MSI_LOG(ERROR) << "Serving Error: RESTful server start failed"; - std::cout << "Serving Error: RESTful server start failed" << std::endl; - ClearEnv(); - return FAILED; - } - auto exit_http = [eb, http_server]() { - evhttp_free(http_server); - event_base_free(eb); - }; - int32_t http_port = option_args->rest_api_port; - std::string http_addr = kServerHttpIp; - - evhttp_set_timeout(http_server, 60); - evhttp_set_gencb(http_server, http_handler_msg, nullptr); - - // grpc server - MSServiceImpl ms_service; - grpc::EnableDefaultHealthCheckService(true); - grpc::reflection::InitProtoReflectionServerBuilderPlugin(); - // Set the port is not reuseable - auto option = grpc::MakeChannelArgumentOption(GRPC_ARG_ALLOW_REUSEPORT, 0); - grpc::ServerBuilder serverBuilder; - serverBuilder.SetOption(std::move(option)); - serverBuilder.SetMaxMessageSize(uint32max); - serverBuilder.AddListeningPort(server_address, grpc::InsecureServerCredentials()); - serverBuilder.RegisterService(&ms_service); - std::unique_ptr server(serverBuilder.BuildAndStart()); - if (server == nullptr) { - MSI_LOG(ERROR) << "Serving Error: create server failed, gRPC address " << server_address << ", RESTful address " - << http_addr << ":" << http_port << ", model directory " << option_args->model_path << " model name " - << option_args->model_name << ", device type " << option_args->device_type << ", device id " - << option_args->device_id; - std::cout << "Serving Error: create server failed, gRPC address " << server_address << ", RESTful address " - << http_addr << ":" << http_port << ", model directory " << option_args->model_path << " model name " - << option_args->model_name << ", device type " << option_args->device_type << ", device id " - << option_args->device_id << std::endl; - ClearEnv(); - exit_http(); - return FAILED; - } - auto grpc_server_run = [&server, &server_address]() { - MSI_LOG(INFO) << "MS Serving grpc listening on " << server_address; - std::cout << "Serving: MS Serving gRPC start success, listening on " << server_address << std::endl; - server->Wait(); - }; - auto http_server_run = [&eb, &http_addr, &http_port]() { - MSI_LOG(INFO) << "MS Serving restful listening on " << http_addr << ":" << http_port; - std::cout << "Serving: MS Serving RESTful start success, listening on " << http_addr << ":" << http_port - << std::endl; - event_base_dispatch(eb); - }; - std::thread grpc_thread(grpc_server_run); - std::thread restful_thread(http_server_run); - auto exit_future = exit_requested.get_future(); - exit_future.wait(); - ClearEnv(); - server->Shutdown(); - event_base_loopexit(eb, nullptr); - grpc_thread.join(); - restful_thread.join(); - exit_http(); - return SUCCESS; -} -} // namespace serving -} // namespace mindspore diff --git a/serving/core/server.h b/serving/core/server.h deleted file mode 100644 index 8c9ac046d85..00000000000 --- a/serving/core/server.h +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MINDSPORE_SERVER_H -#define MINDSPORE_SERVER_H - -#include "util/status.h" -namespace mindspore { -namespace serving { -class Server { - public: - Server() = default; - ~Server() = default; - Status BuildAndStart(); -}; -} // namespace serving -} // namespace mindspore -#endif // MINDSPORE_SERVER_H diff --git a/serving/core/serving_tensor.cc b/serving/core/serving_tensor.cc deleted file mode 100644 index 3fc1871ee4d..00000000000 --- a/serving/core/serving_tensor.cc +++ /dev/null @@ -1,194 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "core/serving_tensor.h" -#include -#include -#include -#include -#include "include/infer_log.h" - -using std::string; -using std::unordered_map; -using std::vector; - -namespace mindspore { -namespace serving { -using inference::DataType; -using inference::InferTensorBase; - -const size_t kMaxShapeElementCount = INT32_MAX; -const size_t kMaxDataBufferSize = UINT32_MAX; - -ServingTensor::ServingTensor(ms_serving::Tensor &other) : tensor_(other) {} - -ServingTensor::~ServingTensor() {} - -DataType ServingTensor::data_type() const { - const std::unordered_map type2id_map{ - {ms_serving::MS_UNKNOWN, inference::kMSI_Unknown}, {ms_serving::MS_BOOL, inference::kMSI_Bool}, - {ms_serving::MS_INT8, inference::kMSI_Int8}, {ms_serving::MS_UINT8, inference::kMSI_Uint8}, - {ms_serving::MS_INT16, inference::kMSI_Int16}, {ms_serving::MS_UINT16, inference::kMSI_Uint16}, - {ms_serving::MS_INT32, inference::kMSI_Int32}, {ms_serving::MS_UINT32, inference::kMSI_Uint32}, - {ms_serving::MS_INT64, inference::kMSI_Int64}, {ms_serving::MS_UINT64, inference::kMSI_Uint64}, - {ms_serving::MS_FLOAT16, inference::kMSI_Float16}, {ms_serving::MS_FLOAT32, inference::kMSI_Float32}, - {ms_serving::MS_FLOAT64, inference::kMSI_Float64}, - }; - auto it = type2id_map.find(tensor_.tensor_type()); - if (it == type2id_map.end()) { - MSI_LOG_WARNING << "failed to get data type, undefined data type " << tensor_.tensor_type(); - return inference::kMSI_Unknown; - } else { - return it->second; - } -} - -void ServingTensor::set_data_type(DataType data_type) { - const std::unordered_map id2type_map{ - {inference::kMSI_Unknown, ms_serving::MS_UNKNOWN}, {inference::kMSI_Bool, ms_serving::MS_BOOL}, - {inference::kMSI_Float64, ms_serving::MS_FLOAT64}, {inference::kMSI_Int8, ms_serving::MS_INT8}, - {inference::kMSI_Uint8, ms_serving::MS_UINT8}, {inference::kMSI_Int16, ms_serving::MS_INT16}, - {inference::kMSI_Uint16, ms_serving::MS_UINT16}, {inference::kMSI_Int32, ms_serving::MS_INT32}, - {inference::kMSI_Uint32, ms_serving::MS_UINT32}, {inference::kMSI_Int64, ms_serving::MS_INT64}, - {inference::kMSI_Uint64, ms_serving::MS_UINT64}, {inference::kMSI_Float16, ms_serving::MS_FLOAT16}, - {inference::kMSI_Float32, ms_serving::MS_FLOAT32}, - }; - auto it = id2type_map.find(data_type); - if (it == id2type_map.end()) { - MSI_LOG_WARNING << "failed to set data type, undefined data type " << data_type; - tensor_.set_tensor_type(ms_serving::MS_UNKNOWN); - } else { - tensor_.set_tensor_type(it->second); - } -} - -std::vector ServingTensor::shape() const { - std::vector result; - auto dims = tensor_.tensor_shape().dims(); - std::transform(dims.begin(), dims.end(), std::back_inserter(result), [](const int64_t dim) { return dim; }); - return result; -} - -void ServingTensor::set_shape(const std::vector &shape) { - auto tensor_shape = tensor_.mutable_tensor_shape(); - tensor_shape->Clear(); - size_t element_count = 1; - for (auto dim : shape) { - if (dim <= 0 || element_count > kMaxShapeElementCount / dim) { - MSI_LOG_ERROR << "failed to set shape, invalid dim num " << dim; - tensor_shape->Clear(); - return; - } - element_count *= dim; - tensor_shape->add_dims(dim); - } -} - -bool ServingTensor::resize_data(size_t data_len) { - string *buffer = tensor_.mutable_data(); - if (buffer == nullptr) { - MSI_LOG_ERROR << "invalid buffer data"; - return false; - } - buffer->resize(data_len); - return true; -} - -size_t ServingTensor::data_size() const { return tensor_.data().size(); } - -void *ServingTensor::mutable_data() { return const_cast(tensor_.mutable_data()->data()); } - -const void *ServingTensor::data() const { return tensor_.data().data(); } - -ServingRequest::ServingRequest(const ms_serving::PredictRequest &request) : request_(request) { - auto &data = request_.data(); - std::transform(data.begin(), data.end(), std::back_inserter(cache_), - [](const ms_serving::Tensor &item) { return ServingTensor(const_cast(item)); }); -} - -size_t ServingRequest::size() const { return cache_.size(); } - -const InferTensorBase *ServingRequest::operator[](size_t index) const { - if (index >= cache_.size()) { - MSI_LOG_ERROR << "visit invalid index " << index << " total size " << cache_.size(); - return nullptr; - } - return &(cache_[index]); -} - -ServingImages::ServingImages(const ms_serving::Images &images) : images_(images) {} - -size_t ServingImages::batch_size() const { return images_.images_size(); } - -bool ServingImages::get(size_t index, const void *&pic_buffer, uint32_t &pic_size) const { - if (index >= static_cast(images_.images_size())) { - MSI_LOG_ERROR << "visit invalid index " << index << " total size " << images_.images_size(); - return false; - } - pic_buffer = images_.images(index).data(); - pic_size = images_.images(index).size(); - return true; -} - -size_t ServingImages::input_index() const { return static_cast(images_.input_index()); } - -size_t ServingReply::size() const { return cache_.size(); } - -InferTensorBase *ServingReply::operator[](size_t index) { - if (index >= cache_.size()) { - MSI_LOG_ERROR << "visit invalid index " << index << " total size " << cache_.size(); - return nullptr; - } - return &(cache_[index]); -} - -const InferTensorBase *ServingReply::operator[](size_t index) const { - if (index >= cache_.size()) { - MSI_LOG_ERROR << "visit invalid index " << index << " total size " << cache_.size(); - return nullptr; - } - return &(cache_[index]); -} - -InferTensorBase *ServingReply::add() { - auto new_item = reply_.add_result(); - if (new_item == nullptr) { - MSI_LOG_ERROR << "add new item failed, current total size " << cache_.size(); - return nullptr; - } - cache_.push_back(ServingTensor(*new_item)); - return &(cache_.back()); -} - -void ServingReply::clear() { reply_.mutable_result()->Clear(); } - -ServingImagesRequest::ServingImagesRequest(const ms_serving::PredictRequest &request) : request_(request) { - auto &images_inputs = request_.images(); - std::transform(images_inputs.begin(), images_inputs.end(), std::back_inserter(cache_), - [](const ms_serving::Images &item) { return ServingImages(const_cast(item)); }); -} - -size_t ServingImagesRequest::size() const { return cache_.size(); } - -const inference::InferImagesBase *ServingImagesRequest::operator[](size_t index) const { - if (index >= cache_.size()) { - MSI_LOG_ERROR << "visit invalid index " << index << " total size " << cache_.size(); - return nullptr; - } - return &(cache_[index]); -} -} // namespace serving -} // namespace mindspore diff --git a/serving/core/serving_tensor.h b/serving/core/serving_tensor.h deleted file mode 100644 index 002f2dbbe4a..00000000000 --- a/serving/core/serving_tensor.h +++ /dev/null @@ -1,105 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef MINDSPORE_SERVING_TENSOR_H_ -#define MINDSPORE_SERVING_TENSOR_H_ - -#include -#include -#include -#include "include/infer_tensor.h" -#include "serving/ms_service.pb.h" - -namespace mindspore { -namespace serving { -class MS_API ServingTensor : public inference::InferTensorBase { - public: - // the other's lifetime must longer than this object - explicit ServingTensor(ms_serving::Tensor &other); - ~ServingTensor(); - - inference::DataType data_type() const override; - void set_data_type(inference::DataType type) override; - std::vector shape() const override; - void set_shape(const std::vector &shape) override; - const void *data() const override; - size_t data_size() const override; - bool resize_data(size_t data_len) override; - void *mutable_data() override; - - private: - // if tensor_ is reference from other ms_serving::Tensor, the other's lifetime must - // longer than this object - ms_serving::Tensor &tensor_; -}; - -class ServingImages : public inference::InferImagesBase { - public: - explicit ServingImages(const ms_serving::Images &images); - ~ServingImages() = default; - - size_t batch_size() const override; - bool get(size_t index, const void *&pic_buffer, uint32_t &pic_size) const override; - size_t input_index() const override; - - private: - const ms_serving::Images &images_; -}; - -class ServingRequest : public inference::RequestBase { - public: - explicit ServingRequest(const ms_serving::PredictRequest &request); - ~ServingRequest() = default; - - size_t size() const override; - const inference::InferTensorBase *operator[](size_t index) const override; - - private: - const ms_serving::PredictRequest &request_; - std::vector cache_; -}; - -class ServingReply : public inference::ReplyBase { - public: - explicit ServingReply(ms_serving::PredictReply &reply) : reply_(reply) {} - ~ServingReply() = default; - - size_t size() const override; - inference::InferTensorBase *operator[](size_t index) override; - const inference::InferTensorBase *operator[](size_t index) const override; - inference::InferTensorBase *add() override; - void clear() override; - - private: - ms_serving::PredictReply &reply_; - std::vector cache_; -}; - -class ServingImagesRequest : public inference::ImagesRequestBase { - public: - explicit ServingImagesRequest(const ms_serving::PredictRequest &request); - ~ServingImagesRequest() = default; - - size_t size() const override; - const inference::InferImagesBase *operator[](size_t index) const override; - - private: - const ms_serving::PredictRequest &request_; - std::vector cache_; -}; -} // namespace serving -} // namespace mindspore -#endif // MINDSPORE_SERVING_TENSOR_H_ diff --git a/serving/core/session.cc b/serving/core/session.cc deleted file mode 100644 index fb5d11cc450..00000000000 --- a/serving/core/session.cc +++ /dev/null @@ -1,154 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "core/session.h" -#include -#include -#include -#include -#include -#include -#include - -#include "include/infer_log.h" -#include "serving/ms_service.grpc.pb.h" -#include "core/util/option_parser.h" -#include "core/version_control/version_controller.h" -#include "core/util/file_system_operation.h" -#include "core/serving_tensor.h" - -using ms_serving::MSService; -using ms_serving::PredictReply; -using ms_serving::PredictRequest; - -namespace mindspore { -namespace serving { -Status Session::CreatDeviceSession(const std::string &device, uint32_t device_id) { - session_ = inference::InferSession::CreateSession(device, device_id); - if (session_ == nullptr) { - MSI_LOG(ERROR) << "Creat Session Failed"; - return FAILED; - } - device_type_ = device; - return SUCCESS; -} - -Session &Session::Instance() { - static Session instance; - return instance; -} - -Status Session::Predict(const PredictRequest &request, PredictReply &reply) { - try { - auto status = PredictInner(request, reply); - return status; - } catch (const std::bad_alloc &ex) { - MSI_LOG(ERROR) << "Serving Error: malloc memory failed"; - std::cout << "Serving Error: malloc memory failed" << std::endl; - } catch (const std::runtime_error &ex) { - MSI_LOG(ERROR) << "Serving Error: runtime error occurred: " << ex.what(); - std::cout << "Serving Error: runtime error occurred: " << ex.what() << std::endl; - } catch (const std::exception &ex) { - MSI_LOG(ERROR) << "Serving Error: exception occurred: " << ex.what(); - std::cout << "Serving Error: exception occurred: " << ex.what() << std::endl; - } catch (...) { - MSI_LOG(ERROR) << "Serving Error: exception occurred"; - std::cout << "Serving Error: exception occurred"; - } - return FAILED; -} - -Status Session::PredictInner(const PredictRequest &request, PredictReply &reply) { - if (!model_loaded_) { - MSI_LOG(ERROR) << "the model has not loaded"; - return FAILED; - } - if (session_ == nullptr) { - MSI_LOG(ERROR) << "the inference session has not be initialized"; - return FAILED; - } - std::lock_guard lock(mutex_); - MSI_LOG(INFO) << "run Predict"; - - if (request.images_size() > 0) { - ServingImagesRequest serving_images(request); - ServingRequest serving_request(request); - ServingReply serving_reply(reply); - Status ret = session_->ExecuteModel(graph_id_, serving_images, serving_request, serving_reply); - if (ret != SUCCESS) { - MSI_LOG(ERROR) << "execute model with images return failed"; - return ret; - } - } else if (request.data_size() > 0) { - ServingRequest serving_request(request); - ServingReply serving_reply(reply); - Status ret = session_->ExecuteModel(graph_id_, serving_request, serving_reply); - if (ret != SUCCESS) { - MSI_LOG(ERROR) << "execute model with datas return failed"; - return ret; - } - } - - MSI_LOG(INFO) << "run Predict finished"; - return SUCCESS; -} - -Status Session::Warmup(const MindSporeModelPtr model) { - if (session_ == nullptr) { - MSI_LOG(ERROR) << "The CreatDeviceSession should be called, before warmup"; - return FAILED; - } - std::lock_guard lock(mutex_); - std::string file_name = model->GetModelPath() + '/' + model->GetModelName(); - model_loaded_ = false; - MSI_TIME_STAMP_START(LoadModelFromFile) - auto ret = session_->LoadModelFromFile(file_name, graph_id_); - MSI_TIME_STAMP_END(LoadModelFromFile) - if (ret != SUCCESS) { - MSI_LOG(ERROR) << "Load graph model failed, file name is " << file_name.c_str(); - return ret; - } - model_loaded_ = true; - MSI_LOG(INFO) << "Session Warmup finished"; - return SUCCESS; -} - -Status Session::Clear() { - if (session_ != nullptr) { - session_->UnloadModel(graph_id_); - session_->FinalizeEnv(); - session_ = nullptr; - } - return SUCCESS; -} - -Status Session::GetModelInputsInfo(std::vector &tensor_list) { - if (!model_loaded_) { - MSI_LOG(ERROR) << "the model has not loaded"; - return FAILED; - } - if (session_ == nullptr) { - MSI_LOG(ERROR) << "the inference session has not be initialized"; - return FAILED; - } - std::lock_guard lock(mutex_); - Status ret = session_->GetModelInputsInfo(graph_id_, &tensor_list); - if (ret != SUCCESS) { - MSI_LOG(ERROR) << "get model inputs info failed"; - } - return ret; -} -} // namespace serving -} // namespace mindspore diff --git a/serving/core/session.h b/serving/core/session.h deleted file mode 100644 index fd597b192b9..00000000000 --- a/serving/core/session.h +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MINDSPORE_SERVING_SESSION_H -#define MINDSPORE_SERVING_SESSION_H - -#include -#include -#include -#include -#include "util/status.h" -#include "version_control/model.h" -#include "include/inference.h" -#include "serving/ms_service.pb.h" -#include "serving/ms_service.grpc.pb.h" - -namespace mindspore { -namespace serving { -using inference::FAILED; -using inference::INVALID_INPUTS; -using inference::Status; -using inference::SUCCESS; -using ms_serving::PredictReply; -using ms_serving::PredictRequest; - -class Session { - public: - static Session &Instance(); - Status CreatDeviceSession(const std::string &device, uint32_t device_id); - Status Predict(const PredictRequest &request, PredictReply &reply); - Status Warmup(const MindSporeModelPtr model); - Status Clear(); - Status GetModelInputsInfo(std::vector &tensor_list); - - private: - Session() = default; - ~Session() = default; - int sesseion_id_{0}; - std::shared_ptr session_{nullptr}; - bool model_loaded_ = false; - uint32_t graph_id_{0}; - std::mutex mutex_; - std::string device_type_; - - Status PredictInner(const PredictRequest &request, PredictReply &reply); -}; -} // namespace serving -} // namespace mindspore -#endif // MINDSPORE_SERVER_H diff --git a/serving/core/util/file_system_operation.cc b/serving/core/util/file_system_operation.cc deleted file mode 100644 index e66ccbfb1ae..00000000000 --- a/serving/core/util/file_system_operation.cc +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "core/util/file_system_operation.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "include/infer_log.h" - -namespace mindspore { -namespace serving { -bool DirOrFileExist(const std::string &file_path) { - int ret = access(file_path.c_str(), 0); - return (ret == -1) ? false : true; -} - -std::vector GetAllSubDirs(const std::string &dir_path) { - DIR *dir = nullptr; - struct dirent *ptr = nullptr; - std::vector SubDirs; - - if ((dir = opendir(dir_path.c_str())) == NULL) { - MSI_LOG(ERROR) << "Open " << dir_path << " error!"; - return std::vector(); - } - - while ((ptr = readdir(dir)) != NULL) { - std::string name = ptr->d_name; - if (name == "." || name == "..") { - continue; - } - if (ptr->d_type == DT_DIR) { - SubDirs.push_back(dir_path + "/" + name); - } - } - closedir(dir); - std::sort(SubDirs.begin(), SubDirs.end()); - return SubDirs; -} - -time_t GetModifyTime(const std::string &file_path) { - struct stat info; - (void)stat(file_path.c_str(), &info); - return info.st_mtime; -} -} // namespace serving -} // namespace mindspore diff --git a/serving/core/util/file_system_operation.h b/serving/core/util/file_system_operation.h deleted file mode 100644 index e03883b8121..00000000000 --- a/serving/core/util/file_system_operation.h +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef MINDSPORE_SERVING_FILE_SYSTEM_OPERATION_H_ -#define MINDSPORE_SERVING_FILE_SYSTEM_OPERATION_H_ - -#include -#include -#include - -namespace mindspore { -namespace serving { -char *ReadFile(const char *file, size_t *size); -bool DirOrFileExist(const std::string &file_path); -std::vector GetAllSubDirs(const std::string &dir_path); -time_t GetModifyTime(const std::string &file_path); -} // namespace serving -} // namespace mindspore - -#endif // !MINDSPORE_SERVING_FILE_SYSTEM_OPERATION_H_ diff --git a/serving/core/util/option_parser.cc b/serving/core/util/option_parser.cc deleted file mode 100644 index 25adf3e24c4..00000000000 --- a/serving/core/util/option_parser.cc +++ /dev/null @@ -1,259 +0,0 @@ -/** - * Copyright 2020 Huawei Technologies Co., Ltd - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "core/util/option_parser.h" -#include -#include -#include -#include -#include -#include "include/infer_log.h" - -namespace mindspore { -namespace serving { -bool StartWith(const std::string &str, const std::string &expected) { - return expected.empty() || - (str.size() >= expected.size() && memcmp(str.data(), expected.data(), expected.size()) == 0); -} - -bool RemovePrefix(std::string *const str, const std::string &prefix) { - if (!StartWith(*str, prefix)) return false; - str->replace(str->begin(), str->begin() + prefix.size(), ""); - return true; -} - -bool Option::ParseInt32(std::string *const arg) { - if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) { - int32_t parsed_value; - try { - parsed_value = std::stoi(arg->data()); - } catch (std::invalid_argument) { - std::cout << "Parse " << name_ << " Error for option " << *arg << std::endl; - return false; - } - *int32_default_ = parsed_value; - return true; - } - return false; -} - -bool Option::ParseBool(std::string *const arg) { - if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) { - if (*arg == "true") { - *bool_default_ = true; - } else if (*arg == "false") { - *bool_default_ = false; - } else { - std::cout << "Parse " << name_ << " Error for option " << *arg << std::endl; - return false; - } - return true; - } - - return false; -} - -bool Option::ParseString(std::string *const arg) { - if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) { - *string_default_ = *arg; - return true; - } - return false; -} - -bool Option::ParseFloat(std::string *const arg) { - if (RemovePrefix(arg, "--") && RemovePrefix(arg, name_) && RemovePrefix(arg, "=")) { - float parsed_value; - try { - parsed_value = std::stof(arg->data()); - } catch (std::invalid_argument) { - std::cout << "Parse " << name_ << " Error for option " << *arg << std::endl; - return false; - } - *float_default_ = parsed_value; - return true; - } - return false; -} - -Option::Option(const std::string &name, int32_t *const default_point, const std::string &usage) - : name_(name), - type_(MS_TYPE_INT32), - int32_default_(default_point), - bool_default_(nullptr), - string_default_(nullptr), - float_default_(nullptr), - usage_(usage) {} - -Option::Option(const std::string &name, bool *const default_point, const std::string &usage) - : name_(name), - type_(MS_TYPE_BOOL), - int32_default_(nullptr), - bool_default_(default_point), - string_default_(nullptr), - float_default_(nullptr), - usage_(usage) {} - -Option::Option(const std::string &name, std::string *const default_point, const std::string &usage) - : name_(name), - type_(MS_TYPE_STRING), - int32_default_(nullptr), - bool_default_(nullptr), - string_default_(default_point), - float_default_(nullptr), - usage_(usage) {} - -Option::Option(const std::string &name, float *const default_point, const std::string &usage) - : name_(name), - type_(MS_TYPE_FLOAT), - int32_default_(nullptr), - bool_default_(nullptr), - string_default_(nullptr), - float_default_(default_point), - usage_(usage) {} - -bool Option::Parse(std::string *const arg) { - bool result = false; - switch (type_) { - case MS_TYPE_BOOL: - result = ParseBool(arg); - break; - case MS_TYPE_FLOAT: - result = ParseFloat(arg); - break; - case MS_TYPE_INT32: - result = ParseInt32(arg); - break; - case MS_TYPE_STRING: - result = ParseString(arg); - break; - default: - break; - } - return result; -} - -std::shared_ptr Options::inst_ = nullptr; - -Options &Options::Instance() { - static Options instance; - return instance; -} - -Options::Options() : args_(nullptr) { CreateOptions(); } - -void Options::CreateOptions() { - args_ = std::make_shared(); - std::vector