From c0d251448980e6f53826eac1ed85962c51b444fd Mon Sep 17 00:00:00 2001 From: TinaMengtingZhang Date: Tue, 17 Aug 2021 14:49:12 -0400 Subject: [PATCH] Update grpc from 1.27.3 to 1.36.1 and subdependency (re2 & absl) --- cmake/external_libs/absl.cmake | 14 ++++++++----- cmake/external_libs/grpc.cmake | 19 ++++++++++++----- cmake/external_libs/re2.cmake | 21 +++++++++++++++++++ cmake/mind_expression.cmake | 1 + cmake/package.cmake | 18 ++++++++-------- cmake/package_tar.cmake | 18 ++++++++-------- .../dataset/engine/cache/cache_grpc_client.cc | 4 ++-- .../dataset/engine/cache/cache_grpc_server.cc | 4 ++-- 8 files changed, 67 insertions(+), 32 deletions(-) create mode 100644 cmake/external_libs/re2.cmake diff --git a/cmake/external_libs/absl.cmake b/cmake/external_libs/absl.cmake index 2f110f04ec6..60a174dfc1d 100644 --- a/cmake/external_libs/absl.cmake +++ b/cmake/external_libs/absl.cmake @@ -1,13 +1,17 @@ if(ENABLE_GITEE) - set(REQ_URL "https://gitee.com/mirrors/abseil-cpp/repository/archive/20200225.2.tar.gz") - set(MD5 "7e84ac40ee4541f645f5b9c90c9c98e6") + set(REQ_URL "https://gitee.com/mirrors/abseil-cpp/repository/archive/20200923.3.tar.gz") + set(MD5 "425588c377891eaf368cfad0ea934601") else() - set(REQ_URL "https://github.com/abseil/abseil-cpp/archive/20200225.2.tar.gz") - set(MD5 "73f2b6e72f1599a9139170c29482ddc4") + set(REQ_URL "https://github.com/abseil/abseil-cpp/archive/20200923.3.tar.gz") + set(MD5 "daba6e99c7a84e2242a0107bbd873669") +endif() + +if(NOT ENABLE_GLIBCXX) + set(absl_CXXFLAGS "${absl_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") endif() mindspore_add_pkg(absl - VER 20200225.2 + VER 20200923.3 LIBS absl_strings absl_throw_delegate absl_raw_logging_internal absl_int128 absl_bad_optional_access URL ${REQ_URL} MD5 ${MD5} diff --git a/cmake/external_libs/grpc.cmake b/cmake/external_libs/grpc.cmake index 0740b487ef0..85663c6702d 100644 --- a/cmake/external_libs/grpc.cmake +++ b/cmake/external_libs/grpc.cmake @@ -27,20 +27,27 @@ else() endif() message("grpc using absl_DIR : " ${_FINDPACKAGE_ABSL_CONFIG_DIR}) +if(EXISTS ${re2_ROOT}/lib64) + set(_FINDPACKAGE_RE2_CONFIG_DIR "${re2_ROOT}/lib64/cmake/re2") +else() + set(_FINDPACKAGE_RE2_CONFIG_DIR "${re2_ROOT}/lib/cmake/re2") +endif() +message("grpc using re2_DIR : " ${_FINDPACKAGE_RE2_CONFIG_DIR}) + if(EXISTS ${openssl_ROOT}) set(_CMAKE_ARGS_OPENSSL_ROOT_DIR "-DOPENSSL_ROOT_DIR:PATH=${openssl_ROOT}") endif() if(ENABLE_GITEE) - set(REQ_URL "https://gitee.com/mirrors/grpc/repository/archive/v1.27.3.tar.gz") - set(MD5 "b8b6d8defeda0355105e3b64b4201786") + set(REQ_URL "https://gitee.com/mirrors/grpc/repository/archive/v1.36.1.tar.gz") + set(MD5 "59e85aa3f583a65bf1135e119b75162d") else() - set(REQ_URL "https://github.com/grpc/grpc/archive/v1.27.3.tar.gz") - set(MD5 "0c6c3fc8682d4262dd0e5e6fabe1a7e2") + set(REQ_URL "https://github.com/grpc/grpc/archive/v1.36.1.tar.gz") + set(MD5 "90c93203e95e89af5f46738588217057") endif() mindspore_add_pkg(grpc - VER 1.27.3 + VER 1.36.1 LIBS mindspore_grpc++ mindspore_grpc mindspore_gpr mindspore_upb mindspore_address_sorting EXE grpc_cpp_plugin URL ${REQ_URL} @@ -61,6 +68,8 @@ mindspore_add_pkg(grpc -Dc-ares_DIR:PATH=${c-ares_ROOT}/lib/cmake/c-ares -DgRPC_SSL_PROVIDER:STRING=package ${_CMAKE_ARGS_OPENSSL_ROOT_DIR} + -DgRPC_RE2_PROVIDER:STRING=package + -Dre2_DIR:PATH=${_FINDPACKAGE_RE2_CONFIG_DIR} ) include_directories(${grpc_INC}) diff --git a/cmake/external_libs/re2.cmake b/cmake/external_libs/re2.cmake new file mode 100644 index 00000000000..8f2e36941d3 --- /dev/null +++ b/cmake/external_libs/re2.cmake @@ -0,0 +1,21 @@ +if(ENABLE_GITEE) + set(REQ_URL "https://gitee.com/mirrors/re2/repository/archive/2019-12-01.tar.gz") + set(MD5 "f581b6356d8e8e3debfc230197f91b54") +else() + set(REQ_URL "https://github.com/google/re2/archive/2019-12-01.tar.gz") + set(MD5 "527eab0c75d6a1a0044c6eefd816b2fb") +endif() + +if(NOT ENABLE_GLIBCXX) + set(re2_CXXFLAGS "${re2_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") +endif() + +mindspore_add_pkg(re2 + VER 20191201 + LIBS re2 + URL ${REQ_URL} + MD5 ${MD5} + CMAKE_OPTION -DCMAKE_BUILD_TYPE:STRING=Release -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=TRUE) + +include_directories(${re2_INC}) +add_library(mindspore::re2 ALIAS re2::re2) \ No newline at end of file diff --git a/cmake/mind_expression.cmake b/cmake/mind_expression.cmake index f811e3bfdb6..476f1e0f473 100644 --- a/cmake/mind_expression.cmake +++ b/cmake/mind_expression.cmake @@ -21,6 +21,7 @@ if(MS_BUILD_GRPC) include(${CMAKE_SOURCE_DIR}/cmake/external_libs/absl.cmake) include(${CMAKE_SOURCE_DIR}/cmake/external_libs/c-ares.cmake) include(${CMAKE_SOURCE_DIR}/cmake/external_libs/zlib.cmake) + include(${CMAKE_SOURCE_DIR}/cmake/external_libs/re2.cmake) # build gRPC include(${CMAKE_SOURCE_DIR}/cmake/external_libs/grpc.cmake) # build event diff --git a/cmake/package.cmake b/cmake/package.cmake index e4b1a1663f9..1fb3227d1f9 100644 --- a/cmake/package.cmake +++ b/cmake/package.cmake @@ -235,16 +235,16 @@ if(NOT ENABLE_GE) endif() if(MS_BUILD_GRPC) - install(FILES ${grpc_LIBPATH}/libmindspore_grpc++.so.1.27.3 + install(FILES ${grpc_LIBPATH}/libmindspore_grpc++.so.1.36.1 DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_grpc++.so.1 COMPONENT mindspore) - install(FILES ${grpc_LIBPATH}/libmindspore_grpc.so.9.0.0 - DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_grpc.so.9 COMPONENT mindspore) - install(FILES ${grpc_LIBPATH}/libmindspore_gpr.so.9.0.0 - DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_gpr.so.9 COMPONENT mindspore) - install(FILES ${grpc_LIBPATH}/libmindspore_upb.so.9.0.0 - DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_upb.so.9 COMPONENT mindspore) - install(FILES ${grpc_LIBPATH}/libmindspore_address_sorting.so.9.0.0 - DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_address_sorting.so.9 COMPONENT mindspore) + install(FILES ${grpc_LIBPATH}/libmindspore_grpc.so.15.0.0 + DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_grpc.so.15 COMPONENT mindspore) + install(FILES ${grpc_LIBPATH}/libmindspore_gpr.so.15.0.0 + DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_gpr.so.15 COMPONENT mindspore) + install(FILES ${grpc_LIBPATH}/libmindspore_upb.so.15.0.0 + DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_upb.so.15 COMPONENT mindspore) + install(FILES ${grpc_LIBPATH}/libmindspore_address_sorting.so.15.0.0 + DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_address_sorting.so.15 COMPONENT mindspore) endif() if(CMAKE_SYSTEM_NAME MATCHES "Windows") diff --git a/cmake/package_tar.cmake b/cmake/package_tar.cmake index 1f04942d82e..bf9c1ab4f4a 100644 --- a/cmake/package_tar.cmake +++ b/cmake/package_tar.cmake @@ -123,16 +123,16 @@ if(ENABLE_CPU) endif() if(MS_BUILD_GRPC) - install(FILES ${grpc_LIBPATH}/libmindspore_grpc++.so.1.27.3 + install(FILES ${grpc_LIBPATH}/libmindspore_grpc++.so.1.36.1 DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_grpc++.so.1 COMPONENT mindspore) - install(FILES ${grpc_LIBPATH}/libmindspore_grpc.so.9.0.0 - DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_grpc.so.9 COMPONENT mindspore) - install(FILES ${grpc_LIBPATH}/libmindspore_gpr.so.9.0.0 - DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_gpr.so.9 COMPONENT mindspore) - install(FILES ${grpc_LIBPATH}/libmindspore_upb.so.9.0.0 - DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_upb.so.9 COMPONENT mindspore) - install(FILES ${grpc_LIBPATH}/libmindspore_address_sorting.so.9.0.0 - DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_address_sorting.so.9 COMPONENT mindspore) + install(FILES ${grpc_LIBPATH}/libmindspore_grpc.so.15.0.0 + DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_grpc.so.15 COMPONENT mindspore) + install(FILES ${grpc_LIBPATH}/libmindspore_gpr.so.15.0.0 + DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_gpr.so.15 COMPONENT mindspore) + install(FILES ${grpc_LIBPATH}/libmindspore_upb.so.15.0.0 + DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_upb.so.15 COMPONENT mindspore) + install(FILES ${grpc_LIBPATH}/libmindspore_address_sorting.so.15.0.0 + DESTINATION ${INSTALL_LIB_DIR} RENAME libmindspore_address_sorting.so.15 COMPONENT mindspore) endif() ## Public header files diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_client.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_client.cc index 1c91a6f89a7..125f7ec1889 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_client.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_client.cc @@ -111,7 +111,7 @@ Status CacheClientGreeter::WorkerEntry() { auto deadline = std::chrono::system_clock::now() + std::chrono::seconds(kWaitForNewEventDeadlineInSec); // Set a timeout for one second. Check for interrupt if we need to do early exit. auto r = cq_.AsyncNext(&tag, &success, deadline); - if (r == grpc_impl::CompletionQueue::NextStatus::GOT_EVENT) { + if (r == grpc::CompletionQueue::NextStatus::GOT_EVENT) { auto rq = reinterpret_cast(tag); if (success) { auto &rc = rq->rc_; @@ -137,7 +137,7 @@ Status CacheClientGreeter::WorkerEntry() { auto n = req_.erase(seq_no); CHECK_FAIL_RETURN_UNEXPECTED(n == 1, "Sequence " + std::to_string(seq_no) + " not found"); } - } else if (r == grpc_impl::CompletionQueue::NextStatus::TIMEOUT) { + } else if (r == grpc::CompletionQueue::NextStatus::TIMEOUT) { // If we are interrupted, exit. Otherwise wait again. RETURN_IF_INTERRUPTED(); } else { diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_server.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_server.cc index c7272d348ad..6935c165bd9 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_server.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_grpc_server.cc @@ -97,7 +97,7 @@ Status CacheServerGreeterImpl::HandleRequest(int32_t worker_id) { auto deadline = std::chrono::system_clock::now() + std::chrono::seconds(1); // Set a timeout for one second. Check for interrupt if we need to do early exit. auto r = cq_->AsyncNext(&tag, &success, deadline); - if (r == grpc_impl::CompletionQueue::NextStatus::GOT_EVENT) { + if (r == grpc::CompletionQueue::NextStatus::GOT_EVENT) { auto rq = static_cast(tag); if (success) { if (rq->st_ == CacheServerRequest::STATE::PROCESS) { @@ -115,7 +115,7 @@ Status CacheServerGreeterImpl::HandleRequest(int32_t worker_id) { } else { RETURN_IF_NOT_OK(CacheServer::ReturnRequestTag(rq)); } - } else if (r == grpc_impl::CompletionQueue::NextStatus::TIMEOUT) { + } else if (r == grpc::CompletionQueue::NextStatus::TIMEOUT) { // If we are interrupted, exit. Otherwise wait again. } else { // Queue is drained.