From 0cb6adce9ecc68e2d695d973455f54d3f59632a3 Mon Sep 17 00:00:00 2001 From: yepei6 Date: Sat, 20 Mar 2021 19:56:45 +0800 Subject: [PATCH] add ENABLE_GLIBCXX param --- CMakeLists.txt | 4 +- cmake/external_libs/glog.cmake | 5 +- cmake/external_libs/grpc.cmake | 15 +-- cmake/external_libs/gtest.cmake | 4 +- cmake/external_libs/opencv.cmake | 5 +- cmake/external_libs/protobuf.cmake | 17 ++-- cmake/external_libs/sentencepiece.cmake | 30 ++++-- cmake/options.cmake | 5 + tests/ut/cpp/CMakeLists.txt | 3 + .../sentencepiece/sentencepiece.patch001_cpu | 91 +++++++++++++++++++ 10 files changed, 153 insertions(+), 26 deletions(-) create mode 100644 third_party/patch/sentencepiece/sentencepiece.patch001_cpu diff --git a/CMakeLists.txt b/CMakeLists.txt index 33f4ef35a83..e7ab0792977 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,9 @@ include(${CMAKE_SOURCE_DIR}/cmake/options.cmake) include(${CMAKE_SOURCE_DIR}/cmake/check_requirements.cmake) set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake/modules/") if(NOT CMAKE_SYSTEM_NAME MATCHES "Windows") - add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=0) + if(NOT ENABLE_GLIBCXX) + add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=0) + endif() endif() if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") diff --git a/cmake/external_libs/glog.cmake b/cmake/external_libs/glog.cmake index 5a0e939a87a..a7c713b3d14 100644 --- a/cmake/external_libs/glog.cmake +++ b/cmake/external_libs/glog.cmake @@ -1,4 +1,7 @@ -set(glog_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2 ${SECURE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") +set(glog_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2 ${SECURE_CXX_FLAGS}") +if(NOT ENABLE_GLIBCXX) + set(glog_CXXFLAGS "${glog_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") +endif() set(glog_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") if(ENABLE_GITEE) set(REQ_URL "https://gitee.com/mirrors/glog/repository/archive/v0.4.0.tar.gz") diff --git a/cmake/external_libs/grpc.cmake b/cmake/external_libs/grpc.cmake index 1a7bc88fb69..dff52d8c4a3 100644 --- a/cmake/external_libs/grpc.cmake +++ b/cmake/external_libs/grpc.cmake @@ -1,13 +1,16 @@ set(grpc_USE_STATIC_LIBS ON) if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(grpc_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden \ - -D_FORTIFY_SOURCE=2 -O2") + set(grpc_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC \ + -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2") elseif(${CMAKE_SYSTEM_NAME} MATCHES "Windows") - set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden \ - -D_FORTIFY_SOURCE=2 -O2") + set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter \ + -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2") else() - set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC -fvisibility=hidden \ - -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2") + set(grpc_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter \ + -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2") + if(NOT ENABLE_GLIBCXX) + set(grpc_CXXFLAGS "${grpc_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") + endif() endif() set(grpc_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") diff --git a/cmake/external_libs/gtest.cmake b/cmake/external_libs/gtest.cmake index 700da26a3c4..8de4989a243 100644 --- a/cmake/external_libs/gtest.cmake +++ b/cmake/external_libs/gtest.cmake @@ -24,7 +24,9 @@ if(BUILD_LITE) ${CMAKE_OPTION}) endif() else() - set(gtest_CXXFLAGS "${gtest_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") + if(NOT ENABLE_GLIBCXX) + set(gtest_CXXFLAGS "${gtest_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") + endif() endif() if(ENABLE_GITEE) diff --git a/cmake/external_libs/opencv.cmake b/cmake/external_libs/opencv.cmake index a0273ee28ac..cd5784713b8 100644 --- a/cmake/external_libs/opencv.cmake +++ b/cmake/external_libs/opencv.cmake @@ -9,7 +9,10 @@ elseif(${CMAKE_SYSTEM_NAME} MATCHES "Windows") set(opencv_CXXFLAGS "${opencv_CXXFLAGS} -Wno-unused-value -Wno-implicit-fallthrough") else() set(opencv_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2") - set(opencv_CXXFLAGS "${opencv_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0 -O2") + set(opencv_CXXFLAGS "${opencv_CXXFLAGS} -O2") + if(NOT ENABLE_GLIBCXX) + set(opencv_CXXFLAGS "${opencv_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") + endif() set(opencv_CFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -D_FORTIFY_SOURCE=2 -O2") set(opencv_LDFLAGS "-Wl,-z,relro,-z,now,-z,noexecstack") endif() diff --git a/cmake/external_libs/protobuf.cmake b/cmake/external_libs/protobuf.cmake index ab240e1d4f6..c7d786bd3c2 100644 --- a/cmake/external_libs/protobuf.cmake +++ b/cmake/external_libs/protobuf.cmake @@ -1,17 +1,20 @@ set(protobuf_USE_STATIC_LIBS ON) if(BUILD_LITE) - set(protobuf_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC \ - -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2") + set(protobuf_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter \ + -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2") else() if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") set(protobuf_CXXFLAGS "-fstack-protector-all -Wno-uninitialized -Wno-unused-parameter -fPIC \ - -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2") + -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2") elseif(${CMAKE_SYSTEM_NAME} MATCHES "Windows") - set(protobuf_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC \ - -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2") + set(protobuf_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter \ + -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2") else() - set(protobuf_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter -fPIC \ - -fvisibility=hidden -D_FORTIFY_SOURCE=2 -D_GLIBCXX_USE_CXX11_ABI=0 -O2") + set(protobuf_CXXFLAGS "-fstack-protector-all -Wno-maybe-uninitialized -Wno-unused-parameter \ + -fPIC -fvisibility=hidden -D_FORTIFY_SOURCE=2 -O2") + if(NOT ENABLE_GLIBCXX) + set(protobuf_CXXFLAGS "${protobuf_CXXFLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") + endif() endif() endif() diff --git a/cmake/external_libs/sentencepiece.cmake b/cmake/external_libs/sentencepiece.cmake index be480197ae4..c3954df527e 100644 --- a/cmake/external_libs/sentencepiece.cmake +++ b/cmake/external_libs/sentencepiece.cmake @@ -9,27 +9,39 @@ endif() if(WIN32) set(sentencepiece_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2 -Wno-unused-result -Wno-stringop-overflow \ - -Wno-format-extra-args -Wno-format") + -Wno-format-extra-args -Wno-format") set(sentencepiece_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") mindspore_add_pkg(sentencepiece - VER 0.1.92 - LIBS sentencepiece sentencepiece_train - URL ${REQ_URL} - CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DSPM_USE_BUILTIN_PROTOBUF=ON - MD5 ${MD5} - ) + VER 0.1.92 + LIBS sentencepiece sentencepiece_train + URL ${REQ_URL} + CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DSPM_USE_BUILTIN_PROTOBUF=ON + MD5 ${MD5} + ) else() set(sentencepiece_CXXFLAGS "-D_FORTIFY_SOURCE=2 -O2 -Wno-unused-result -Wno-sign-compare") set(sentencepiece_CFLAGS "-D_FORTIFY_SOURCE=2 -O2") - mindspore_add_pkg(sentencepiece + if(ENABLE_GLIBCXX) + mindspore_add_pkg(sentencepiece VER 0.1.92 LIBS sentencepiece sentencepiece_train URL ${REQ_URL} CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DSPM_USE_BUILTIN_PROTOBUF=OFF -DSPM_ENABLE_SHARED=OFF - -DPROTOBUF_INC=${protobuf_INC} + -DPROTOBUF_INC=${protobuf_INC} + MD5 ${MD5} + PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/sentencepiece/sentencepiece.patch001_cpu + ) + else() + mindspore_add_pkg(sentencepiece + VER 0.1.92 + LIBS sentencepiece sentencepiece_train + URL ${REQ_URL} + CMAKE_OPTION -DCMAKE_BUILD_TYPE=Release -DSPM_USE_BUILTIN_PROTOBUF=OFF -DSPM_ENABLE_SHARED=OFF + -DPROTOBUF_INC=${protobuf_INC} MD5 ${MD5} PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/sentencepiece/sentencepiece.patch001 ) + endif() endif() include_directories(${sentencepiece_INC}) add_library(mindspore::sentencepiece ALIAS sentencepiece::sentencepiece) diff --git a/cmake/options.cmake b/cmake/options.cmake index 31a82eeacb6..6f1040b9840 100644 --- a/cmake/options.cmake +++ b/cmake/options.cmake @@ -21,6 +21,11 @@ option(ENABLE_DEBUGGER "enable debugger" OFF) option(ENABLE_IBVERBS "enable IBVERBS for parameter server" OFF) option(ENABLE_PYTHON "Enable python" ON) option(ENABLE_ACL "enable acl" OFF) +option(ENABLE_GLIBCXX "enable_glibcxx" OFF) + +if(NOT ENABLE_D AND NOT ENABLE_TESTCASES AND NOT ENABLE_ACL) + set(ENABLE_GLIBCXX ON) +endif() if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") if(WIN32) diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index 3552533fd01..42d3fe57983 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -10,6 +10,9 @@ endif() if(ENABLE_D) add_compile_definitions(ENABLE_D) endif() +if(NOT ENABLE_GLIBCXX) + add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=0) +endif() #add python lib and include for all ut executables; message("PYTHON_INCLUDE_DIRS = ${PYTHON_INCLUDE_DIRS}") diff --git a/third_party/patch/sentencepiece/sentencepiece.patch001_cpu b/third_party/patch/sentencepiece/sentencepiece.patch001_cpu new file mode 100644 index 00000000000..cea482b7e3b --- /dev/null +++ b/third_party/patch/sentencepiece/sentencepiece.patch001_cpu @@ -0,0 +1,91 @@ +diff -Npur sentencepiece-0.1.92/src/CMakeLists.txt sentencepiece-0.1.92_bak/src/CMakeLists.txt +--- sentencepiece-0.1.92/src/CMakeLists.txt 2020-06-08 16:25:01.000000000 +0800 ++++ sentencepiece-0.1.92_bak/src/CMakeLists.txt 2020-07-02 17:42:33.306933546 +0800 +@@ -11,6 +11,46 @@ + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License.! ++ ++ ++ ++function(protobuf_generate c_var h_var) ++ if(NOT ARGN) ++ message(SEND_ERROR "Error: ms_protobuf_generate() called without any proto files") ++ return() ++ endif() ++ ++ set(${c_var}) ++ set(${h_var}) ++ ++ find_program(PROTOC_EXE NAMES "protoc" PATHS ${PROTOBUF_INC}/../bin NO_DEFAULT_PATH) ++ ++ foreach(file ${ARGN}) ++ get_filename_component(abs_file ${file} ABSOLUTE) ++ get_filename_component(file_name ${file} NAME_WE) ++ get_filename_component(file_dir ${abs_file} PATH) ++ file(RELATIVE_PATH rel_path ${CMAKE_CURRENT_SOURCE_DIR} ${file_dir}) ++ ++ list(APPEND ${c_var} "${CMAKE_BINARY_DIR}/${file_name}.pb.cc") ++ list(APPEND ${h_var} "${CMAKE_BINARY_DIR}/${file_name}.pb.h") ++ ++ add_custom_command( ++ OUTPUT "${CMAKE_BINARY_DIR}/${file_name}.pb.cc" ++ "${CMAKE_BINARY_DIR}/${file_name}.pb.h" ++ WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} ++ COMMAND ${CMAKE_COMMAND} -E make_directory "${CMAKE_BINARY_DIR}" ++ COMMAND ${PROTOC_EXE} -I${file_dir} --cpp_out=${CMAKE_BINARY_DIR} ${abs_file} ++ DEPENDS ${PROTOC_EXE} ${abs_file} ++ COMMENT "Running C++ protocol buffer compiler on ${file}" VERBATIM) ++ endforeach() ++ ++ set_source_files_properties(${${c_var}} ${${h_var}} PROPERTIES GENERATED TRUE) ++ set(${c_var} ${${c_var}} PARENT_SCOPE) ++ set(${h_var} ${${h_var}} PARENT_SCOPE) ++ ++endfunction() ++ ++ + + if (SPM_USE_BUILTIN_PROTOBUF) + set(SPM_PROTO_HDRS builtin_pb/sentencepiece.pb.h) +@@ -52,12 +92,9 @@ if (SPM_USE_BUILTIN_PROTOBUF) + include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../third_party/protobuf-lite) + include_directories(builtin_pb) + else() +- find_package(Protobuf REQUIRED) +- include_directories(${Protobuf_INCLUDE_DIRS}) +- protobuf_generate_cpp(SPM_PROTO_SRCS SPM_PROTO_HDRS sentencepiece.proto) +- protobuf_generate_cpp(SPM_MODEL_PROTO_SRCS SPM_MODEL_PROTO_HDRS sentencepiece_model.proto) +- set(PROTOBUF_LITE_SRCS "") +- include_directories(${PROTOBUF_INCLUDE_DIR}) ++ include_directories(${PROTOBUF_INC}) ++ protobuf_generate(SPM_PROTO_SRCS SPM_PROTO_HDRS sentencepiece.proto) ++ protobuf_generate(SPM_MODEL_PROTO_SRCS SPM_MODEL_PROTO_HDRS sentencepiece_model.proto) + endif() + + include_directories(${CMAKE_CURRENT_BINARY_DIR}) +@@ -191,11 +228,13 @@ endif() + add_library(sentencepiece-static STATIC ${SPM_SRCS}) + add_library(sentencepiece_train-static STATIC ${SPM_TRAIN_SRCS}) + +-target_link_libraries(sentencepiece-static INTERFACE ${SPM_LIBS}) ++find_library(PROTO_LIB NAMES "libprotobuf.a" PATHS ${PROTOBUF_INC}/../lib NO_DEFAULT_PATH) ++ ++target_link_libraries(sentencepiece-static INTERFACE ${PROTO_LIB} ${SPM_LIBS}) + target_link_libraries(sentencepiece_train-static INTERFACE sentencepiece-static ${SPM_LIBS}) + + if (SPM_ENABLE_SHARED) +- target_link_libraries(sentencepiece ${SPM_LIBS}) ++ target_link_libraries(sentencepiece ${SPM_LIBS} ${PROTO_LIB}) + target_link_libraries(sentencepiece_train ${SPM_LIBS} sentencepiece) + set(SPM_INSTALLTARGETS sentencepiece sentencepiece_train sentencepiece-static sentencepiece_train-static) + set_target_properties(sentencepiece sentencepiece_train PROPERTIES SOVERSION 0 VERSION 0.0.0) +@@ -265,7 +304,7 @@ install(TARGETS ${SPM_INSTALLTARGETS} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) +-install(FILES sentencepiece_trainer.h sentencepiece_processor.h ++install(FILES sentencepiece_trainer.h sentencepiece_processor.h "${CMAKE_BINARY_DIR}/sentencepiece_model.pb.h" + DESTINATION ${CMAKE_INSTALL_INCDIR}) + + file(TO_NATIVE_PATH "${PROJECT_SOURCE_DIR}/data" data_dir) \ No newline at end of file