Add pslite dependency

2020-06-08 15:31:37 +08:00 · 2020-06-08 15:31:37 +08:00 · a057639abc
parent bd3f3b51e4
commit a057639abc
9 changed files with 319 additions and 8 deletions
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -38,6 +38,10 @@ if (NOT Patch_FOUND)
 endif ()
 message(PATCH_EXECUTABLE = ${Patch_EXECUTABLE})

+if (ENABLE_AKG AND ENABLE_D)
+    add_subdirectory("${CMAKE_SOURCE_DIR}/akg")
+endif()
+
 include(${CMAKE_SOURCE_DIR}/cmake/mind_expression.cmake)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR})
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/third_party/flatbuffers/include)
@ -86,10 +90,6 @@ if (ENABLE_GE OR ENABLE_D OR ENABLE_TESTCASES)
    include_directories(${CMAKE_CURRENT_SOURCE_DIR}/graphengine/third_party/fwkacllib/inc/toolchain)
 endif()

-if (ENABLE_AKG AND ENABLE_D)
-    add_subdirectory("${CMAKE_SOURCE_DIR}/akg")
-endif()
-
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fvisibility=hidden")
 add_subdirectory(mindspore/ccsrc)
 if (ENABLE_TESTCASES)
--- a/build.sh
+++ b/build.sh
@ -25,7 +25,7 @@ usage()
  echo "Usage:"
  echo "bash build.sh [-d] [-r] [-v] [-c on|off] [-t on|off] [-g on|off] [-h] [-b ge] [-m infer|train] \\"
  echo "              [-a on|off] [-Q on|off] [-p on|off] [-i] [-L] [-R] [-D on|off] [-j[n]] [-e gpu|d|cpu] \\"
-  echo "              [-P on|off] [-z [on|off]] [-M on|off] [-V 9.2|10.1] [-I] [-K] [-B on|off]"
+  echo "              [-P on|off] [-z [on|off]] [-M on|off] [-V 9.2|10.1] [-I] [-K] [-B on|off] [-E]"
  echo ""
  echo "Options:"
  echo "    -d Debug mode"
@ -55,6 +55,7 @@ usage()
  echo "    -K Compile with AKG, default on"
  echo "    -s Enable serving module, default off"
  echo "    -B Enable debugger, default off"
+  echo "    -E Enable IBVERBS for parameter server, default off"
 }

 # check value of input is 'on' or 'off'
@ -96,9 +97,10 @@ checkopts()
  ENABLE_AKG="on"
  ENABLE_SERVING="off"
  ENABLE_DEBUGGER="off"
+  ENABLE_IBVERBS="off"

  # Process the options
-  while getopts 'drvj:c:t:hsb:a:g:p:ie:m:I:LRP:Q:D:zM:V:K:sB:' opt
+  while getopts 'drvj:c:t:hsb:a:g:p:ie:m:I:LRP:Q:D:zM:V:K:sB:E' opt
  do
    OPTARG=$(echo ${OPTARG} | tr '[A-Z]' '[a-z]')
    case "${opt}" in
@ -252,6 +254,10 @@ checkopts()
        ENABLE_DEBUGGER="on"
        echo "enable debugger"
        ;;
+      E)
+        ENABLE_IBVERBS="on"
+        echo "enable IBVERBS for parameter server"
+        ;;
      *)
        echo "Unknown option ${opt}!"
        usage
@ -338,6 +344,9 @@ build_mindspore()
        CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_DEBUGGER=ON"
    fi

+    if [[ "X$ENABLE_IBVERBS" = "Xon" ]]; then
+        CMAKE_ARGS="${CMAKE_ARGS} -DENABLE_IBVERBS=ON"
+    fi
    echo "${CMAKE_ARGS}"
    if [[ "X$INC_BUILD" = "Xoff" ]]; then
      cmake ${CMAKE_ARGS} ../..
--- a/cmake/external_libs/pslite.cmake
+++ b/cmake/external_libs/pslite.cmake
@ -0,0 +1,14 @@
+set(pslite_USE_STATIC_LIBS ON)
+if (${ENABLE_IBVERBS} STREQUAL "ON")
+    set(pslite_CXXFLAGS "USE_IBVERBS=1")
+endif()
+mindspore_add_pkg(pslite
+        LIBS ps
+        URL https://github.com/dmlc/ps-lite/archive/34fd45cae457d59850fdcb2066467778d0673f21.zip
+        MD5 393c0e27b68bfaf96718caa3aa96f5a3
+        PATCHES ${CMAKE_SOURCE_DIR}/third_party/patch/pslite/ps_lite.patch001
+        ONLY_MAKE True
+        ONLY_MAKE_INCS include/*
+        ONLY_MAKE_LIBS build/*)
+include_directories(${pslite_INC})
+add_library(mindspore::pslite ALIAS pslite::ps)
--- a/cmake/external_libs/zeromq.cmake
+++ b/cmake/external_libs/zeromq.cmake
@ -0,0 +1,5 @@
+mindspore_add_pkg(zeromq
+        VER 4.1.4
+        HEAD_ONLY ./
+        URL  https://raw.githubusercontent.com/mli/deps/master/build/zeromq-4.1.4.tar.gz
+        MD5  a611ecc93fffeb6d058c0e6edf4ad4fb)
--- a/cmake/mind_expression.cmake
+++ b/cmake/mind_expression.cmake
@ -30,6 +30,10 @@ include(${CMAKE_SOURCE_DIR}/cmake/external_libs/flatbuffers.cmake)
 if(USE_GLOG)
    include(${CMAKE_SOURCE_DIR}/cmake/external_libs/glog.cmake)
 endif()
+if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Windows")
+    include(${CMAKE_SOURCE_DIR}/cmake/external_libs/zeromq.cmake)
+    include(${CMAKE_SOURCE_DIR}/cmake/external_libs/pslite.cmake)
+endif()

 find_package(Python3)
 include_directories(${Python3_INCLUDE_DIRS})
--- a/cmake/options.cmake
+++ b/cmake/options.cmake
@ -18,6 +18,7 @@ option(ENABLE_DUMP_IR "Enable dump funciton graph ir, default on" ON)
 option(ENABLE_MPI "enable mpi" OFF)
 option(ENABLE_AKG "enable akg" OFF)
 option(ENABLE_DEBUGGER "enable debugger" OFF)
+option(ENABLE_IBVERBS "enable IBVERBS for parameter server" OFF)

 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
    if (WIN32)
--- a/cmake/utils.cmake
+++ b/cmake/utils.cmake
@ -206,7 +206,7 @@ function(mindspore_add_pkg pkg_name )

    set(options )
    set(oneValueArgs URL MD5 GIT_REPOSITORY GIT_TAG VER EXE DIR HEAD_ONLY CMAKE_PATH RELEASE LIB_PATH CUSTOM_CMAKE)
-    set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES SUBMODULES SOURCEMODULES)
+    set(multiValueArgs CMAKE_OPTION LIBS PRE_CONFIGURE_COMMAND CONFIGURE_COMMAND BUILD_OPTION INSTALL_INCS INSTALL_LIBS PATCHES SUBMODULES SOURCEMODULES ONLY_MAKE ONLY_MAKE_INCS ONLY_MAKE_LIBS)
    cmake_parse_arguments(PKG "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN} )

    if (NOT PKG_LIB_PATH)
@ -290,7 +290,7 @@ function(mindspore_add_pkg pkg_name )
    foreach(_PATCH_FILE ${PKG_PATCHES})
        get_filename_component(_PATCH_FILE_NAME ${_PATCH_FILE} NAME)
        set(_LF_PATCH_FILE ${CMAKE_BINARY_DIR}/_ms_patch/${_PATCH_FILE_NAME})
-        configure_file(${_PATCH_FILE} ${_LF_PATCH_FILE} NEWLINE_STYLE LF)
+        configure_file(${_PATCH_FILE} ${_LF_PATCH_FILE} NEWLINE_STYLE LF @ONLY)

        message("patching ${${pkg_name}_SOURCE_DIR} -p1 < ${_LF_PATCH_FILE}")
        execute_process(COMMAND ${Patch_EXECUTABLE} -p1 INPUT_FILE ${_LF_PATCH_FILE}
@ -324,6 +324,16 @@ function(mindspore_add_pkg pkg_name )
                target_include_directories(${pkg_name} INTERFACE ${${pkg_name}_INC})
            endif ()

+        elseif (PKG_ONLY_MAKE)
+            __exec_cmd(COMMAND ${CMAKE_MAKE_PROGRAM} ${${pkg_name}_CXXFLAGS} -j${THNUM}
+                    WORKING_DIRECTORY ${${pkg_name}_SOURCE_DIR})
+            set(PKG_INSTALL_INCS ${PKG_ONLY_MAKE_INCS})
+            set(PKG_INSTALL_LIBS ${PKG_ONLY_MAKE_LIBS})
+            file(GLOB ${pkg_name}_INSTALL_INCS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_INCS})
+            file(GLOB ${pkg_name}_INSTALL_LIBS ${${pkg_name}_SOURCE_DIR}/${PKG_INSTALL_LIBS})
+            file(COPY ${${pkg_name}_INSTALL_INCS} DESTINATION ${${pkg_name}_BASE_DIR}/include)
+            file(COPY ${${pkg_name}_INSTALL_LIBS} DESTINATION ${${pkg_name}_BASE_DIR}/lib)
+
        elseif (PKG_CMAKE_OPTION)
            # in cmake
            file(MAKE_DIRECTORY ${${pkg_name}_SOURCE_DIR}/_build)
--- a/mindspore/ccsrc/CMakeLists.txt
+++ b/mindspore/ccsrc/CMakeLists.txt
@ -230,6 +230,10 @@ else ()
    target_link_libraries(_c_expression PRIVATE -Wl,--whole-archive mindspore -Wl,--no-whole-archive)
    target_link_libraries(_c_expression PRIVATE mindspore::pybind11_module)
    target_link_libraries(_c_expression PRIVATE mindspore_gvar)
+    target_link_libraries(_c_expression PRIVATE mindspore::pslite mindspore::protobuf ${zeromq_DIRPATH}/zmq_install/lib/libzmq.a)
+    if (${ENABLE_IBVERBS} STREQUAL "ON")
+        target_link_libraries(_c_expression PRIVATE ibverbs rdmacm)
+    endif()
 endif ()

 if (USE_GLOG)
--- a/third_party/patch/pslite/ps_lite.patch001
+++ b/third_party/patch/pslite/ps_lite.patch001
@ -0,0 +1,264 @@
+diff -Npur ps-lite-master/include/dmlc/base.h ps-lite-master-new/include/dmlc/base.h
+--- ps-lite-master/include/dmlc/base.h	2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/include/dmlc/base.h	2020-07-01 11:56:50.444833389 +0800
+@@ -8,7 +8,7 @@
+ 
+ /*! \brief whether use glog for logging */
+ #ifndef DMLC_USE_GLOG
+-#define DMLC_USE_GLOG 0
+#define DMLC_USE_GLOG 1
+ #endif
+ 
+ /*!
+diff -Npur ps-lite-master/include/dmlc/logging.h ps-lite-master-new/include/dmlc/logging.h
+--- ps-lite-master/include/dmlc/logging.h	2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/include/dmlc/logging.h	2020-07-01 11:58:00.015919207 +0800
+@@ -13,7 +13,7 @@
+ #include <string>
+ #include <vector>
+ #include <memory>
+-#include "./base.h"
+//#include "./base.h"
+ 
+ #if DMLC_LOG_STACK_TRACE
+ #include <cxxabi.h>
+@@ -52,7 +52,7 @@ struct Error : public std::runtime_error
+ 
+ namespace dmlc {
+ inline void InitLogging(const char* argv0) {
+-  google::InitGoogleLogging(argv0);
+  //google::InitGoogleLogging(argv0);
+ }
+ }  // namespace dmlc
+ 
+diff -Npur ps-lite-master/make/deps.mk ps-lite-master-new/make/deps.mk
+--- ps-lite-master/make/deps.mk	2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/make/deps.mk	2020-06-17 10:35:46.253837426 +0800
+@@ -1,69 +1,7 @@
+ # Install dependencies
+-
+-URL1=https://raw.githubusercontent.com/mli/deps/master/build
+-URL2=https://github.com/google/protobuf/releases/download/v3.5.1
+-ifndef WGET
+-WGET = wget
+-endif
+-
+-# protobuf
+-PROTOBUF = ${DEPS_PATH}/include/google/protobuf/message.h
+-${PROTOBUF}:
+-	$(eval FILE=protobuf-cpp-3.5.1.tar.gz)
+-	$(eval DIR=protobuf-3.5.1)
+-	rm -rf $(FILE) $(DIR)
+-	$(WGET) $(URL2)/$(FILE) && tar --no-same-owner -zxf $(FILE)
+-	cd $(DIR) && export CFLAGS=-fPIC && export CXXFLAGS=-fPIC && ./configure -prefix=$(DEPS_PATH) && $(MAKE) && $(MAKE) install
+-	rm -rf $(FILE) $(DIR)
+-
+ # zmq
+-ZMQ = ${DEPS_PATH}/include/zmq.h
+ZMQ = $(MS_ZMQ_INSTALL_PATH)/lib/libzmq.a
+ 
+ ${ZMQ}:
+-	$(eval FILE=zeromq-4.1.4.tar.gz)
+-	$(eval DIR=zeromq-4.1.4)
+-	rm -rf $(FILE) $(DIR)
+-	$(WGET) $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE)
+-	cd $(DIR) && export CFLAGS=-fPIC && export CXXFLAGS=-fPIC && ./configure -prefix=$(DEPS_PATH) --with-libsodium=no --with-libgssapi_krb5=no && $(MAKE) && $(MAKE) install
+-	rm -rf $(FILE) $(DIR)
+-
+-# lz4
+-LZ4 = ${DEPS_PATH}/include/lz4.h
+-${LZ4}:
+-	$(eval FILE=lz4-r129.tar.gz)
+-	$(eval DIR=lz4-r129)
+-	rm -rf $(FILE) $(DIR)
+-	wget $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE)
+-	cd $(DIR) && $(MAKE) && PREFIX=$(DEPS_PATH) $(MAKE) install
+-	rm -rf $(FILE) $(DIR)
+-
+-# cityhash
+-CITYHASH = ${DEPS_PATH}/include/city.h
+-${CITYHASH}:
+-	$(eval FILE=cityhash-1.1.1.tar.gz)
+-	$(eval DIR=cityhash-1.1.1)
+-	rm -rf $(FILE) $(DIR)
+-	wget $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE)
+-	cd $(DIR) && ./configure -prefix=$(DEPS_PATH) --enable-sse4.2 && $(MAKE) CXXFLAGS="-g -O3 -msse4.2" && $(MAKE) install
+-	rm -rf $(FILE) $(DIR)
+-
+-
+-# # gflags
+-# ${DEPS_PATH}/include/google/gflags.h:
+-# 	$(eval FILE=gflags-2.0-no-svn-files.tar.gz)
+-# 	$(eval DIR=gflags-2.0)
+-# 	rm -rf $(FILE) $(DIR)
+-# 	wget $(URL)/$(FILE) && tar -zxf $(FILE)
+-# 	cd $(DIR) && ./configure -prefix=$(DEPS_PATH) && $(MAKE) && $(MAKE) install
+-# 	rm -rf $(FILE) $(DIR)
+-# gflags: | ${DEPS_PATH}/include/google/gflags.h
+	cd $(MS_ZMQ_DIR) && export CFLAGS="-fPIC -D_GLIBCXX_USE_CXX11_ABI=0" && export CXXFLAGS=-fPIC && ./configure -prefix=$(MS_ZMQ_INSTALL_PATH) --with-libsodium=no --with-libgssapi_krb5=no && $(MAKE) && $(MAKE) install
+ 
+-# # glog
+-# ${DEPS_PATH}/include/glog/logging.h: | ${DEPS_PATH}/include/google/gflags.h
+-# 	$(eval FILE=v0.3.4.tar.gz)
+-# 	$(eval DIR=glog-0.3.4)
+-# 	rm -rf $(FILE) $(DIR)
+-# 	wget https://github.com/google/glog/archive/$(FILE) && tar -zxf $(FILE)
+-# 	cd $(DIR) && ./configure -prefix=$(DEPS_PATH) --with-gflags=$(DEPS_PATH) && $(MAKE) && $(MAKE) install
+-# 	rm -rf $(FILE) $(DIR)
+-# glog: | ${DEPS_PATH}/include/glog/logging.h
+diff -Npur ps-lite-master/make/ps.mk ps-lite-master-new/make/ps.mk
+--- ps-lite-master/make/ps.mk	2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/make/ps.mk	2020-06-05 09:28:35.337740291 +0800
+@@ -9,5 +9,5 @@ ifeq ($(USE_KEY32), 1)
+ ADD_CFLAGS += -DUSE_KEY32=1
+ endif
+ 
+-PS_LDFLAGS_SO = -L$(DEPS_PATH)/lib -lprotobuf-lite -lzmq
+-PS_LDFLAGS_A = $(addprefix $(DEPS_PATH)/lib/, libprotobuf-lite.a libzmq.a)
+PS_LDFLAGS_SO = -L$(MS_ZMQ_INSTALL_PATH)/lib -lzmq -L$(MS_PROTO_LIB_DIR) -lprotobuf-lite
+PS_LDFLAGS_A = $(addprefix $(MS_ZMQ_INSTALL_PATH)/lib -L$(MS_PROTO_LIB_DIR), libprotobuf-lite.a libzmq.a)
+diff -Npur ps-lite-master/Makefile ps-lite-master-new/Makefile
+--- ps-lite-master/Makefile	2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/Makefile	2020-06-17 11:09:20.240322660 +0800
+@@ -12,13 +12,24 @@ ifndef DEPS_PATH
+ DEPS_PATH = $(shell pwd)/deps
+ endif
+ 
+MS_PROTO_DIR = @protobuf_DIRPATH@
+MS_GLOG_DIR = @glog_DIRPATH@
+MS_ZMQ_DIR = @zeromq_DIRPATH@
+
+MS_PROTO_LIB_DIR = @protobuf_LIBPATH@
+MS_GLOG_LIB_DIR = @glog_LIBPATH@
+MS_ZMQ_INSTALL_PATH = $(MS_ZMQ_DIR)/zmq_install
+ 
+ ifndef PROTOC
+-PROTOC = ${DEPS_PATH}/bin/protoc
+PROTOC = $(MS_PROTO_DIR)/bin/protoc
+ endif
+ 
+-INCPATH = -I./src -I./include -I$(DEPS_PATH)/include
+-CFLAGS = -std=c++11 -msse2 -fPIC -O3 -ggdb -Wall -finline-functions $(INCPATH) $(ADD_CFLAGS)
+INCPATH = -I./src -I./include -I$(MS_ZMQ_INSTALL_PATH)/include
+INCPATH += -I$(MS_PROTO_DIR)/include
+INCPATH += -I$(MS_GLOG_DIR)/include
+
+CXXFLAGS = -D_GLIBCXX_USE_CXX11_ABI=0
+CFLAGS = -std=c++11 -fPIC -O3 -ggdb -Wall -finline-functions $(INCPATH) $(ADD_CFLAGS) -D_GLIBCXX_USE_CXX11_ABI=0
+ LIBS = -pthread
+ 
+ ifdef USE_IBVERBS
+@@ -30,6 +41,7 @@ ifdef ASAN
+ CFLAGS += -fsanitize=address -fno-omit-frame-pointer -fno-optimize-sibling-calls
+ endif
+ 
+LIBS += -L$(MS_GLOG_LIB_DIR) -lglog
+ 
+ all: ps test
+ 
+@@ -51,9 +63,9 @@ build/libps.a: $(OBJS)
+ build/%.o: src/%.cc ${ZMQ} src/meta.pb.h
+ 	@mkdir -p $(@D)
+ 	$(CXX) $(INCPATH) -std=c++11 -MM -MT build/$*.o $< >build/$*.d
+-	$(CXX) $(CFLAGS) $(LIBS) -c $< -o $@
+	$(CXX) $(CFLAGS) $(CXXFLAGS) $(LIBS) -c $< -o $@
+ 
+-src/%.pb.cc src/%.pb.h : src/%.proto ${PROTOBUF}
+src/%.pb.cc src/%.pb.h : src/%.proto
+ 	$(PROTOC) --cpp_out=./src --proto_path=./src $<
+ 
+ -include build/*.d
+diff -Npur ps-lite-master/src/ibverbs_van.h ps-lite-master-new/src/ibverbs_van.h
+--- ps-lite-master/src/ibverbs_van.h	2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/src/ibverbs_van.h	2020-06-02 20:52:11.076230014 +0800
+@@ -145,15 +145,15 @@ class SimpleMempool {
+       total_allocated_size += new_mem_size;
+     }
+ 
+-    CHECK_NE(free_list.end(), it) << "Not enough memory";
+    //CHECK_NE(free_list.end(), it) << "Not enough memory";
+     CHECK_GE(it->first, proper_size);
+ 
+     char *addr = it->second;
+     size_t space_left = it->first - proper_size;
+ 
+     free_list.erase(it);
+-    CHECK_EQ(used_list.find(addr), used_list.end())
+-        << "Address is already allocated";
+    //CHECK_EQ(used_list.find(addr), used_list.end())
+        //<< "Address is already allocated";
+ 
+     used_list.emplace(addr, proper_size);
+ 
+@@ -173,8 +173,8 @@ class SimpleMempool {
+     std::lock_guard<std::mutex> lk(mu_);
+ 
+     auto it = used_list.find(addr);
+-    CHECK_NE(used_list.end(), it)
+-        << "Cannot find info about address: " << (uintptr_t)addr;
+    //CHECK_NE(used_list.end(), it)
+        //<< "Cannot find info about address: " << (uintptr_t)addr;
+ 
+     size_t size = it->second;
+     used_list.erase(it);
+@@ -208,7 +208,7 @@ class SimpleMempool {
+   // Convert the memory address to its associated RDMA memory region
+   inline struct ibv_mr *Addr2MR(char *addr) {
+     auto it = mr_list.lower_bound(addr);
+-    CHECK_NE(it, mr_list.end()) << "cannot find the associated memory region";
+    //CHECK_NE(it, mr_list.end()) << "cannot find the associated memory region";
+     return it->second;
+   }
+ };
+@@ -330,7 +330,7 @@ class AddressPool {
+     CHECK(ptr);
+     uint32_t idx = indices_.front();
+     indices_.pop();
+-    CHECK_EQ(table_[idx], nullptr);
+    //CHECK_EQ(table_[idx], nullptr);
+     table_[idx] = ptr;
+     return idx;
+   }
+@@ -636,7 +636,7 @@ class IBVerbsVan : public Van {
+     PBMeta meta;
+     PackMetaPB(msg.meta, &meta);
+ 
+-    CHECK_NE(endpoints_.find(remote_id), endpoints_.end());
+    //CHECK_NE(endpoints_.find(remote_id), endpoints_.end());
+     Endpoint *endpoint = endpoints_[remote_id].get();
+     MessageBuffer *msg_buf = new MessageBuffer();
+ 
+diff -Npur ps-lite-master/src/van.cc ps-lite-master-new/src/van.cc
+--- ps-lite-master/src/van.cc	2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/src/van.cc	2020-06-02 20:52:43.330405828 +0800
+@@ -448,6 +448,7 @@ void Van::PackMetaPB(const Meta& meta, P
+   if (meta.timestamp != Meta::kEmpty) pb->set_timestamp(meta.timestamp);
+   if (meta.body.size()) pb->set_body(meta.body);
+   pb->set_push(meta.push);
+  pb->set_pull(meta.pull);
+   pb->set_request(meta.request);
+   pb->set_simple_app(meta.simple_app);
+   pb->set_priority(meta.priority);
+diff -Npur ps-lite-master/tests/test.mk ps-lite-master-new/tests/test.mk
+--- ps-lite-master/tests/test.mk	2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/tests/test.mk	2020-06-16 19:15:06.025087897 +0800
+@@ -1,10 +1,10 @@
+-TEST_SRC = $(wildcard tests/test_*.cc)
+-TEST = $(patsubst tests/test_%.cc, tests/test_%, $(TEST_SRC))
+#TEST_SRC = $(wildcard tests/test_*.cc)
+#TEST = $(patsubst tests/test_%.cc, tests/test_%, $(TEST_SRC))
+ 
+-# -ltcmalloc_and_profiler
+-LDFLAGS = -Wl,-rpath,$(DEPS_PATH)/lib $(PS_LDFLAGS_SO) -pthread
+-tests/% : tests/%.cc build/libps.a
+-	$(CXX) $(CFLAGS) $(LIBS) -MM -MT tests/$* $< >tests/$*.d
+-	$(CXX) $(CFLAGS) $(LIBS) -o $@ $(filter %.cc %.a, $^) $(LDFLAGS)
+-
+--include tests/*.d
+## -ltcmalloc_and_profiler
+#LDFLAGS = -Wl,-rpath,$(DEPS_PATH)/lib $(PS_LDFLAGS_SO) -pthread
+#tests/% : tests/%.cc build/libps.a
+#	$(CXX) $(CFLAGS) $(LIBS) -MM -MT tests/$* $< >tests/$*.d
+#	$(CXX) $(CFLAGS) $(LIBS) -o $@ $(filter %.cc %.a, $^) $(LDFLAGS)
+#
+#-include tests/*.d