sfqcyy
/
mindspore
forked from OSSInnovation/mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity
mindspore/third_party/patch/pslite/ps_lite.patch001

256 lines
9.6 KiB
Plaintext
Raw Blame History

diff -Npur ps-lite-master/include/dmlc/base.h ps-lite-master-new/include/dmlc/base.h
--- ps-lite-master/include/dmlc/base.h 2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/include/dmlc/base.h 2020-07-01 11:56:50.444833389 +0800
@@ -8,7 +8,7 @@
/*! \brief whether use glog for logging */
#ifndef DMLC_USE_GLOG
-#define DMLC_USE_GLOG 0
+#define DMLC_USE_GLOG 1
#endif
/*!
diff -Npur ps-lite-master/include/dmlc/logging.h ps-lite-master-new/include/dmlc/logging.h
--- ps-lite-master/include/dmlc/logging.h 2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/include/dmlc/logging.h 2020-07-08 21:35:33.334584767 +0800
@@ -52,7 +52,7 @@ struct Error : public std::runtime_error
namespace dmlc {
inline void InitLogging(const char* argv0) {
- google::InitGoogleLogging(argv0);
+ //google::InitGoogleLogging(argv0);
}
} // namespace dmlc
diff -Npur ps-lite-master/make/deps.mk ps-lite-master-new/make/deps.mk
--- ps-lite-master/make/deps.mk 2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/make/deps.mk 2020-06-17 10:35:46.253837426 +0800
@@ -1,69 +1,7 @@
# Install dependencies
-
-URL1=https://raw.githubusercontent.com/mli/deps/master/build
-URL2=https://github.com/google/protobuf/releases/download/v3.5.1
-ifndef WGET
-WGET = wget
-endif
-
-# protobuf
-PROTOBUF = ${DEPS_PATH}/include/google/protobuf/message.h
-${PROTOBUF}:
- $(eval FILE=protobuf-cpp-3.5.1.tar.gz)
- $(eval DIR=protobuf-3.5.1)
- rm -rf $(FILE) $(DIR)
- $(WGET) $(URL2)/$(FILE) && tar --no-same-owner -zxf $(FILE)
- cd $(DIR) && export CFLAGS=-fPIC && export CXXFLAGS=-fPIC && ./configure -prefix=$(DEPS_PATH) && $(MAKE) && $(MAKE) install
- rm -rf $(FILE) $(DIR)
-
# zmq
-ZMQ = ${DEPS_PATH}/include/zmq.h
+ZMQ = $(MS_ZMQ_INSTALL_PATH)/lib/libzmq.a
${ZMQ}:
- $(eval FILE=zeromq-4.1.4.tar.gz)
- $(eval DIR=zeromq-4.1.4)
- rm -rf $(FILE) $(DIR)
- $(WGET) $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE)
- cd $(DIR) && export CFLAGS=-fPIC && export CXXFLAGS=-fPIC && ./configure -prefix=$(DEPS_PATH) --with-libsodium=no --with-libgssapi_krb5=no && $(MAKE) && $(MAKE) install
- rm -rf $(FILE) $(DIR)
-
-# lz4
-LZ4 = ${DEPS_PATH}/include/lz4.h
-${LZ4}:
- $(eval FILE=lz4-r129.tar.gz)
- $(eval DIR=lz4-r129)
- rm -rf $(FILE) $(DIR)
- wget $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE)
- cd $(DIR) && $(MAKE) && PREFIX=$(DEPS_PATH) $(MAKE) install
- rm -rf $(FILE) $(DIR)
-
-# cityhash
-CITYHASH = ${DEPS_PATH}/include/city.h
-${CITYHASH}:
- $(eval FILE=cityhash-1.1.1.tar.gz)
- $(eval DIR=cityhash-1.1.1)
- rm -rf $(FILE) $(DIR)
- wget $(URL1)/$(FILE) && tar --no-same-owner -zxf $(FILE)
- cd $(DIR) && ./configure -prefix=$(DEPS_PATH) --enable-sse4.2 && $(MAKE) CXXFLAGS="-g -O3 -msse4.2" && $(MAKE) install
- rm -rf $(FILE) $(DIR)
-
-
-# # gflags
-# ${DEPS_PATH}/include/google/gflags.h:
-# $(eval FILE=gflags-2.0-no-svn-files.tar.gz)
-# $(eval DIR=gflags-2.0)
-# rm -rf $(FILE) $(DIR)
-# wget $(URL)/$(FILE) && tar -zxf $(FILE)
-# cd $(DIR) && ./configure -prefix=$(DEPS_PATH) && $(MAKE) && $(MAKE) install
-# rm -rf $(FILE) $(DIR)
-# gflags: | ${DEPS_PATH}/include/google/gflags.h
+ cd $(MS_ZMQ_DIR) && export CFLAGS="-fPIC -D_GLIBCXX_USE_CXX11_ABI=0" && export CXXFLAGS=-fPIC && ./configure -prefix=$(MS_ZMQ_INSTALL_PATH) --with-libsodium=no --with-libgssapi_krb5=no && $(MAKE) && $(MAKE) install
-# # glog
-# ${DEPS_PATH}/include/glog/logging.h: | ${DEPS_PATH}/include/google/gflags.h
-# $(eval FILE=v0.3.4.tar.gz)
-# $(eval DIR=glog-0.3.4)
-# rm -rf $(FILE) $(DIR)
-# wget https://github.com/google/glog/archive/$(FILE) && tar -zxf $(FILE)
-# cd $(DIR) && ./configure -prefix=$(DEPS_PATH) --with-gflags=$(DEPS_PATH) && $(MAKE) && $(MAKE) install
-# rm -rf $(FILE) $(DIR)
-# glog: | ${DEPS_PATH}/include/glog/logging.h
diff -Npur ps-lite-master/make/ps.mk ps-lite-master-new/make/ps.mk
--- ps-lite-master/make/ps.mk 2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/make/ps.mk 2020-06-05 09:28:35.337740291 +0800
@@ -9,5 +9,5 @@ ifeq ($(USE_KEY32), 1)
ADD_CFLAGS += -DUSE_KEY32=1
endif
-PS_LDFLAGS_SO = -L$(DEPS_PATH)/lib -lprotobuf-lite -lzmq
-PS_LDFLAGS_A = $(addprefix $(DEPS_PATH)/lib/, libprotobuf-lite.a libzmq.a)
+PS_LDFLAGS_SO = -L$(MS_ZMQ_INSTALL_PATH)/lib -lzmq -L$(MS_PROTO_LIB_DIR) -lprotobuf-lite
+PS_LDFLAGS_A = $(addprefix $(MS_ZMQ_INSTALL_PATH)/lib -L$(MS_PROTO_LIB_DIR), libprotobuf-lite.a libzmq.a)
diff -Npur ps-lite-master/Makefile ps-lite-master-new/Makefile
--- ps-lite-master/Makefile 2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/Makefile 2020-06-17 11:09:20.240322660 +0800
@@ -12,13 +12,24 @@ ifndef DEPS_PATH
DEPS_PATH = $(shell pwd)/deps
endif
+MS_PROTO_DIR = @protobuf_DIRPATH@
+MS_GLOG_DIR = @glog_DIRPATH@
+MS_ZMQ_DIR = @zeromq_DIRPATH@
+
+MS_PROTO_LIB_DIR = @protobuf_LIBPATH@
+MS_GLOG_LIB_DIR = @glog_LIBPATH@
+MS_ZMQ_INSTALL_PATH = $(MS_ZMQ_DIR)/zmq_install
ifndef PROTOC
-PROTOC = ${DEPS_PATH}/bin/protoc
+PROTOC = $(MS_PROTO_DIR)/bin/protoc
endif
-INCPATH = -I./src -I./include -I$(DEPS_PATH)/include
-CFLAGS = -std=c++11 -msse2 -fPIC -O3 -ggdb -Wall -finline-functions $(INCPATH) $(ADD_CFLAGS)
+INCPATH = -I./src -I./include -I$(MS_ZMQ_INSTALL_PATH)/include
+INCPATH += -I$(MS_PROTO_DIR)/include
+INCPATH += -I$(MS_GLOG_DIR)/include
+
+CXXFLAGS = -D_GLIBCXX_USE_CXX11_ABI=0
+CFLAGS = -std=c++11 -fPIC -O3 -ggdb -Wall -finline-functions $(INCPATH) $(ADD_CFLAGS) -D_GLIBCXX_USE_CXX11_ABI=0
LIBS = -pthread
ifdef USE_IBVERBS
@@ -30,6 +41,7 @@ ifdef ASAN
CFLAGS += -fsanitize=address -fno-omit-frame-pointer -fno-optimize-sibling-calls
endif
+LIBS += -L$(MS_GLOG_LIB_DIR) -lglog
all: ps test
@@ -51,9 +63,9 @@ build/libps.a: $(OBJS)
build/%.o: src/%.cc ${ZMQ} src/meta.pb.h
@mkdir -p $(@D)
$(CXX) $(INCPATH) -std=c++11 -MM -MT build/$*.o $< >build/$*.d
- $(CXX) $(CFLAGS) $(LIBS) -c $< -o $@
+ $(CXX) $(CFLAGS) $(CXXFLAGS) $(LIBS) -c $< -o $@
-src/%.pb.cc src/%.pb.h : src/%.proto ${PROTOBUF}
+src/%.pb.cc src/%.pb.h : src/%.proto
$(PROTOC) --cpp_out=./src --proto_path=./src $<
-include build/*.d
diff -Npur ps-lite-master/src/ibverbs_van.h ps-lite-master-new/src/ibverbs_van.h
--- ps-lite-master/src/ibverbs_van.h 2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/src/ibverbs_van.h 2020-06-02 20:52:11.076230014 +0800
@@ -145,15 +145,15 @@ class SimpleMempool {
total_allocated_size += new_mem_size;
}
- CHECK_NE(free_list.end(), it) << "Not enough memory";
+ //CHECK_NE(free_list.end(), it) << "Not enough memory";
CHECK_GE(it->first, proper_size);
char *addr = it->second;
size_t space_left = it->first - proper_size;
free_list.erase(it);
- CHECK_EQ(used_list.find(addr), used_list.end())
- << "Address is already allocated";
+ //CHECK_EQ(used_list.find(addr), used_list.end())
+ //<< "Address is already allocated";
used_list.emplace(addr, proper_size);
@@ -173,8 +173,8 @@ class SimpleMempool {
std::lock_guard<std::mutex> lk(mu_);
auto it = used_list.find(addr);
- CHECK_NE(used_list.end(), it)
- << "Cannot find info about address: " << (uintptr_t)addr;
+ //CHECK_NE(used_list.end(), it)
+ //<< "Cannot find info about address: " << (uintptr_t)addr;
size_t size = it->second;
used_list.erase(it);
@@ -208,7 +208,7 @@ class SimpleMempool {
// Convert the memory address to its associated RDMA memory region
inline struct ibv_mr *Addr2MR(char *addr) {
auto it = mr_list.lower_bound(addr);
- CHECK_NE(it, mr_list.end()) << "cannot find the associated memory region";
+ //CHECK_NE(it, mr_list.end()) << "cannot find the associated memory region";
return it->second;
}
};
@@ -330,7 +330,7 @@ class AddressPool {
CHECK(ptr);
uint32_t idx = indices_.front();
indices_.pop();
- CHECK_EQ(table_[idx], nullptr);
+ //CHECK_EQ(table_[idx], nullptr);
table_[idx] = ptr;
return idx;
}
@@ -636,7 +636,7 @@ class IBVerbsVan : public Van {
PBMeta meta;
PackMetaPB(msg.meta, &meta);
- CHECK_NE(endpoints_.find(remote_id), endpoints_.end());
+ //CHECK_NE(endpoints_.find(remote_id), endpoints_.end());
Endpoint *endpoint = endpoints_[remote_id].get();
MessageBuffer *msg_buf = new MessageBuffer();
diff -Npur ps-lite-master/src/van.cc ps-lite-master-new/src/van.cc
--- ps-lite-master/src/van.cc 2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/src/van.cc 2020-06-02 20:52:43.330405828 +0800
@@ -448,6 +448,7 @@ void Van::PackMetaPB(const Meta& meta, P
if (meta.timestamp != Meta::kEmpty) pb->set_timestamp(meta.timestamp);
if (meta.body.size()) pb->set_body(meta.body);
pb->set_push(meta.push);
+ pb->set_pull(meta.pull);
pb->set_request(meta.request);
pb->set_simple_app(meta.simple_app);
pb->set_priority(meta.priority);
diff -Npur ps-lite-master/tests/test.mk ps-lite-master-new/tests/test.mk
--- ps-lite-master/tests/test.mk 2020-02-29 13:59:55.000000000 +0800
+++ ps-lite-master-new/tests/test.mk 2020-06-16 19:15:06.025087897 +0800
@@ -1,10 +1,10 @@
-TEST_SRC = $(wildcard tests/test_*.cc)
-TEST = $(patsubst tests/test_%.cc, tests/test_%, $(TEST_SRC))
+#TEST_SRC = $(wildcard tests/test_*.cc)
+#TEST = $(patsubst tests/test_%.cc, tests/test_%, $(TEST_SRC))
-# -ltcmalloc_and_profiler
-LDFLAGS = -Wl,-rpath,$(DEPS_PATH)/lib $(PS_LDFLAGS_SO) -pthread
-tests/% : tests/%.cc build/libps.a
- $(CXX) $(CFLAGS) $(LIBS) -MM -MT tests/$* $< >tests/$*.d
- $(CXX) $(CFLAGS) $(LIBS) -o $@ $(filter %.cc %.a, $^) $(LDFLAGS)
-
--include tests/*.d
+## -ltcmalloc_and_profiler
+#LDFLAGS = -Wl,-rpath,$(DEPS_PATH)/lib $(PS_LDFLAGS_SO) -pthread
+#tests/% : tests/%.cc build/libps.a
+# $(CXX) $(CFLAGS) $(LIBS) -MM -MT tests/$* $< >tests/$*.d
+# $(CXX) $(CFLAGS) $(LIBS) -o $@ $(filter %.cc %.a, $^) $(LDFLAGS)
+#
+#-include tests/*.d
Reference in New Issue View Git Blame Copy Permalink