p60975123
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!28643 fix issue I4NAFI 

Merge pull request !28643 from tan-wei-cheng/develop-fix
This commit is contained in:
i-robot 2022-01-06 12:59:48 +00:00 committed by Gitee
parent de34e90e4d 90dbadf889
commit 6cfed47430
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
16 changed files with 63 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
mindspore/ccsrc/ps/constants.h
View File

@ -186,12 +186,8 @@ constexpr char kCaCert[] = "ca.crt";
constexpr char kColon = ':';
const std::map<std::string, size_t> kCiphers = {
{"ECDHE-RSA-AES128-GCM-SHA256", 0}, {"ECDHE-ECDSA-AES128-GCM-SHA256", 1}, {"ECDHE-RSA-AES256-GCM-SHA384", 2},
{"ECDHE-ECDSA-AES256-GCM-SHA384", 3}, {"DHE-RSA-AES128-GCM-SHA256", 4}, {"DHE-DSS-AES128-GCM-SHA256", 5},
{"DHE-RSA-AES256-GCM-SHA384", 6}, {"DHE-DSS-AES256-GCM-SHA384", 7}, {"DHE-PSK-AES128-GCM-SHA256", 8},
{"DHE-PSK-AES256-GCM-SHA384", 9}, {"DHE-PSK-CHACHA20-POLY1305", 10}, {"ECDHE-RSA-CHACHA20-POLY1305", 11},
{"ECDHE-PSK-CHACHA20-POLY1305", 12}, {"DHE-RSA-AES128-CCM", 13}, {"DHE-RSA-AES256-CCM", 14},
{"DHE-RSA-CHACHA20-POLY1305", 15}, {"DHE-PSK-AES128-CCM", 16}, {"DHE-PSK-AES256-CCM", 17},
{"ECDHE-ECDSA-AES128-CCM", 18}, {"ECDHE-ECDSA-AES256-CCM", 19}, {"ECDHE-ECDSA-CHACHA20-POLY1305", 20}};
{"ECDHE-ECDSA-AES256-GCM-SHA384", 3}, {"ECDHE-RSA-CHACHA20-POLY1305", 4}, {"ECDHE-PSK-CHACHA20-POLY1305", 5},
{"ECDHE-ECDSA-AES128-CCM", 6}, {"ECDHE-ECDSA-AES256-CCM", 7}, {"ECDHE-ECDSA-CHACHA20-POLY1305", 8}};
#ifdef __APPLE__
using DataPtr = std::shared_ptr<unsigned char>;

8
mindspore/ccsrc/ps/core/abstract_node.cc
View File

@ -38,10 +38,10 @@ void AbstractNode::Register(const std::shared_ptr<TcpClient> &client) {
MS_LOG(INFO) << "The node role:" << CommUtil::NodeRoleToString(node_info_.node_role_)
<< " the node id:" << node_info_.node_id_ << " begin to register to the scheduler!";
if (!SendMessageSync(client, message_meta, Protos::PROTOBUF, register_message.SerializeAsString().data(),
register_message.ByteSizeLong())) {
MS_LOG(EXCEPTION) << "The node role:" << CommUtil::NodeRoleToString(node_info_.node_role_)
<< " the node id:" << node_info_.node_id_ << " register timeout!";
if (!SendMessageAsync(client, message_meta, Protos::PROTOBUF, register_message.SerializeAsString().data(),
register_message.ByteSizeLong())) {
MS_LOG(ERROR) << "The node role:" << CommUtil::NodeRoleToString(node_info_.node_role_)
<< " the node id:" << node_info_.node_id_ << " register timeout!";
} else {
MS_LOG(INFO) << "The node role:" << CommUtil::NodeRoleToString(node_info_.node_role_)
<< " the node id:" << node_info_.node_id_ << " send register success!";

23
mindspore/ccsrc/ps/core/comm_util.cc
View File

@ -17,6 +17,8 @@
#include "ps/core/comm_util.h"
#include <arpa/inet.h>
#include <unistd.h>
#include <sys/stat.h>
#include <cstdio>
#include <cstdlib>
#include <cstring>
@ -229,6 +231,27 @@ bool CommUtil::IsFileEmpty(const std::string &file) {
return str.empty();
}
bool CommUtil::CreateDirectory(const std::string &directoryPath) {
uint32_t dirPathLen = directoryPath.length();
uint32_t MAX_PATH_LEN = 512;
if (dirPathLen > MAX_PATH_LEN) {
return false;
}
char tmpDirPath[MAX_PATH_LEN] = {0};
for (uint32_t i = 0; i < dirPathLen; ++i) {
tmpDirPath[i] = directoryPath[i];
if (tmpDirPath[i] == '/') {
if (access(tmpDirPath, 0) != 0) {
int32_t ret = mkdir(tmpDirPath, S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
if (ret != 0) {
return false;
}
}
}
}
return true;
}
std::string CommUtil::ClusterStateToString(const ClusterState &state) {
MS_LOG(DEBUG) << "The cluster state:" << state;
if (state < SizeToInt(kClusterState.size())) {

1
mindspore/ccsrc/ps/core/comm_util.h
View File

@ -134,6 +134,7 @@ class CommUtil {
static bool verifyExtendedAttributes(const X509 *caCert);
static void verifyCertPipeline(const X509 *caCert, const X509 *subCert);
static bool checkCRLTime(const std::string &crlPath);
static bool CreateDirectory(const std::string &directoryPath);
private:
static std::random_device rd;

6
mindspore/ccsrc/ps/core/file_configuration.cc
View File

@ -21,7 +21,11 @@ namespace ps {
namespace core {
bool FileConfiguration::Initialize() {
if (!CommUtil::IsFileExists(file_path_)) {
MS_LOG(INFO) << "The file path:" << file_path_ << " is not exist.";
MS_LOG(WARNING) << "The file path:" << file_path_ << " is not exist.";
if (CommUtil::CreateDirectory(file_path_)) {
MS_LOG(INFO) << "Create Directory :" << file_path_ << " success.";
}
return false;
}

5
mindspore/ccsrc/ps/core/node_manager.cc
View File

@ -359,10 +359,11 @@ bool NodeManager::IsNodeRegistered(const std::string &node_id) {
return false;
}
const NodeInfo &NodeManager::QueryNodeInfo(const std::string &node_id) const {
const NodeInfo NodeManager::QueryNodeInfo(const std::string &node_id) const {
auto iter = registered_nodes_info_.find(node_id);
if (iter == registered_nodes_info_.end()) {
MS_LOG(EXCEPTION) << "Cannot find node of id: " << node_id;
MS_LOG(DEBUG) << "Cannot find node of id: " << node_id;
return NodeInfo();
}
return iter->second;
}

2
mindspore/ccsrc/ps/core/node_manager.h
View File

@ -124,7 +124,7 @@ class NodeManager {
void setPersistCallback(const OnPersist &onPersist);
// Query node information by node id.
const NodeInfo &QueryNodeInfo(const std::string &node_id) const;
const NodeInfo QueryNodeInfo(const std::string &node_id) const;
// Determine whether the node is in persistent.
bool IsNodePersisting(const std::string &node_id) const;

21
mindspore/ccsrc/ps/core/scheduler_node.cc
View File

@ -144,15 +144,18 @@ void SchedulerNode::ProcessHeartbeat(const std::shared_ptr<TcpServer> &server,
HeartbeatRespMessage heartbeat_resp_message;
heartbeat_resp_message.set_persistent_cmd(PersistentCommand::DEFAULT);
NodeRole node_role = (node_manager_.QueryNodeInfo(node_id)).node_role_;
// The worker role does not support disaster recovery for the time being.
if (node_role == NodeRole::SERVER && persistent_cmd_ == PersistentCommand::BEGIN_PERSIST) {
if (!node_manager_.IsNodePersisting(node_id)) {
heartbeat_resp_message.set_persistent_cmd(PersistentCommand::BEGIN_PERSIST);
node_manager_.AddPersistingNode(node_id);
}
if (node_manager_.IsAllNodeInPersisting()) {
persistent_cmd_ = PersistentCommand::DEFAULT;
NodeInfo nodeInfo = node_manager_.QueryNodeInfo(node_id);
if (nodeInfo.node_id_ != "") {
// The worker role does not support disaster recovery for the time being.
NodeRole node_role = nodeInfo.node_role_;
if (node_role == NodeRole::SERVER && persistent_cmd_ == PersistentCommand::BEGIN_PERSIST) {
if (!node_manager_.IsNodePersisting(node_id)) {
heartbeat_resp_message.set_persistent_cmd(PersistentCommand::BEGIN_PERSIST);
node_manager_.AddPersistingNode(node_id);
}
if (node_manager_.IsAllNodeInPersisting()) {
persistent_cmd_ = PersistentCommand::DEFAULT;
}
}
}

3
mindspore/ccsrc/ps/core/server_node.cc
View File

@ -24,9 +24,6 @@ bool ServerNode::Start(const uint32_t &timeout) {
MS_LOG(INFO) << "[Server start]: 1. Begin to start server node!";
Initialize();
Register(client_to_scheduler_);
if (node_info_.rank_id_ == UINT32_MAX) {
MS_LOG(EXCEPTION) << "Register to scheduler failed, so finish the node.";
}
MS_LOG(INFO) << "[Server start]: 4. The node role:" << CommUtil::NodeRoleToString(node_info_.node_role_)
<< " the node id:" << node_info_.node_id_ << " successfully registered to the scheduler!";

3
mindspore/ccsrc/ps/core/worker_node.cc
View File

@ -24,9 +24,6 @@ bool WorkerNode::Start(const uint32_t &timeout) {
MS_LOG(INFO) << "[Worker start]: 1. Begin to start worker node!";
Initialize();
Register(client_to_scheduler_);
if (node_info_.rank_id_ == UINT32_MAX) {
MS_LOG(EXCEPTION) << "Register to scheduler failed, so finish the node.";
}
MS_LOG(INFO) << "[Worker start]: 4. The node role:" << CommUtil::NodeRoleToString(node_info_.node_role_)
<< " the node id:" << node_info_.node_id_ << " successfully registered to the scheduler!";

4
tests/st/fl/albert/config.json
View File

@ -8,9 +8,9 @@
"crl_path": "",
"client_cert_path": "client.p12",
"ca_cert_path": "ca.crt",
"cipher_list": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-PSK-AES128-GCM-SHA256:DHE-PSK-AES256-GCM-SHA384:DHE-PSK-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-PSK-CHACHA20-POLY1305:DHE-RSA-AES128-CCM:DHE-RSA-AES256-CCM:DHE-RSA-CHACHA20-POLY1305:DHE-PSK-AES128-CCM:DHE-PSK-AES256-CCM:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-CHACHA20-POLY1305",
"cipher_list": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-PSK-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-CHACHA20-POLY1305",
"cert_expire_warning_time_in_day": 90,
"connection_num":10000,
"connection_num": 10000,
"metrics": {
"storage_type": 1,
"storage_file_path": "metrics.json"

4
tests/st/fl/cross_silo_faster_rcnn/config.json
View File

@ -8,9 +8,9 @@
"crl_path": "",
"client_cert_path": "client.p12",
"ca_cert_path": "ca.crt",
"cipher_list": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-PSK-AES128-GCM-SHA256:DHE-PSK-AES256-GCM-SHA384:DHE-PSK-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-PSK-CHACHA20-POLY1305:DHE-RSA-AES128-CCM:DHE-RSA-AES256-CCM:DHE-RSA-CHACHA20-POLY1305:DHE-PSK-AES128-CCM:DHE-PSK-AES256-CCM:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-CHACHA20-POLY1305",
"cipher_list": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-PSK-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-CHACHA20-POLY1305",
"cert_expire_warning_time_in_day": 90,
"connection_num":10000,
"connection_num": 10000,
"metrics": {
"storage_type": 1,
"storage_file_path": "metrics.json"

4
tests/st/fl/cross_silo_femnist/config.json
View File

@ -8,9 +8,9 @@
"crl_path": "",
"client_cert_path": "client.p12",
"ca_cert_path": "ca.crt",
"cipher_list": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-PSK-AES128-GCM-SHA256:DHE-PSK-AES256-GCM-SHA384:DHE-PSK-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-PSK-CHACHA20-POLY1305:DHE-RSA-AES128-CCM:DHE-RSA-AES256-CCM:DHE-RSA-CHACHA20-POLY1305:DHE-PSK-AES128-CCM:DHE-PSK-AES256-CCM:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-CHACHA20-POLY1305",
"cipher_list": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-PSK-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-CHACHA20-POLY1305",
"cert_expire_warning_time_in_day": 90,
"connection_num":10000,
"connection_num": 10000,
"metrics": {
"storage_type": 1,
"storage_file_path": "metrics.json"

4
tests/st/fl/cross_silo_lenet/config.json
View File

@ -8,9 +8,9 @@
"crl_path": "",
"client_cert_path": "client.p12",
"ca_cert_path": "ca.crt",
"cipher_list": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-PSK-AES128-GCM-SHA256:DHE-PSK-AES256-GCM-SHA384:DHE-PSK-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-PSK-CHACHA20-POLY1305:DHE-RSA-AES128-CCM:DHE-RSA-AES256-CCM:DHE-RSA-CHACHA20-POLY1305:DHE-PSK-AES128-CCM:DHE-PSK-AES256-CCM:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-CHACHA20-POLY1305",
"cipher_list": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-PSK-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-CHACHA20-POLY1305",
"cert_expire_warning_time_in_day": 90,
"connection_num":10000,
"connection_num": 10000,
"metrics": {
"storage_type": 1,
"storage_file_path": "metrics.json"

4
tests/st/fl/hybrid_lenet/config.json
View File

@ -8,9 +8,9 @@
"crl_path": "",
"client_cert_path": "client.p12",
"ca_cert_path": "ca.crt",
"cipher_list": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-PSK-AES128-GCM-SHA256:DHE-PSK-AES256-GCM-SHA384:DHE-PSK-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-PSK-CHACHA20-POLY1305:DHE-RSA-AES128-CCM:DHE-RSA-AES256-CCM:DHE-RSA-CHACHA20-POLY1305:DHE-PSK-AES128-CCM:DHE-PSK-AES256-CCM:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-CHACHA20-POLY1305",
"cipher_list": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-PSK-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-CHACHA20-POLY1305",
"cert_expire_warning_time_in_day": 90,
"connection_num":10000,
"connection_num": 10000,
"metrics": {
"storage_type": 1,
"storage_file_path": "metrics.json"

4
tests/st/fl/mobile/config.json
View File

@ -8,9 +8,9 @@
"crl_path": "",
"client_cert_path": "client.p12",
"ca_cert_path": "ca.crt",
"cipher_list": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:DHE-DSS-AES256-GCM-SHA384:DHE-PSK-AES128-GCM-SHA256:DHE-PSK-AES256-GCM-SHA384:DHE-PSK-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-PSK-CHACHA20-POLY1305:DHE-RSA-AES128-CCM:DHE-RSA-AES256-CCM:DHE-RSA-CHACHA20-POLY1305:DHE-PSK-AES128-CCM:DHE-PSK-AES256-CCM:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-CHACHA20-POLY1305",
"cipher_list": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-PSK-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-CCM:ECDHE-ECDSA-AES256-CCM:ECDHE-ECDSA-CHACHA20-POLY1305",
"cert_expire_warning_time_in_day": 90,
"connection_num":10000,
"connection_num": 10000,
"metrics": {
"storage_type": 1,
"storage_file_path": "metrics.json"