mindspore-Ecosystem
/
mindspore
21
0
Fork 27
Code Issues Pull Requests 6 Projects Releases Wiki Activity

!18935 add compile package instruction of fl_client to build.sh 

Merge pull request !18935 from zhoushan33/flclient0626_om
This commit is contained in:
i-robot 2021-06-29 01:46:01 +00:00 committed by Gitee
parent f8e16822b9 9aae26643b
commit 229d908dcf
11 changed files with 109 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
build.sh
View File

@ -538,6 +538,21 @@ build_lite_x86_64_jni_and_jar()
gradle build
cp ${LITE_JAVA_PATH}/java/common/build/libs/mindspore-lite-java-common.jar ${LITE_JAVA_PATH}/java/linux_x86/libs/
# build java fl_client
if [[ "X$is_train" = "Xon" ]]; then
cd ${LITE_JAVA_PATH}/java/fl_client
gradle clean
echo "--------------------building createFlatBuffers for fl_client------------------------"
gradle createFlatBuffers
echo "--------------------create FlatBuffers for fl_client success--------------------"
gradle build
gradle clearJar
echo "--------------------building flReleaseJar for fl_client------------------------"
gradle flReleaseJarX86 --rerun-tasks
echo "--------------------build jar for fl_client success ------------------------"
cp ${LITE_JAVA_PATH}/java/fl_client/build/libs/jarX86/mindspore-lite-java-flclient.jar ${BASEPATH}/output/tmp/${pkg_name}/runtime/lib/
fi
# build jar
cd ${LITE_JAVA_PATH}/java/linux_x86/
gradle clean
@ -777,6 +792,27 @@ build_aar() {
build_lite_arm32_and_jni
export MSLITE_ENABLE_NPU=${npu_bak}
# build java fl_client
local is_train=on
local train_so=${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/libmindspore-lite-train-jni.so
if [ ! -f "$train_so" ]; then
echo "not exist"
is_train=off
fi
if [[ "X$is_train" = "Xon" ]]; then
cd ${LITE_JAVA_PATH}/java/fl_client
gradle clean
echo "--------------------building createFlatBuffers for fl_client------------------------"
gradle createFlatBuffers
echo "--------------------create FlatBuffers for fl_client success--------------------"
gradle build
gradle clearJar
echo "--------------------building flReleaseJar for fl_client------------------------"
gradle flReleaseJarAAR --rerun-tasks
echo "--------------------build jar for fl_client success ------------------------"
cp ${LITE_JAVA_PATH}/java/fl_client/build/libs/jarAAR/mindspore-lite-java-flclient.jar ${LITE_JAVA_PATH}/java/app/libs
fi
cp ${LITE_JAVA_PATH}/java/common/build/libs/mindspore-lite-java-common.jar ${LITE_JAVA_PATH}/java/app/libs
cd ${LITE_JAVA_PATH}/java/app
gradle clean

15
mindspore/lite/java/java/fl_client/build.gradle
View File

@ -64,9 +64,22 @@ dependencies {
}
import io.netifi.flatbuffers.plugin.tasks.FlatBuffers
flatbuffers {
flatcPath = '../../../build/_deps/flatbuffers-src/_build/flatc'
String msLib = System.getenv("MSLIBS_CACHE_PATH")
FileTree msLibTree = fileTree(dir: msLib, include: ["flatbuffers_*/bin/flatc"]) //flatbuffers_*/bin/flatc
String msLibPath = ''
msLibTree.each {File file ->
msLibPath = file.toString()
}
if (msLibTree.empty) {
flatcPath = '../../../build/_deps/flatbuffers-src/_build/flatc'
} else {
flatcPath = msLibPath
}
println("-----------------flatc path: " + flatcPath + "-----------------")
}
task createFlatBuffers(type: FlatBuffers) {
println("-----------------executing task: createFlatBuffers-----------------")
inputDir = file("../../../../schema")

8
mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/CipherClient.java
View File

@ -260,7 +260,7 @@ public class CipherClient {
public FLClientStatus requestExchangeKeys() {
LOGGER.info(Common.addTag("[PairWiseMask] ==============request flID: " + localFLParameter.getFlID() + "=============="));
String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
LOGGER.info(Common.addTag("[PairWiseMask] ==============requestExchangeKeys url: " + url + "=============="));
genDHKeyPairs();
byte[] cPK = cKey.get(0);
@ -312,7 +312,7 @@ public class CipherClient {
}
public FLClientStatus getExchangeKeys() {
String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
LOGGER.info(Common.addTag("[PairWiseMask] ==============getExchangeKeys url: " + url + "=============="));
FlatBufferBuilder fbBuilder = new FlatBufferBuilder();
int id = fbBuilder.createString(localFLParameter.getFlID());
@ -377,7 +377,7 @@ public class CipherClient {
}
public FLClientStatus requestShareSecrets() throws Exception {
String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
LOGGER.info(Common.addTag("[PairWiseMask] ==============requestShareSecrets url: " + url + "=============="));
genIndividualSecret();
genEncryptExchangedKeys();
@ -447,7 +447,7 @@ public class CipherClient {
}
public FLClientStatus getShareSecrets() {
String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
LOGGER.info(Common.addTag("[PairWiseMask] ==============getShareSecrets url: " + url + "=============="));
FlatBufferBuilder fbBuilder = new FlatBufferBuilder();
int id = fbBuilder.createString(localFLParameter.getFlID());

22
mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/Common.java
View File

@ -22,13 +22,20 @@ import java.util.Date;
import java.util.List;
import java.util.Random;
import java.util.logging.Logger;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class Common {
public static final String LOG_TITLE = "<FLClient> ";
private static final Logger LOGGER = Logger.getLogger(Common.class.toString());
private static List<String> flNameTrustList = new ArrayList<>(Arrays.asList("lenet", "adbert"));
public static String generateUrl(boolean useElb, String ip, int port, int serverNum) {
public static String generateUrl(boolean useHttps, boolean useElb, String ip, int port, int serverNum) {
if (useHttps) {
ip = "https://" + ip + ":";
} else {
ip = "http://" + ip + ":";
}
String url;
if (useElb) {
Random rand = new Random();
@ -120,7 +127,7 @@ public class Common {
public static boolean checkPath(String path) {
boolean tag = true;
String [] paths = path.split(",");
String[] paths = path.split(",");
for (int i = 0; i < paths.length; i++) {
LOGGER.info(addTag("[check path]:" + paths[i]));
File file = new File(paths[i]);
@ -130,4 +137,15 @@ public class Common {
}
return tag;
}
public static boolean checkIP(String ip) {
String regex = "(25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])[.](25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])[.](25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])[.](25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])";
Pattern pattern = Pattern.compile(regex);
Matcher matcher = pattern.matcher(ip);
return matcher.matches();
}
public static boolean checkPort(int port) {
return port > 0 && port <= 65535;
}
}

8
mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLCommunication.java
View File

@ -27,6 +27,7 @@ import okhttp3.Response;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.io.IOException;
@ -43,6 +44,9 @@ import static com.mindspore.flclient.FLParameter.TIME_OUT;
public class FLCommunication implements IFLCommunication {
private static int timeOut;
private static boolean ssl = false;
private static String env;
private static SSLSocketFactory sslSocketFactory;
private static X509TrustManager x509TrustManager;
private FLParameter flParameter = FLParameter.getInstance();
private static final MediaType MEDIA_TYPE_JSON = MediaType.parse("applicatiom/json;charset=utf-8");
private static final Logger LOGGER = Logger.getLogger(FLCommunication.class.toString());
@ -57,10 +61,10 @@ public class FLCommunication implements IFLCommunication {
timeOut = TIME_OUT;
}
ssl = flParameter.isUseSSL();
client = getUnsafeOkHttpClient();
client = getOkHttpClient();
}
private static OkHttpClient getUnsafeOkHttpClient() {
private static OkHttpClient getOkHttpClient() {
X509TrustManager trustManager = new X509TrustManager() {
@Override

6
mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLLiteClient.java
View File

@ -159,7 +159,7 @@ public class FLLiteClient {
public FLClientStatus startFLJob() {
LOGGER.info(Common.addTag("[startFLJob] ====================================Verify server===================================="));
String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
LOGGER.info(Common.addTag("[startFLJob] ==============startFLJob url: " + url + "=============="));
StartFLJob startFLJob = StartFLJob.getInstance();
Date date = new Date();
@ -249,7 +249,7 @@ public class FLLiteClient {
}
public FLClientStatus updateModel() {
String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
LOGGER.info(Common.addTag("[updateModel] ==============updateModel url: " + url + "=============="));
UpdateModel updateModelBuf = UpdateModel.getInstance();
byte[] updateModelBuffer = updateModelBuf.getRequestUpdateFLJob(iteration, secureProtocol, trainDataSize);
@ -287,7 +287,7 @@ public class FLLiteClient {
}
public FLClientStatus getModel() {
String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
LOGGER.info(Common.addTag("[getModel] ===========getModel url: " + url + "=============="));
GetModel getModelBuf = GetModel.getInstance();
byte[] buffer = getModelBuf.getRequestGetModel(flParameter.getFlName(), iteration);

25
mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLParameter.java
View File

@ -22,8 +22,10 @@ public class FLParameter {
public static final int TIME_OUT = 100;
public static final int SLEEP_TIME = 1000;
private String hostName;
private String certPath;
private boolean useHttps = false;
private String trainDataset;
private String vocabFile = "null";
@ -84,6 +86,14 @@ public class FLParameter {
this.certPath = certPath;
}
public boolean isUseHttps() {
return useHttps;
}
public void setUseHttps(boolean useHttps) {
this.useHttps = useHttps;
}
public String getTrainDataset() {
if ("".equals(trainDataset) || trainDataset.isEmpty()) {
LOGGER.severe(Common.addTag("[flParameter] the parameter of <trainDataset> is null, please set it before use"));
@ -208,7 +218,12 @@ public class FLParameter {
}
public void setIp(String ip) {
this.ip = ip;
if (Common.checkIP(ip)) {
this.ip = ip;
} else {
LOGGER.severe(Common.addTag("[flParameter] the parameter of <ip> is not valid, please check it before set"));
throw new RuntimeException();
}
}
public boolean isUseSSL() {
@ -228,10 +243,14 @@ public class FLParameter {
}
public void setPort(int port) {
this.port = port;
if (Common.checkPort(port)) {
this.port = port;
} else {
LOGGER.severe(Common.addTag("[flParameter] the parameter of <port> is not valid, please check it before set"));
throw new RuntimeException();
}
}
public int getTimeOut() {
return timeOut;
}

2
mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/SyncFLJob.java
View File

@ -202,7 +202,7 @@ public class SyncFLJob {
}
flParameter.setUseSSL(useSSL);
FLCommunication flCommunication = FLCommunication.getInstance();
String url = Common.generateUrl(useElb, ip, port, serverNum);
String url = Common.generateUrl(flParameter.isUseHttps(), useElb, ip, port, serverNum);
LOGGER.info(Common.addTag("[getModel] ===========getModel url: " + url + "=============="));
GetModel getModelBuf = GetModel.getInstance();
byte[] buffer = getModelBuf.getRequestGetModel(flName, 0);

2
mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/cipher/ClientListReq.java
View File

@ -64,7 +64,7 @@ public class ClientListReq {
}
public FLClientStatus getClientList(int iteration, List<String> u3ClientList, List<DecryptShareSecrets> decryptSecretsList, List<EncryptShare> returnShareList, Map<String, byte[]> cuvKeys) {
String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
LOGGER.info(Common.addTag("[PairWiseMask] ==============getClientList url: " + url + "=============="));
FlatBufferBuilder builder = new FlatBufferBuilder();
int id = builder.createString(localFLParameter.getFlID());

2
mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/cipher/ReconstructSecretReq.java
View File

@ -56,7 +56,7 @@ public class ReconstructSecretReq {
}
public FLClientStatus sendReconstructSecret(List<DecryptShareSecrets> decryptShareSecretsList, List<String> u3ClientList, int iteration) {
String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum());
LOGGER.info(Common.addTag("[PairWiseMask] ==============sendReconstructSecret url: " + url + "=============="));
FlatBufferBuilder builder = new FlatBufferBuilder();
int desFlId = builder.createString(localFLParameter.getFlID());

1
mindspore/lite/java/java/linux_x86/build.gradle
View File

@ -2,6 +2,7 @@ apply plugin: 'java'
dependencies {
implementation fileTree(dir: "libs", include: ["*.jar"])
implementation project(':common')
}
archivesBaseName = 'mindspore-lite-java'