From 9aae26643bc69c3b8f8ee60b973e5f52269b3759 Mon Sep 17 00:00:00 2001 From: zhoushan Date: Sat, 26 Jun 2021 17:22:49 +0800 Subject: [PATCH] add compile package instruction of fl_client to build.sh --- build.sh | 36 +++++++++++++++++++ .../lite/java/java/fl_client/build.gradle | 15 +++++++- .../com/mindspore/flclient/CipherClient.java | 8 ++--- .../java/com/mindspore/flclient/Common.java | 22 ++++++++++-- .../mindspore/flclient/FLCommunication.java | 8 +++-- .../com/mindspore/flclient/FLLiteClient.java | 6 ++-- .../com/mindspore/flclient/FLParameter.java | 25 +++++++++++-- .../com/mindspore/flclient/SyncFLJob.java | 2 +- .../flclient/cipher/ClientListReq.java | 2 +- .../flclient/cipher/ReconstructSecretReq.java | 2 +- .../lite/java/java/linux_x86/build.gradle | 1 + 11 files changed, 109 insertions(+), 18 deletions(-) diff --git a/build.sh b/build.sh index 347e3ff555d..c97be425462 100755 --- a/build.sh +++ b/build.sh @@ -538,6 +538,21 @@ build_lite_x86_64_jni_and_jar() gradle build cp ${LITE_JAVA_PATH}/java/common/build/libs/mindspore-lite-java-common.jar ${LITE_JAVA_PATH}/java/linux_x86/libs/ + # build java fl_client + if [[ "X$is_train" = "Xon" ]]; then + cd ${LITE_JAVA_PATH}/java/fl_client + gradle clean + echo "--------------------building createFlatBuffers for fl_client------------------------" + gradle createFlatBuffers + echo "--------------------create FlatBuffers for fl_client success--------------------" + gradle build + gradle clearJar + echo "--------------------building flReleaseJar for fl_client------------------------" + gradle flReleaseJarX86 --rerun-tasks + echo "--------------------build jar for fl_client success ------------------------" + cp ${LITE_JAVA_PATH}/java/fl_client/build/libs/jarX86/mindspore-lite-java-flclient.jar ${BASEPATH}/output/tmp/${pkg_name}/runtime/lib/ + fi + # build jar cd ${LITE_JAVA_PATH}/java/linux_x86/ gradle clean @@ -778,6 +793,27 @@ build_aar() { build_lite_arm32_and_jni export MSLITE_ENABLE_NPU=${npu_bak} + # build java fl_client + local is_train=on + local train_so=${LITE_JAVA_PATH}/java/app/libs/arm64-v8a/libmindspore-lite-train-jni.so + if [ ! -f "$train_so" ]; then + echo "not exist" + is_train=off + fi + if [[ "X$is_train" = "Xon" ]]; then + cd ${LITE_JAVA_PATH}/java/fl_client + gradle clean + echo "--------------------building createFlatBuffers for fl_client------------------------" + gradle createFlatBuffers + echo "--------------------create FlatBuffers for fl_client success--------------------" + gradle build + gradle clearJar + echo "--------------------building flReleaseJar for fl_client------------------------" + gradle flReleaseJarAAR --rerun-tasks + echo "--------------------build jar for fl_client success ------------------------" + cp ${LITE_JAVA_PATH}/java/fl_client/build/libs/jarAAR/mindspore-lite-java-flclient.jar ${LITE_JAVA_PATH}/java/app/libs + fi + cp ${LITE_JAVA_PATH}/java/common/build/libs/mindspore-lite-java-common.jar ${LITE_JAVA_PATH}/java/app/libs cd ${LITE_JAVA_PATH}/java/app gradle clean diff --git a/mindspore/lite/java/java/fl_client/build.gradle b/mindspore/lite/java/java/fl_client/build.gradle index 7433a4c6261..9e1e2fedeb3 100644 --- a/mindspore/lite/java/java/fl_client/build.gradle +++ b/mindspore/lite/java/java/fl_client/build.gradle @@ -64,9 +64,22 @@ dependencies { } import io.netifi.flatbuffers.plugin.tasks.FlatBuffers + flatbuffers { - flatcPath = '../../../build/_deps/flatbuffers-src/_build/flatc' + String msLib = System.getenv("MSLIBS_CACHE_PATH") + FileTree msLibTree = fileTree(dir: msLib, include: ["flatbuffers_*/bin/flatc"]) //flatbuffers_*/bin/flatc + String msLibPath = '' + msLibTree.each {File file -> + msLibPath = file.toString() + } + if (msLibTree.empty) { + flatcPath = '../../../build/_deps/flatbuffers-src/_build/flatc' + } else { + flatcPath = msLibPath + } + println("-----------------flatc path: " + flatcPath + "-----------------") } + task createFlatBuffers(type: FlatBuffers) { println("-----------------executing task: createFlatBuffers-----------------") inputDir = file("../../../../schema") diff --git a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/CipherClient.java b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/CipherClient.java index f069d2200b2..a0e03561e2d 100644 --- a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/CipherClient.java +++ b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/CipherClient.java @@ -260,7 +260,7 @@ public class CipherClient { public FLClientStatus requestExchangeKeys() { LOGGER.info(Common.addTag("[PairWiseMask] ==============request flID: " + localFLParameter.getFlID() + "==============")); - String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); + String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); LOGGER.info(Common.addTag("[PairWiseMask] ==============requestExchangeKeys url: " + url + "==============")); genDHKeyPairs(); byte[] cPK = cKey.get(0); @@ -312,7 +312,7 @@ public class CipherClient { } public FLClientStatus getExchangeKeys() { - String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); + String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); LOGGER.info(Common.addTag("[PairWiseMask] ==============getExchangeKeys url: " + url + "==============")); FlatBufferBuilder fbBuilder = new FlatBufferBuilder(); int id = fbBuilder.createString(localFLParameter.getFlID()); @@ -377,7 +377,7 @@ public class CipherClient { } public FLClientStatus requestShareSecrets() throws Exception { - String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); + String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); LOGGER.info(Common.addTag("[PairWiseMask] ==============requestShareSecrets url: " + url + "==============")); genIndividualSecret(); genEncryptExchangedKeys(); @@ -447,7 +447,7 @@ public class CipherClient { } public FLClientStatus getShareSecrets() { - String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); + String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); LOGGER.info(Common.addTag("[PairWiseMask] ==============getShareSecrets url: " + url + "==============")); FlatBufferBuilder fbBuilder = new FlatBufferBuilder(); int id = fbBuilder.createString(localFLParameter.getFlID()); diff --git a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/Common.java b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/Common.java index 474f6131421..2ce24c8ce92 100644 --- a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/Common.java +++ b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/Common.java @@ -22,13 +22,20 @@ import java.util.Date; import java.util.List; import java.util.Random; import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; public class Common { public static final String LOG_TITLE = " "; private static final Logger LOGGER = Logger.getLogger(Common.class.toString()); private static List flNameTrustList = new ArrayList<>(Arrays.asList("lenet", "adbert")); - public static String generateUrl(boolean useElb, String ip, int port, int serverNum) { + public static String generateUrl(boolean useHttps, boolean useElb, String ip, int port, int serverNum) { + if (useHttps) { + ip = "https://" + ip + ":"; + } else { + ip = "http://" + ip + ":"; + } String url; if (useElb) { Random rand = new Random(); @@ -120,7 +127,7 @@ public class Common { public static boolean checkPath(String path) { boolean tag = true; - String [] paths = path.split(","); + String[] paths = path.split(","); for (int i = 0; i < paths.length; i++) { LOGGER.info(addTag("[check path]:" + paths[i])); File file = new File(paths[i]); @@ -130,4 +137,15 @@ public class Common { } return tag; } + + public static boolean checkIP(String ip) { + String regex = "(25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])[.](25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])[.](25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])[.](25[0-4]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])"; + Pattern pattern = Pattern.compile(regex); + Matcher matcher = pattern.matcher(ip); + return matcher.matches(); + } + + public static boolean checkPort(int port) { + return port > 0 && port <= 65535; + } } diff --git a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLCommunication.java b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLCommunication.java index 8aa65add4d1..ea4ed04136d 100644 --- a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLCommunication.java +++ b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLCommunication.java @@ -27,6 +27,7 @@ import okhttp3.Response; import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSession; +import javax.net.ssl.SSLSocketFactory; import javax.net.ssl.TrustManager; import javax.net.ssl.X509TrustManager; import java.io.IOException; @@ -43,6 +44,9 @@ import static com.mindspore.flclient.FLParameter.TIME_OUT; public class FLCommunication implements IFLCommunication { private static int timeOut; private static boolean ssl = false; + private static String env; + private static SSLSocketFactory sslSocketFactory; + private static X509TrustManager x509TrustManager; private FLParameter flParameter = FLParameter.getInstance(); private static final MediaType MEDIA_TYPE_JSON = MediaType.parse("applicatiom/json;charset=utf-8"); private static final Logger LOGGER = Logger.getLogger(FLCommunication.class.toString()); @@ -57,10 +61,10 @@ public class FLCommunication implements IFLCommunication { timeOut = TIME_OUT; } ssl = flParameter.isUseSSL(); - client = getUnsafeOkHttpClient(); + client = getOkHttpClient(); } - private static OkHttpClient getUnsafeOkHttpClient() { + private static OkHttpClient getOkHttpClient() { X509TrustManager trustManager = new X509TrustManager() { @Override diff --git a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLLiteClient.java b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLLiteClient.java index e878af23d31..52e1f6d7d94 100644 --- a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLLiteClient.java +++ b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLLiteClient.java @@ -159,7 +159,7 @@ public class FLLiteClient { public FLClientStatus startFLJob() { LOGGER.info(Common.addTag("[startFLJob] ====================================Verify server====================================")); - String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); + String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); LOGGER.info(Common.addTag("[startFLJob] ==============startFLJob url: " + url + "==============")); StartFLJob startFLJob = StartFLJob.getInstance(); Date date = new Date(); @@ -249,7 +249,7 @@ public class FLLiteClient { } public FLClientStatus updateModel() { - String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); + String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); LOGGER.info(Common.addTag("[updateModel] ==============updateModel url: " + url + "==============")); UpdateModel updateModelBuf = UpdateModel.getInstance(); byte[] updateModelBuffer = updateModelBuf.getRequestUpdateFLJob(iteration, secureProtocol, trainDataSize); @@ -287,7 +287,7 @@ public class FLLiteClient { } public FLClientStatus getModel() { - String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); + String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); LOGGER.info(Common.addTag("[getModel] ===========getModel url: " + url + "==============")); GetModel getModelBuf = GetModel.getInstance(); byte[] buffer = getModelBuf.getRequestGetModel(flParameter.getFlName(), iteration); diff --git a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLParameter.java b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLParameter.java index 668dadad39a..6a2bad590c0 100644 --- a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLParameter.java +++ b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/FLParameter.java @@ -22,8 +22,10 @@ public class FLParameter { public static final int TIME_OUT = 100; public static final int SLEEP_TIME = 1000; + private String hostName; private String certPath; + private boolean useHttps = false; private String trainDataset; private String vocabFile = "null"; @@ -84,6 +86,14 @@ public class FLParameter { this.certPath = certPath; } + public boolean isUseHttps() { + return useHttps; + } + + public void setUseHttps(boolean useHttps) { + this.useHttps = useHttps; + } + public String getTrainDataset() { if ("".equals(trainDataset) || trainDataset.isEmpty()) { LOGGER.severe(Common.addTag("[flParameter] the parameter of is null, please set it before use")); @@ -208,7 +218,12 @@ public class FLParameter { } public void setIp(String ip) { - this.ip = ip; + if (Common.checkIP(ip)) { + this.ip = ip; + } else { + LOGGER.severe(Common.addTag("[flParameter] the parameter of is not valid, please check it before set")); + throw new RuntimeException(); + } } public boolean isUseSSL() { @@ -228,10 +243,14 @@ public class FLParameter { } public void setPort(int port) { - this.port = port; + if (Common.checkPort(port)) { + this.port = port; + } else { + LOGGER.severe(Common.addTag("[flParameter] the parameter of is not valid, please check it before set")); + throw new RuntimeException(); + } } - public int getTimeOut() { return timeOut; } diff --git a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/SyncFLJob.java b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/SyncFLJob.java index cb91b0a3012..cc419312162 100644 --- a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/SyncFLJob.java +++ b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/SyncFLJob.java @@ -202,7 +202,7 @@ public class SyncFLJob { } flParameter.setUseSSL(useSSL); FLCommunication flCommunication = FLCommunication.getInstance(); - String url = Common.generateUrl(useElb, ip, port, serverNum); + String url = Common.generateUrl(flParameter.isUseHttps(), useElb, ip, port, serverNum); LOGGER.info(Common.addTag("[getModel] ===========getModel url: " + url + "==============")); GetModel getModelBuf = GetModel.getInstance(); byte[] buffer = getModelBuf.getRequestGetModel(flName, 0); diff --git a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/cipher/ClientListReq.java b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/cipher/ClientListReq.java index 55e27265306..fd2b4fbe2e4 100644 --- a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/cipher/ClientListReq.java +++ b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/cipher/ClientListReq.java @@ -64,7 +64,7 @@ public class ClientListReq { } public FLClientStatus getClientList(int iteration, List u3ClientList, List decryptSecretsList, List returnShareList, Map cuvKeys) { - String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); + String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); LOGGER.info(Common.addTag("[PairWiseMask] ==============getClientList url: " + url + "==============")); FlatBufferBuilder builder = new FlatBufferBuilder(); int id = builder.createString(localFLParameter.getFlID()); diff --git a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/cipher/ReconstructSecretReq.java b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/cipher/ReconstructSecretReq.java index 044be955bc1..5e7d05ace7b 100644 --- a/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/cipher/ReconstructSecretReq.java +++ b/mindspore/lite/java/java/fl_client/src/main/java/com/mindspore/flclient/cipher/ReconstructSecretReq.java @@ -56,7 +56,7 @@ public class ReconstructSecretReq { } public FLClientStatus sendReconstructSecret(List decryptShareSecretsList, List u3ClientList, int iteration) { - String url = Common.generateUrl(flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); + String url = Common.generateUrl(flParameter.isUseHttps(), flParameter.isUseElb(), flParameter.getIp(), flParameter.getPort(), flParameter.getServerNum()); LOGGER.info(Common.addTag("[PairWiseMask] ==============sendReconstructSecret url: " + url + "==============")); FlatBufferBuilder builder = new FlatBufferBuilder(); int desFlId = builder.createString(localFLParameter.getFlID()); diff --git a/mindspore/lite/java/java/linux_x86/build.gradle b/mindspore/lite/java/java/linux_x86/build.gradle index 5d3c76b9699..a28c4cf3550 100644 --- a/mindspore/lite/java/java/linux_x86/build.gradle +++ b/mindspore/lite/java/java/linux_x86/build.gradle @@ -2,6 +2,7 @@ apply plugin: 'java' dependencies { implementation fileTree(dir: "libs", include: ["*.jar"]) + implementation project(':common') } archivesBaseName = 'mindspore-lite-java'