python api

.jenkins/check/config/filter_pylint.txt

@@ -155,6 +155,7 @@
 "mindspore/mindspore/ccsrc/plugin/device/cpu/kernel/nnacl/experimental/HPC-generator/generator.py"                                         "global-variable-undefined"
 "mindspore/mindspore/lite/python/api/context.py"                                                         "protected-access"
 "mindspore/mindspore/lite/python/api/model.py"                                                           "protected-access"
+"mindspore/mindspore/lite/python/api/model.py"                                                           "len-as-condition"
 "mindspore/mindspore/lite/python/api/tensor.py"                                                          "protected-access"
 "mindspore/mindspore/lite/test"                                                                          "missing-docstring"
 "mindspore/mindspore/lite/test"                                                                          "unused-variable"

mindspore/lite/python/api/model.py

@@ -537,10 +537,12 @@ class ModelParallelRunner:
                 raise TypeError(f"outputs element must be Tensor, but got "
                                 f"{type(element)} at index {i}.")
             _outputs.append(element._tensor)
-
-        ret = self._model.predict(_inputs, _outputs, None, None)
-        if not ret.IsOk():
-            raise RuntimeError(f"predict failed! Error is {ret.ToString()}")
+        predict_output = self._model.predict(_inputs, _outputs, None, None)
+        if not isinstance(predict_output, list) or len(predict_output) == 0:
+            raise RuntimeError(f"predict failed!")
+        outputs.clear()
+        for i, element in enumerate(predict_output):
+            outputs.append(Tensor(element))
 
     def get_inputs(self):
         """

mindspore/lite/python/src/model_pybind.cc

@@ -114,7 +114,15 @@ void ModelPyBind(const py::module &m) {
     .def("init", &ModelParallelRunner::Init)
     .def("get_inputs", &ModelParallelRunner::GetInputs)
     .def("get_outputs", &ModelParallelRunner::GetOutputs)
-    .def("predict", &ModelParallelRunner::Predict);
+    .def("predict", [](ModelParallelRunner &runner, const std::vector<MSTensor> &inputs, std::vector<MSTensor> *outputs,
+                       const MSKernelCallBack &before = nullptr, const MSKernelCallBack &after = nullptr) {
+      auto status = runner.Predict(inputs, outputs, before, after);
+      if (status != kSuccess) {
+        std::vector<MSTensor> empty;
+        return empty;
+      }
+      return *outputs;
+    });
 #endif
 }
 }  // namespace mindspore::lite

mindspore/lite/src/runtime/cxx_api/model_pool/model_pool.cc

@@ -29,8 +29,8 @@ constexpr int kNumDeviceInfo = 2;
 constexpr int kNumIndex = 2;
 constexpr int kNumCoreDataLen = 3;
 constexpr int kNumMaxTaskQueueSize = 1000;
-constexpr int kNumPhysicalCoreThreshold = 32;
-constexpr int kDefaultWorkerNumPerPhysicalCpu = 4;
+constexpr int kNumPhysicalCoreThreshold = 16;
+constexpr int kDefaultWorkerNumPerPhysicalCpu = 2;
 constexpr int kDefaultThreadsNum = 8;
 constexpr int kInvalidNumaId = -1;
 int GetCoreNum() {
@@ -108,7 +108,7 @@ int GetDefaultThreadNum() {
     return 0;
   }
   auto physical_core_size = physical_core_lite.size();
-  if (physical_core_lite.size() < kNumPhysicalCoreThreshold) {
+  if (physical_core_lite.size() <= kNumPhysicalCoreThreshold) {
     return physical_core_size / kDefaultWorkerNumPerPhysicalCpu;
   } else {
     return kDefaultThreadsNum;
@@ -566,9 +566,8 @@ std::vector<MSTensor> ModelPool::GetOutputs() {
     return {};
   }
   for (size_t i = 0; i < model_pool_outputs_.size(); i++) {
-    auto tensor =
-      mindspore::MSTensor::CreateTensor(model_pool_outputs_.at(i).Name(), model_pool_outputs_.at(i).DataType(),
-                                        model_pool_outputs_.at(i).Shape(), nullptr, 0);
+    auto tensor = mindspore::MSTensor::CreateTensor(model_pool_outputs_.at(i).Name(),
+                                                    model_pool_outputs_.at(i).DataType(), {}, nullptr, 0);
     if (tensor == nullptr) {
       MS_LOG(ERROR) << "create tensor failed.";
       return {};

mindspore/lite/src/runtime/cxx_api/model_pool/model_worker.cc

@@ -178,7 +178,7 @@ Status ModelWorker::Predict(const std::vector<MSTensor> &inputs, std::vector<MST
   bool need_copy_output = true;
   auto model_output = model_->GetOutputs();
   for (size_t i = 0; i < outputs->size(); i++) {
-    if (outputs->at(i).MutableData() != nullptr) {
+    if (outputs->at(i).Data() != nullptr) {
       /* user set graph-output-tensor from outside */
       model_output[i].SetData(outputs->at(i).MutableData());
       model_output[i].SetAllocator(nullptr);

mindspore/lite/test/st/python/test_inference.py

@@ -98,3 +98,24 @@ def test_server_inference_01():
     model_parallel_runner.predict(inputs, outputs)
     data = outputs[0].get_data_to_numpy()
     print("data: ", data)
+
+
+# ============================ server inference ============================
+def test_server_inference_02():
+    cpu_device_info = mslite.CPUDeviceInfo()
+    print("cpu_device_info: ", cpu_device_info)
+    context = mslite.Context(thread_num=4)
+    context.append_device_info(cpu_device_info)
+    runner_config = mslite.RunnerConfig(context, 1)
+    model_parallel_runner = mslite.ModelParallelRunner()
+    cpu_model_path = "mobilenetv2.ms"
+    cpu_in_data_path = "mobilenetv2.ms.bin"
+    model_parallel_runner.init(model_path=cpu_model_path, runner_config=runner_config)
+
+    inputs = model_parallel_runner.get_inputs()
+    in_data = np.fromfile(cpu_in_data_path, dtype=np.float32)
+    inputs[0].set_data_from_numpy(in_data)
+    outputs = []
+    model_parallel_runner.predict(inputs, outputs)
+    data = outputs[0].get_data_to_numpy()
+    print("data: ", data)