diff --git a/.jenkins/check/config/filter_cpplint.txt b/.jenkins/check/config/filter_cpplint.txt index 804c3e45134..ce7a8bd7e16 100644 --- a/.jenkins/check/config/filter_cpplint.txt +++ b/.jenkins/check/config/filter_cpplint.txt @@ -61,4 +61,6 @@ "mindspore/mindspore/lite/src/ops/ops_def.cc" "runtime/int" "mindspore/mindspore/lite/examples/runtime_gpu_extend/src/cl" "legal/copyright" "mindspore/mindspore/lite/examples/runtime_gpu_extend/src/cl" "readability/casting" -"mindspore/mindspore/lite/examples/runtime_gpu_extend/src/cl" "readability/fn_size" \ No newline at end of file +"mindspore/mindspore/lite/examples/runtime_gpu_extend/src/cl" "readability/fn_size" +"mindspore/mindspore/lite/examples/quick_start_c/main.c" "readability/casting" +"mindspore/mindspore/lite/examples/quick_start_c/main.c" "runtime/threadsafe_fn" \ No newline at end of file diff --git a/mindspore/lite/examples/quick_start_c/CMakeLists.txt b/mindspore/lite/examples/quick_start_c/CMakeLists.txt new file mode 100644 index 00000000000..4333ab1575f --- /dev/null +++ b/mindspore/lite/examples/quick_start_c/CMakeLists.txt @@ -0,0 +1,30 @@ +cmake_minimum_required(VERSION 3.14) +project(QuickStartC) + +if(CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 7.3.0) + message(FATAL_ERROR "GCC version ${CMAKE_C_COMPILER_VERSION} must not be less than 7.3.0") +endif() + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + +link_directories(${CMAKE_CURRENT_SOURCE_DIR}/lib) + +add_executable(mindspore_quick_start_c main.c) + +target_link_libraries( + mindspore_quick_start_c + -Wl,--whole-archive mindspore-lite -Wl,--no-whole-archive + pthread +) + +if(WIN32) + target_link_libraries( + mindspore_quick_start_c + ssp + ) +else() + target_link_libraries( + mindspore_quick_start_c + dl + ) +endif() diff --git a/mindspore/lite/examples/quick_start_c/build.sh b/mindspore/lite/examples/quick_start_c/build.sh new file mode 100644 index 00000000000..c5cd5370785 --- /dev/null +++ b/mindspore/lite/examples/quick_start_c/build.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Copyright 2021 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +set -e +BASEPATH=$(cd "$(dirname $0)"; pwd) + +MODEL_DOWNLOAD_URL="https://download.mindspore.cn/model_zoo/official/lite/quick_start/mobilenetv2.ms" + +if [ ! -e ${BASEPATH}/model/mobilenetv2.ms ]; then + mkdir -p model + wget -c -O ${BASEPATH}/model/mobilenetv2.ms --no-check-certificate ${MODEL_DOWNLOAD_URL} +fi + +mkdir -p build +cd build +cmake .. +make diff --git a/mindspore/lite/examples/quick_start_c/main.c b/mindspore/lite/examples/quick_start_c/main.c new file mode 100644 index 00000000000..5fde7b57ce8 --- /dev/null +++ b/mindspore/lite/examples/quick_start_c/main.c @@ -0,0 +1,121 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include "include/c_api/model_c.h" + +int GenerateInputDataWithRandom(MSTensorHandleArray inputs) { + for (size_t i = 0; i < inputs.handle_num; ++i) { + float *input_data = (float *)MSTensorGetMutableData(inputs.handle_list[i]); + if (input_data == NULL) { + printf("MSTensorGetMutableData failed.\n"); + return kMSStatusLiteError; + } + int64_t num = MSTensorGetElementNum(inputs.handle_list[i]); + const int divisor = 10; + for (size_t j = 0; j < num; j++) { + input_data[j] = (float)(rand() % divisor) / divisor; // 0--0.9f + } + } + return kMSStatusSuccess; +} + +int QuickStart(int argc, const char **argv) { + if (argc < 2) { + printf("Model file must be provided.\n"); + return kMSStatusLiteError; + } + + // Create and init context, add CPU device info + MSContextHandle context = MSContextCreate(); + if (context == NULL) { + printf("MSContextCreate failed.\n"); + return kMSStatusLiteError; + } + const int thread_num = 2; + MSContextSetThreadNum(context, thread_num); + MSContextSetThreadAffinityMode(context, 1); + + MSDeviceInfoHandle cpu_device_info = MSDeviceInfoCreate(kMSDeviceTypeCPU); + if (cpu_device_info == NULL) { + printf("MSDeviceInfoCreate failed.\n"); + MSContextDestroy(&context); + return kMSStatusLiteError; + } + MSDeviceInfoSetEnableFP16(cpu_device_info, false); + MSContextAddDeviceInfo(context, cpu_device_info); + + // Create model + MSModelHandle model = MSModelCreate(); + if (model == NULL) { + printf("MSModelCreate failed.\n"); + MSContextDestroy(&context); + return kMSStatusLiteError; + } + + // Build model + int ret = MSModelBuildFromFile(model, argv[1], kMSModelTypeMindIR, context); + if (ret != kMSStatusSuccess) { + printf("MSModelBuildFromFile failed, ret: %d.\n", ret); + MSModelDestroy(&model); + return ret; + } + + // Get Inputs + MSTensorHandleArray inputs = MSModelGetInputs(model); + if (inputs.handle_list == NULL) { + printf("MSModelGetInputs failed, ret: %d.\n", ret); + MSModelDestroy(&model); + return ret; + } + + // Generate random data as input data. + ret = GenerateInputDataWithRandom(inputs); + if (ret != kMSStatusSuccess) { + printf("GenerateInputDataWithRandom failed, ret: %d.\n", ret); + MSModelDestroy(&model); + return ret; + } + + // Model Predict + MSTensorHandleArray outputs; + ret = MSModelPredict(model, inputs, &outputs, NULL, NULL); + if (ret != kMSStatusSuccess) { + printf("MSModelPredict failed, ret: %d.\n", ret); + MSModelDestroy(&model); + return ret; + } + + // Print Output Tensor Data. + for (size_t i = 0; i < inputs.handle_num; ++i) { + MSTensorHandle tensor = inputs.handle_list[i]; + int64_t element_num = MSTensorGetElementNum(tensor); + printf("Tensor name: %s, elements num: %ld.\n", MSTensorGetName(tensor), element_num); + const float *data = (const float *)MSTensorGetData(tensor); + printf("output data is:\n"); + const int max_print_num = 50; + for (int j = 0; j < element_num && j <= max_print_num; ++j) { + printf("%f ", data[i]); + } + printf("\n"); + } + + // Delete model. + MSModelDestroy(&model); + return kMSStatusSuccess; +} + +int main(int argc, const char **argv) { return QuickStart(argc, argv); } diff --git a/mindspore/lite/src/lite_model.cc b/mindspore/lite/src/lite_model.cc index 01e5cd2453a..d85ac386c5e 100644 --- a/mindspore/lite/src/lite_model.cc +++ b/mindspore/lite/src/lite_model.cc @@ -106,7 +106,7 @@ int LiteModel::ConvertAttrToTensors() { void LiteModel::Free() { if (this->buf != nullptr) { - free(this->buf); + delete[](this->buf); this->buf = nullptr; } auto nodes_size = this->all_nodes_.size(); @@ -397,7 +397,7 @@ int InitModelBuffer(LiteModel *model, const char *model_buf, size_t size, bool t MS_LOG(ERROR) << "Input model buffer size invalid, require (0, 2GB]."; return RET_ERROR; } - model->buf = reinterpret_cast(malloc(size)); + model->buf = new char[size]; if (model->buf == nullptr) { MS_LOG(ERROR) << "new inner model buf fail!"; return RET_NULL_PTR; diff --git a/mindspore/lite/src/runtime/kernel/arm/string/normalize.cc b/mindspore/lite/src/runtime/kernel/arm/string/normalize.cc index 7ceee965d26..82f8e98c124 100644 --- a/mindspore/lite/src/runtime/kernel/arm/string/normalize.cc +++ b/mindspore/lite/src/runtime/kernel/arm/string/normalize.cc @@ -114,7 +114,7 @@ int NormalizeCPUKernel::Run() { auto chars = all_string_pack[i]; std::string str(chars.data, chars.len); std::string result = Normalize(str); - int str_length = result.size(); + size_t str_length = result.size(); char *normalized_str = nullptr; normalized_str = reinterpret_cast(ms_context_->allocator->Malloc(sizeof(char) * str_length)); @@ -126,7 +126,7 @@ int NormalizeCPUKernel::Run() { normalized_strs[i] = normalized_str; memcpy(normalized_str, result.data(), str_length); - out_string_pack.push_back({str_length, normalized_str}); + out_string_pack.push_back({static_cast(str_length), normalized_str}); } if (string_num == 0) { out_string_pack.push_back({1, ""}); diff --git a/mindspore/lite/src/runtime/kernel/arm/string/skip_gram.cc b/mindspore/lite/src/runtime/kernel/arm/string/skip_gram.cc index 90dccc0f1e2..d161bbbe238 100644 --- a/mindspore/lite/src/runtime/kernel/arm/string/skip_gram.cc +++ b/mindspore/lite/src/runtime/kernel/arm/string/skip_gram.cc @@ -80,7 +80,7 @@ int SkipGramCPUKernel::Run() { std::vector stack(skip_gram_parameter_->ngram_size, 0); int index = 1; - int size = words.size(); + int size = static_cast(words.size()); while (index >= 0) { if (index < skip_gram_parameter_->ngram_size && stack.at(index) + 1 < size && (index == 0 || stack.at(index) - stack.at(index - 1) <= skip_gram_parameter_->max_skip_size)) { diff --git a/mindspore/lite/test/CMakeLists.txt b/mindspore/lite/test/CMakeLists.txt index 929873f23ad..ca13657c605 100644 --- a/mindspore/lite/test/CMakeLists.txt +++ b/mindspore/lite/test/CMakeLists.txt @@ -98,6 +98,7 @@ if(MSLITE_ENABLE_CONVERTER) ${LITE_DIR}/tools/benchmark/run_benchmark.cc ${LITE_DIR}/tools/benchmark/benchmark_base.cc ${LITE_DIR}/tools/benchmark/benchmark_unified_api.cc + ${LITE_DIR}/tools/benchmark/benchmark_c_api.cc ${LITE_DIR}/tools/benchmark/benchmark.cc ) endif() diff --git a/mindspore/lite/test/config/cropped_size.cfg b/mindspore/lite/test/config/cropped_size.cfg index ef1de966661..f2d398bff0b 100644 --- a/mindspore/lite/test/config/cropped_size.cfg +++ b/mindspore/lite/test/config/cropped_size.cfg @@ -1 +1 @@ -760144 +790864 diff --git a/mindspore/lite/tools/benchmark/CMakeLists.txt b/mindspore/lite/tools/benchmark/CMakeLists.txt index 26effb47d55..87646eebf22 100644 --- a/mindspore/lite/tools/benchmark/CMakeLists.txt +++ b/mindspore/lite/tools/benchmark/CMakeLists.txt @@ -15,6 +15,7 @@ add_executable(benchmark ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_base.cc ${CMAKE_CURRENT_SOURCE_DIR}/benchmark.cc ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_unified_api.cc + ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_c_api.cc ${COMMON_SRC}) add_dependencies(benchmark fbs_src) diff --git a/mindspore/lite/tools/benchmark/benchmark.cc b/mindspore/lite/tools/benchmark/benchmark.cc index b0c80a5cb6b..6e3214a5b05 100644 --- a/mindspore/lite/tools/benchmark/benchmark.cc +++ b/mindspore/lite/tools/benchmark/benchmark.cc @@ -114,43 +114,13 @@ int Benchmark::ReadInputFile() { return RET_OK; } -int Benchmark::ReadTensorData(std::ifstream &in_file_stream, const std::string &tensor_name, - const std::vector &dims) { - std::string line; - getline(in_file_stream, line); - std::stringstream line_stream(line); - if (this->benchmark_data_.find(tensor_name) != this->benchmark_data_.end()) { - return RET_OK; - } - tensor::MSTensor *tensor = session_->GetOutputByTensorName(tensor_name); - if (tensor == nullptr) { - MS_LOG(ERROR) << "Get tensor failed, tensor name: " << tensor_name; - return RET_ERROR; - } - std::vector data; - std::vector strings_data; - size_t shape_size = std::accumulate(dims.begin(), dims.end(), 1, std::multiplies()); - if (tensor->data_type() == kObjectTypeString) { - strings_data.push_back(line); - for (size_t i = 1; i < shape_size; i++) { - getline(in_file_stream, line); - strings_data.push_back(line); - } +int Benchmark::GetDataTypeByTensorName(const std::string &tensor_name) { + auto tensor = session_->GetOutputByTensorName(tensor_name); + if (tensor != nullptr) { + return tensor->data_type(); } else { - for (size_t i = 0; i < shape_size; i++) { - float tmp_data; - line_stream >> tmp_data; - data.push_back(tmp_data); - } + return kTypeUnknown; } - auto *check_tensor = new (std::nothrow) CheckTensor(dims, data, strings_data); - if (check_tensor == nullptr) { - MS_LOG(ERROR) << "New CheckTensor failed, tensor name: " << tensor_name; - return RET_ERROR; - } - this->benchmark_tensor_names_.push_back(tensor_name); - this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor)); - return RET_OK; } void Benchmark::InitContext(const std::shared_ptr &context) { @@ -521,13 +491,6 @@ int Benchmark::RunBenchmark() { } if (!flags_->benchmark_data_file_.empty()) { status = MarkAccuracy(); - for (auto &data : benchmark_data_) { - data.second->shape.clear(); - data.second->data.clear(); - delete data.second; - data.second = nullptr; - } - benchmark_data_.clear(); if (status != 0) { MS_LOG(ERROR) << "Run MarkAccuracy error: " << status; std::cout << "Run MarkAccuracy error: " << status << std::endl; diff --git a/mindspore/lite/tools/benchmark/benchmark.h b/mindspore/lite/tools/benchmark/benchmark.h index 310d43bb135..45fa4d813d7 100644 --- a/mindspore/lite/tools/benchmark/benchmark.h +++ b/mindspore/lite/tools/benchmark/benchmark.h @@ -53,8 +53,7 @@ class MS_API Benchmark : public BenchmarkBase { int ReadInputFile() override; - int ReadTensorData(std::ifstream &in_file_stream, const std::string &tensor_name, - const std::vector &dims) override; + int GetDataTypeByTensorName(const std::string &tensor_name) override; void InitContext(const std::shared_ptr &context); diff --git a/mindspore/lite/tools/benchmark/benchmark_base.cc b/mindspore/lite/tools/benchmark/benchmark_base.cc index f4578446c83..91aea2ced19 100644 --- a/mindspore/lite/tools/benchmark/benchmark_base.cc +++ b/mindspore/lite/tools/benchmark/benchmark_base.cc @@ -173,6 +173,40 @@ int BenchmarkBase::ReadCalibData() { return RET_OK; } +int BenchmarkBase::ReadTensorData(std::ifstream &in_file_stream, const std::string &tensor_name, + const std::vector &dims) { + std::string line; + getline(in_file_stream, line); + std::stringstream line_stream(line); + if (this->benchmark_data_.find(tensor_name) != this->benchmark_data_.end()) { + return RET_OK; + } + std::vector data; + std::vector strings_data; + size_t shape_size = std::accumulate(dims.begin(), dims.end(), 1, std::multiplies()); + if (GetDataTypeByTensorName(tensor_name) == static_cast(kObjectTypeString)) { + strings_data.push_back(line); + for (size_t i = 1; i < shape_size; i++) { + getline(in_file_stream, line); + strings_data.push_back(line); + } + } else { + for (size_t i = 0; i < shape_size; i++) { + float tmp_data; + line_stream >> tmp_data; + data.push_back(tmp_data); + } + } + auto *check_tensor = new (std::nothrow) CheckTensor(dims, data, strings_data); + if (check_tensor == nullptr) { + MS_LOG(ERROR) << "New CheckTensor failed, tensor name: " << tensor_name; + return RET_ERROR; + } + this->benchmark_tensor_names_.push_back(tensor_name); + this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor)); + return RET_OK; +} + int BenchmarkBase::CompareStringData(const std::string &name, const std::vector &calib_strings, const std::vector &output_strings) { size_t compare_num = std::min(calib_strings.size(), output_strings.size()); @@ -594,8 +628,11 @@ int BenchmarkBase::PrintPerfResult(const std::vector &title, #endif BenchmarkBase::~BenchmarkBase() { - for (const auto &iter : this->benchmark_data_) { - delete (iter.second); + for (auto &iter : this->benchmark_data_) { + iter.second->shape.clear(); + iter.second->data.clear(); + delete iter.second; + iter.second = nullptr; } this->benchmark_data_.clear(); } diff --git a/mindspore/lite/tools/benchmark/benchmark_base.h b/mindspore/lite/tools/benchmark/benchmark_base.h index 6da300b6515..f94189385e1 100644 --- a/mindspore/lite/tools/benchmark/benchmark_base.h +++ b/mindspore/lite/tools/benchmark/benchmark_base.h @@ -39,6 +39,12 @@ #include "schema/model_generated.h" namespace mindspore::lite { +#define BENCHMARK_LOG_ERROR(str) \ + do { \ + MS_LOG(ERROR) << str; \ + std::cerr << str << std::endl; \ + } while (0); + enum MS_API InDataType { kImage = 0, kBinary = 1 }; enum MS_API AiModelDescription_Frequency { @@ -188,8 +194,9 @@ class MS_API BenchmarkBase { int ReadCalibData(); - virtual int ReadTensorData(std::ifstream &in_file_stream, const std::string &tensor_name, - const std::vector &dims) = 0; + int ReadTensorData(std::ifstream &in_file_stream, const std::string &tensor_name, const std::vector &dims); + + virtual int GetDataTypeByTensorName(const std::string &tensor_name) = 0; virtual int CompareOutput() = 0; diff --git a/mindspore/lite/tools/benchmark/benchmark_c_api.cc b/mindspore/lite/tools/benchmark/benchmark_c_api.cc new file mode 100644 index 00000000000..317525fce8a --- /dev/null +++ b/mindspore/lite/tools/benchmark/benchmark_c_api.cc @@ -0,0 +1,430 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "tools/benchmark/benchmark_c_api.h" +#include +#include +#include +#include + +using mindspore::lite::GetTimeUs; +using mindspore::lite::kFloatMSEC; +using mindspore::lite::RET_ERROR; +using mindspore::lite::RET_OK; + +namespace mindspore { +namespace tools { +int BenchmarkCApi::RunBenchmark() { + auto start_prepare_time = GetTimeUs(); + int ret = InitContext(); + if (ret != RET_OK) { + BENCHMARK_LOG_ERROR("InitContext failed, ret: " << ret); + return ret; + } + model_ = MSModelCreate(); + ret = MSModelBuildFromFile(model_, flags_->model_file_.c_str(), kMSModelTypeMindIR, context_); + if (ret != kMSStatusSuccess) { + BENCHMARK_LOG_ERROR("MSModelBuildFromFile failed, ret: " << ret); + return ret; + } + inputs_ = MSModelGetInputs(model_); + if (inputs_.handle_list == nullptr) { + BENCHMARK_LOG_ERROR("MSModelGetInputs failed, ret: " << ret); + return ret; + } + if (!flags_->resize_dims_.empty()) { + std::vector shape_infos; + std::transform(flags_->resize_dims_.begin(), flags_->resize_dims_.end(), std::back_inserter(shape_infos), + [&](auto &shapes) { + MSShapeInfo shape_info; + shape_info.shape_num = shapes.size(); + for (size_t i = 0; i < shape_info.shape_num; i++) { + shape_info.shape[i] = shapes[i]; + } + return shape_info; + }); + ret = MSModelResize(model_, inputs_, shape_infos.data(), inputs_.handle_num); + if (ret != kMSStatusSuccess) { + BENCHMARK_LOG_ERROR("MSModelResize failed, ret: " << ret); + return ret; + } + } + auto end_prepare_time = GetTimeUs(); + MS_LOG(INFO) << "PrepareTime = " << ((end_prepare_time - start_prepare_time) / kFloatMSEC) << " ms"; + std::cout << "PrepareTime = " << ((end_prepare_time - start_prepare_time) / kFloatMSEC) << " ms" << std::endl; + + ret = LoadInput(); + if (ret != kMSStatusSuccess) { + BENCHMARK_LOG_ERROR("LoadInput failed, ret: " << ret) + return ret; + } + if (!flags_->benchmark_data_file_.empty()) { + ret = MarkAccuracy(); + } else { + ret = MarkPerformance(); + } + if (ret != kMSStatusSuccess) { + BENCHMARK_LOG_ERROR("Run failed, ret: " << ret); + return ret; + } + if (flags_->dump_tensor_data_) { + BENCHMARK_LOG_ERROR("Dumped file is saved to : " + dump_file_output_dir_) + } + return RET_OK; +} + +int BenchmarkCApi::InitContext() { + constexpr int kFrequencyDefault = 3; + context_ = MSContextCreate(); + if (context_ == nullptr) { + BENCHMARK_LOG_ERROR("MSContextCreate failed"); + return RET_ERROR; + } + MSContextSetThreadNum(context_, flags_->num_threads_); + MSContextSetEnableParallel(context_, flags_->enable_parallel_); + MSContextSetThreadAffinityMode(context_, flags_->cpu_bind_mode_); + + MSDeviceInfoHandle cpu_device_info = MSDeviceInfoCreate(kMSDeviceTypeCPU); + MSDeviceInfoSetEnableFP16(cpu_device_info, flags_->enable_fp16_); + MSContextAddDeviceInfo(context_, cpu_device_info); + + if (flags_->device_ == "GPU") { + MSDeviceInfoHandle gpu_device_info = MSDeviceInfoCreate(kMSDeviceTypeGPU); + MSDeviceInfoSetEnableFP16(gpu_device_info, flags_->enable_fp16_); + MSContextAddDeviceInfo(context_, gpu_device_info); + } + + if (flags_->device_ == "NPU") { + MSDeviceInfoHandle npu_device_info = MSDeviceInfoCreate(kMSDeviceTypeKirinNPU); + MSDeviceInfoSetFrequency(npu_device_info, kFrequencyDefault); + MSContextAddDeviceInfo(context_, npu_device_info); + } + return RET_OK; +} + +int BenchmarkCApi::GenerateInputData() { + for (size_t i = 0; i < inputs_.handle_num; i++) { + MSTensorHandle tensor = inputs_.handle_list[i]; + auto data_type = MSTensorGetDataType(tensor); + if (data_type == kMSDataTypeObjectTypeString) { + BENCHMARK_LOG_ERROR("Unsupported kMSDataTypeObjectTypeString"); + return RET_ERROR; + } else { + auto data_ptr = MSTensorGetMutableData(tensor); + auto data_size = MSTensorGetDataSize(tensor); + (void)GenerateRandomData(data_size, data_ptr, static_cast(data_type)); + } + } + return RET_OK; +} + +int BenchmarkCApi::ReadInputFile() { + if (this->flags_->in_data_type_ == lite::kImage) { + BENCHMARK_LOG_ERROR("Unsupported image input"); + return RET_ERROR; + } else { + for (size_t i = 0; i < flags_->input_data_list_.size(); i++) { + MSTensorHandle tensor = inputs_.handle_list[i]; + size_t size; + auto bin_buf = lite::ReadFile(flags_->input_data_list_[i].c_str(), &size); + if (bin_buf == nullptr) { + BENCHMARK_LOG_ERROR("ReadFile failed"); + return RET_ERROR; + } + if (MSTensorGetDataType(tensor) == kMSDataTypeObjectTypeString) { + BENCHMARK_LOG_ERROR("Unsupported kMSDataTypeObjectTypeString"); + return RET_ERROR; + } else { + auto tensor_data_size = MSTensorGetDataSize(tensor); + if (tensor_data_size != size) { + BENCHMARK_LOG_ERROR("Input file size error, required: " << tensor_data_size << ", in fact: " << size); + delete[] bin_buf; + return RET_ERROR; + } + auto input_data = MSTensorGetMutableData(tensor); + if (input_data == nullptr) { + BENCHMARK_LOG_ERROR("MSTensorGetMutableData failed"); + return RET_ERROR; + } + memcpy(input_data, bin_buf, size); + } + delete[] bin_buf; + } + } + return RET_OK; +} + +int BenchmarkCApi::MarkAccuracy() { + MS_LOG(INFO) << "MarkAccuracy"; + std::cout << "MarkAccuracy" << std::endl; + auto status = PrintInputData(); + if (status != RET_OK) { + BENCHMARK_LOG_ERROR("PrintInputData failed, ret: " << status); + return status; + } + status = MSModelPredict(model_, inputs_, &outputs_, before_call_back_, after_call_back_); + if (status != kMSStatusSuccess) { + BENCHMARK_LOG_ERROR("MSModelPredict failed, ret: " << status); + return RET_ERROR; + } + status = ReadCalibData(); + if (status != RET_OK) { + BENCHMARK_LOG_ERROR("ReadCalibData failed, ret: " << status); + return status; + } + status = CompareOutput(); + if (status != RET_OK) { + BENCHMARK_LOG_ERROR("CompareOutput failed, ret: " << status); + return status; + } + return RET_OK; +} + +int BenchmarkCApi::MarkPerformance() { + MS_LOG(INFO) << "Running warm up loops..."; + std::cout << "Running warm up loops..." << std::endl; + for (int i = 0; i < flags_->warm_up_loop_count_; i++) { + auto ret = MSModelPredict(model_, inputs_, &outputs_, before_call_back_, after_call_back_); + if (ret != kMSStatusSuccess) { + BENCHMARK_LOG_ERROR("MSModelPredict failed, ret: " << kMSStatusSuccess); + return RET_ERROR; + } + } + + MS_LOG(INFO) << "Running benchmark loops..."; + std::cout << "Running benchmark loops..." << std::endl; + uint64_t time_min = 1000000; + uint64_t time_max = 0; + uint64_t time_avg = 0; + + for (int i = 0; i < flags_->loop_count_; i++) { + auto start = GetTimeUs(); + auto ret = MSModelPredict(model_, inputs_, &outputs_, before_call_back_, after_call_back_); + if (ret != kMSStatusSuccess) { + BENCHMARK_LOG_ERROR("MSModelPredict failed, ret: " << kMSStatusSuccess); + return RET_ERROR; + } + auto end = GetTimeUs(); + auto time = end - start; + time_min = std::min(time_min, time); + time_max = std::max(time_max, time); + time_avg += time; + } + + if (flags_->time_profiling_) { + const std::vector per_op_name = {"opName", "avg(ms)", "percent", "calledTimes", "opTotalTime"}; + const std::vector per_op_type = {"opType", "avg(ms)", "percent", "calledTimes", "opTotalTime"}; + PrintResult(per_op_name, op_times_by_name_); + PrintResult(per_op_type, op_times_by_type_); + } + + if (flags_->loop_count_ > 0) { + time_avg /= flags_->loop_count_; + MS_LOG(INFO) << "Model = " + << flags_->model_file_.substr(flags_->model_file_.find_last_of(lite::DELIM_SLASH) + 1).c_str() + << ", NumThreads = " << flags_->num_threads_ << ", MinRunTime = " << time_min / lite::kFloatMSEC + << ", MaxRuntime = " << time_max / lite::kFloatMSEC + << ", AvgRunTime = " << time_avg / lite::kFloatMSEC; + printf("Model = %s, NumThreads = %d, MinRunTime = %f ms, MaxRuntime = %f ms, AvgRunTime = %f ms\n", + flags_->model_file_.substr(flags_->model_file_.find_last_of(lite::DELIM_SLASH) + 1).c_str(), + flags_->num_threads_, time_min / lite::kFloatMSEC, time_max / lite::kFloatMSEC, time_avg / lite::kFloatMSEC); + } + return RET_OK; +} + +int BenchmarkCApi::GetDataTypeByTensorName(const std::string &tensor_name) { + return MSTensorGetDataType(MSModelGetOutputByTensorName(model_, tensor_name.c_str())); +} + +int BenchmarkCApi::CompareOutput() { + constexpr int kPercentageDivisor = 100; + std::cout << "================ Comparing Output data ================" << std::endl; + float total_bias = 0; + int total_size = 0; + for (const auto &calib_tensor : benchmark_data_) { + std::string tensor_name = calib_tensor.first; + MSTensorHandle tensor = MSModelGetOutputByTensorName(model_, tensor_name.c_str()); + if (tensor == nullptr) { + BENCHMARK_LOG_ERROR("Get tensor failed, tensor name: " << tensor_name); + return RET_ERROR; + } + int ret; + if (static_cast(MSTensorGetDataType(tensor)) == kObjectTypeString) { + BENCHMARK_LOG_ERROR("Unsupported kMSDataTypeObjectTypeString"); + return RET_ERROR; + } else { + ret = CompareDataGetTotalBiasAndSize(tensor_name, tensor, &total_bias, &total_size); + } + if (ret != RET_OK) { + BENCHMARK_LOG_ERROR("Error in CompareData"); + BENCHMARK_LOG_ERROR("======================================================="); + return ret; + } + } + float mean_bias; + if (total_size != 0) { + mean_bias = ((total_bias / float_t(total_size)) * kPercentageDivisor); + } else { + mean_bias = 0; + } + + std::cout << "Mean bias of all nodes/tensors: " << mean_bias << "%" << std::endl; + std::cout << "=======================================================" << std::endl << std::endl; + + if (mean_bias > this->flags_->accuracy_threshold_) { + BENCHMARK_LOG_ERROR("Mean bias of all nodes/tensors is too big: " << mean_bias << "%"); + return RET_ERROR; + } + return RET_OK; +} + +int BenchmarkCApi::CompareDataGetTotalBiasAndSize(const std::string &name, MSTensorHandle tensor, float *total_bias, + int *total_size) { + auto tensor_data = MSTensorGetData(tensor); + if (tensor_data == nullptr) { + BENCHMARK_LOG_ERROR("MSTensorGetData failed."); + return RET_ERROR; + } + size_t shape_num; + const int64_t *shape = MSTensorGetShape(tensor, &shape_num); + std::vector vec_shape(shape, shape + shape_num); + float bias = 0; + switch (static_cast(MSTensorGetDataType(tensor))) { + case TypeId::kNumberTypeFloat: + case TypeId::kNumberTypeFloat32: { + bias = CompareData(name, vec_shape, tensor_data); + break; + } + case TypeId::kNumberTypeInt8: { + bias = CompareData(name, vec_shape, tensor_data); + break; + } + case TypeId::kNumberTypeUInt8: { + bias = CompareData(name, vec_shape, tensor_data); + break; + } + case TypeId::kNumberTypeInt32: { + bias = CompareData(name, vec_shape, tensor_data); + break; + } + case TypeId::kNumberTypeInt16: { + bias = CompareData(name, vec_shape, tensor_data); + break; + } + case TypeId::kNumberTypeBool: { + bias = CompareData(name, vec_shape, tensor_data); + break; + } + default: + BENCHMARK_LOG_ERROR("Unsupported data type" << static_cast(MSTensorGetDataType(tensor))); + return RET_ERROR; + } + if (bias < 0) { + BENCHMARK_LOG_ERROR("CompareData failed, name: " << name); + return RET_ERROR; + } + *total_bias += bias; + *total_size += 1; + return RET_OK; +} + +int BenchmarkCApi::PrintInputData() { + constexpr int64_t kPrintDataNum = 20; + for (size_t i = 0; i < inputs_.handle_num; i++) { + auto input = inputs_.handle_list[i]; + std::cout << "InData" << i << ": "; + auto data_type = static_cast(MSTensorGetDataType(input)); + if (data_type == TypeId::kObjectTypeString) { + BENCHMARK_LOG_ERROR("Unsupported kMSDataTypeObjectTypeString."); + return RET_ERROR; + } + auto tensor_data = MSTensorGetData(input); + size_t print_num = std::min(MSTensorGetElementNum(input), kPrintDataNum); + for (size_t j = 0; j < print_num; j++) { + if (data_type == TypeId::kNumberTypeFloat32 || data_type == TypeId::kNumberTypeFloat) { + std::cout << static_cast(tensor_data)[j] << " "; + } else if (data_type == TypeId::kNumberTypeInt8) { + std::cout << static_cast(tensor_data)[j] << " "; + } else if (data_type == TypeId::kNumberTypeUInt8) { + std::cout << static_cast(tensor_data)[j] << " "; + } else if (data_type == TypeId::kNumberTypeInt32) { + std::cout << static_cast(tensor_data)[j] << " "; + } else if (data_type == TypeId::kNumberTypeInt64) { + std::cout << static_cast(tensor_data)[j] << " "; + } else if (data_type == TypeId::kNumberTypeBool) { + std::cout << static_cast(tensor_data)[j] << " "; + } else { + BENCHMARK_LOG_ERROR("Datatype: " << data_type << " is not supported."); + return RET_ERROR; + } + } + std::cout << std::endl; + } + return RET_OK; +} + +int BenchmarkCApi::InitTimeProfilingCallbackParameter() { + before_call_back_ = TimeBeforeCallback; + after_call_back_ = TimeAfterCallback; + return RET_OK; +} + +int BenchmarkCApi::InitPerfProfilingCallbackParameter() { + BENCHMARK_LOG_ERROR("Unsupported feature."); + return RET_ERROR; +} + +int BenchmarkCApi::InitPrintTensorDataCallbackParameter() { + BENCHMARK_LOG_ERROR("Unsupported feature."); + return RET_ERROR; +} +int BenchmarkCApi::InitDumpTensorDataCallbackParameter() { + BENCHMARK_LOG_ERROR("Unsupported feature."); + return RET_ERROR; +} +} // namespace tools +} // namespace mindspore + +uint64_t g_op_begin_ = 0; +int g_op_call_times_total_ = 0; +float g_op_cost_total_ = 0.0f; +std::map> g_op_times_by_type_; +std::map> g_op_times_by_name_; + +bool TimeBeforeCallback(const MSTensorHandleArray inputs, const MSTensorHandleArray outputs, + const MSCallBackParamC kernel_Info) { + if (g_op_times_by_type_.find(kernel_Info.node_type) == g_op_times_by_type_.end()) { + g_op_times_by_type_.insert(std::make_pair(kernel_Info.node_type, std::make_pair(0, 0.0f))); + } + if (g_op_times_by_name_.find(kernel_Info.node_name) == g_op_times_by_name_.end()) { + g_op_times_by_name_.insert(std::make_pair(kernel_Info.node_name, std::make_pair(0, 0.0f))); + } + + g_op_call_times_total_++; + g_op_begin_ = mindspore::lite::GetTimeUs(); + return true; +} + +bool TimeAfterCallback(const MSTensorHandleArray inputs, const MSTensorHandleArray outputs, + const MSCallBackParamC kernel_Info) { + uint64_t opEnd = mindspore::lite::GetTimeUs(); + float cost = static_cast(opEnd - g_op_begin_) / mindspore::lite::kFloatMSEC; + g_op_cost_total_ += cost; + g_op_times_by_type_[kernel_Info.node_type].first++; + g_op_times_by_type_[kernel_Info.node_type].second += cost; + g_op_times_by_name_[kernel_Info.node_name].first++; + g_op_times_by_name_[kernel_Info.node_name].second += cost; + return true; +} diff --git a/mindspore/lite/tools/benchmark/benchmark_c_api.h b/mindspore/lite/tools/benchmark/benchmark_c_api.h new file mode 100644 index 00000000000..46c26a895c4 --- /dev/null +++ b/mindspore/lite/tools/benchmark/benchmark_c_api.h @@ -0,0 +1,76 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef MINDSPORE_LITE_TOOLS_BENCHMARK_BENCHMARK_C_API_H_ +#define MINDSPORE_LITE_TOOLS_BENCHMARK_BENCHMARK_C_API_H_ + +#include +#include +#include "tools/benchmark/benchmark_base.h" +#include "include/c_api/model_c.h" +#include "include/c_api/context_c.h" + +#ifdef __cplusplus +extern "C" { +#endif +bool TimeBeforeCallback(const MSTensorHandleArray inputs, const MSTensorHandleArray outputs, + const MSCallBackParamC kernel_Info); +bool TimeAfterCallback(const MSTensorHandleArray inputs, const MSTensorHandleArray outputs, + const MSCallBackParamC kernel_Info); +#ifdef __cplusplus +} +#endif + +using mindspore::lite::BenchmarkBase; +using mindspore::lite::BenchmarkFlags; + +namespace mindspore::tools { +class MS_API BenchmarkCApi : public BenchmarkBase { + public: + explicit BenchmarkCApi(BenchmarkFlags *flags) : BenchmarkBase(flags) {} + + virtual ~BenchmarkCApi() { MSModelDestroy(&model_); } + + int RunBenchmark() override; + + protected: + int CompareDataGetTotalBiasAndSize(const std::string &name, MSTensorHandle tensor, float *total_bias, + int *total_size); + int InitContext(); + int GenerateInputData() override; + int ReadInputFile() override; + int GetDataTypeByTensorName(const std::string &tensor_name) override; + int CompareOutput() override; + + int InitTimeProfilingCallbackParameter() override; + int InitPerfProfilingCallbackParameter() override; + int InitDumpTensorDataCallbackParameter() override; + int InitPrintTensorDataCallbackParameter() override; + + int PrintInputData(); + int MarkPerformance(); + int MarkAccuracy(); + + private: + MSModelHandle model_ = nullptr; + MSContextHandle context_ = nullptr; + MSTensorHandleArray inputs_; + MSTensorHandleArray outputs_; + + MSKernelCallBackC before_call_back_ = nullptr; + MSKernelCallBackC after_call_back_ = nullptr; +}; +} // namespace mindspore::tools +#endif // MINDSPORE_LITE_TOOLS_BENCHMARK_BENCHMARK_C_API_H_ diff --git a/mindspore/lite/tools/benchmark/benchmark_unified_api.cc b/mindspore/lite/tools/benchmark/benchmark_unified_api.cc index 7fee10500f5..38e98e2c3a4 100644 --- a/mindspore/lite/tools/benchmark/benchmark_unified_api.cc +++ b/mindspore/lite/tools/benchmark/benchmark_unified_api.cc @@ -121,43 +121,8 @@ int BenchmarkUnifiedApi::ReadInputFile() { return RET_OK; } -int BenchmarkUnifiedApi::ReadTensorData(std::ifstream &in_file_stream, const std::string &tensor_name, - const std::vector &dims) { - std::string line; - getline(in_file_stream, line); - std::stringstream line_stream(line); - if (this->benchmark_data_.find(tensor_name) != this->benchmark_data_.end()) { - return RET_OK; - } - mindspore::MSTensor tensor = ms_model_.GetOutputByTensorName(tensor_name); - if (tensor == nullptr) { - MS_LOG(ERROR) << "Get tensor failed, tensor name: " << tensor_name; - return RET_ERROR; - } - std::vector data; - std::vector strings_data; - size_t shape_size = std::accumulate(dims.begin(), dims.end(), 1, std::multiplies()); - if (static_cast(tensor.DataType()) == kObjectTypeString) { - strings_data.push_back(line); - for (size_t i = 1; i < shape_size; i++) { - getline(in_file_stream, line); - strings_data.push_back(line); - } - } else { - for (size_t i = 0; i < shape_size; i++) { - float tmp_data; - line_stream >> tmp_data; - data.push_back(tmp_data); - } - } - auto *check_tensor = new (std::nothrow) CheckTensor(dims, data, strings_data); - if (check_tensor == nullptr) { - MS_LOG(ERROR) << "New CheckTensor failed, tensor name: " << tensor_name; - return RET_ERROR; - } - this->benchmark_tensor_names_.push_back(tensor_name); - this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor)); - return RET_OK; +int BenchmarkUnifiedApi::GetDataTypeByTensorName(const std::string &tensor_name) { + return static_cast(ms_model_.GetOutputByTensorName(tensor_name).DataType()); } void BenchmarkUnifiedApi::UpdateDistributionModelName(const std::shared_ptr &context, @@ -537,13 +502,6 @@ int BenchmarkUnifiedApi::RunBenchmark() { } if (!flags_->benchmark_data_file_.empty()) { status = MarkAccuracy(); - for (auto &data : benchmark_data_) { - data.second->shape.clear(); - data.second->data.clear(); - delete data.second; - data.second = nullptr; - } - benchmark_data_.clear(); if (status != 0) { MS_LOG(ERROR) << "Run MarkAccuracy error: " << status; std::cout << "Run MarkAccuracy error: " << status << std::endl; diff --git a/mindspore/lite/tools/benchmark/benchmark_unified_api.h b/mindspore/lite/tools/benchmark/benchmark_unified_api.h index 7c42bea8d07..61ce335daa7 100644 --- a/mindspore/lite/tools/benchmark/benchmark_unified_api.h +++ b/mindspore/lite/tools/benchmark/benchmark_unified_api.h @@ -57,11 +57,10 @@ class MS_API BenchmarkUnifiedApi : public BenchmarkBase { int ReadInputFile() override; - int ReadTensorData(std::ifstream &in_file_stream, const std::string &tensor_name, - const std::vector &dims) override; - void InitMSContext(const std::shared_ptr &context); + int GetDataTypeByTensorName(const std::string &tensor_name) override; + int CompareOutput() override; int InitTimeProfilingCallbackParameter() override; diff --git a/mindspore/lite/tools/benchmark/run_benchmark.cc b/mindspore/lite/tools/benchmark/run_benchmark.cc index 1fdc6ab0e38..ed6e96c7f1d 100644 --- a/mindspore/lite/tools/benchmark/run_benchmark.cc +++ b/mindspore/lite/tools/benchmark/run_benchmark.cc @@ -17,6 +17,7 @@ #include "tools/benchmark/run_benchmark.h" #include #include +#include "tools/benchmark/benchmark_c_api.h" namespace mindspore { namespace lite { @@ -34,20 +35,23 @@ int RunBenchmark(int argc, const char **argv) { return RET_OK; } - // get dump data output path - auto new_api = std::getenv("ENABLE_NEW_API"); - bool run_old_api = (new_api == nullptr || std::string(new_api) != "true"); - if (flags.config_file_ != "") { - run_old_api = false; + auto api_type = std::getenv("MSLITE_API_TYPE"); + if (api_type != nullptr) { + MS_LOG(INFO) << "MSLITE_API_TYPE = " << api_type; + std::cout << "MSLITE_API_TYPE = " << api_type << std::endl; } - if (IsCharEndWith(flags.model_file_.c_str(), MINDIR_POSTFIX)) { - run_old_api = false; - } - std::unique_ptr benchmark; - if (run_old_api) { - benchmark = std::make_unique(&flags); + BenchmarkBase *benchmark = nullptr; + if (flags.config_file_ != "" || IsCharEndWith(flags.model_file_.c_str(), MINDIR_POSTFIX) || + (api_type != nullptr && std::string(api_type) == "NEW")) { + benchmark = new (std::nothrow) BenchmarkUnifiedApi(&flags); + } else if (api_type == nullptr || std::string(api_type) == "OLD") { + benchmark = new (std::nothrow) Benchmark(&flags); + } else if (std::string(api_type) == "C") { + benchmark = new (std::nothrow) tools::BenchmarkCApi(&flags); } else { - benchmark = std::make_unique(&flags); + MS_LOG(ERROR) << "Invalid MSLITE_API_TYPE, (OLD/NEW/C, default:OLD)"; + std::cerr << "Invalid MSLITE_API_TYPE, (OLD/NEW/C, default:OLD)" << std::endl; + return RET_ERROR; } if (benchmark == nullptr) { MS_LOG(ERROR) << "new benchmark failed "; @@ -75,6 +79,7 @@ int RunBenchmark(int argc, const char **argv) { << " Success."; std::cout << "Run Benchmark " << flags.model_file_.substr(flags.model_file_.find_last_of(DELIM_SLASH) + 1).c_str() << " Success." << std::endl; + delete benchmark; return RET_OK; } } // namespace lite diff --git a/mindspore/lite/tools/cropper/build_cropper_config.sh b/mindspore/lite/tools/cropper/build_cropper_config.sh index 123b3433445..36c1513c352 100644 --- a/mindspore/lite/tools/cropper/build_cropper_config.sh +++ b/mindspore/lite/tools/cropper/build_cropper_config.sh @@ -144,6 +144,7 @@ getCommonFile() { while IFS='' read -r line; do cxx_api_files+=("$line"); done < <(ls mindspore/lite/src/cxx_api/model/*.cc) while IFS='' read -r line; do cxx_api_files+=("$line"); done < <(ls mindspore/lite/src/cxx_api/tensor/*.cc) while IFS='' read -r line; do cxx_api_files+=("$line"); done < <(ls mindspore/lite/src/cxx_api/*.cc) + while IFS='' read -r line; do cxx_api_files+=("$line"); done < <(ls mindspore/lite/src/c_api/*.cc) mindrt_files=() while IFS='' read -r line; do mindrt_files+=("$line"); done < <(ls mindspore/core/mindrt/src/*.cc) while IFS='' read -r line; do mindrt_files+=("$line"); done < <(ls mindspore/core/mindrt/src/async/*.cc)