mindspore-Ecosystem
/
mindspore
21
0
Fork 27
Code Issues Pull Requests 6 Projects Releases Wiki Activity

!47265 [MS][LITE]Add High Precision Comparison For Benchmark 

Merge pull request !47265 from gongdaguo1/master_add_high_pre
This commit is contained in:
i-robot 2022-12-30 08:08:33 +00:00 committed by Gitee
parent 2b6b4e35bf 19a89b33ce
commit 85aec47d70
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
4 changed files with 19 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mindspore/lite/test/config_level0/models_server_inference.cfg
View File

@ -17,3 +17,4 @@ hiai_cv_focusShootOCRModel_02.pb;1:input_0;1,32,512,1;;parallel_predict 0.5
hiai_latin_ocr.pb;1:input_0;1,32,1024,1;;parallel_predict 0.5
deepaudio.onnx;1;5,80,80;;parallel_predict 0.5
sad_conformer4_output128lu512.pb;3;1,128512,1:1,397,128:1,199;;parallel_predict 0.5
dcn_114.pb;65;1,50:1,50:1,50:1,50:1,50:1,50:1,50:1,1:1,50:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,5:1,5:1,5:1,5:1,5:1,5:1,29:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,10:1,20:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1:1,1;;parallel_predict 0.00003

7
mindspore/lite/tools/benchmark/benchmark_base.h
View File

@ -61,8 +61,8 @@ enum MS_API AiModelDescription_Frequency {
enum MS_API DumpMode { DUMP_MODE_ALL = 0, DUMP_MODE_INPUT = 1, DUMP_MODE_OUTPUT = 2 };
constexpr float relativeTolerance = 1e-5;
constexpr float absoluteTolerance = 1e-8;
constexpr float kRelativeTolerance = 1e-5;
constexpr float kAbsoluteTolerance = 1e-8;
constexpr int CosineErrMaxVal = 2;
constexpr float kFloatMSEC = 1000.0f;
@ -264,7 +264,8 @@ class MS_API BenchmarkBase {
// tensorData need to be converter first
template <typename T, typename ST>
float CompareData(const std::string &nodeName, const std::vector<ST> &msShape, const void *tensor_data) {
float CompareData(const std::string &nodeName, const std::vector<ST> &msShape, const void *tensor_data,
float relativeTolerance = kRelativeTolerance, float absoluteTolerance = kAbsoluteTolerance) {
const T *msTensorData = static_cast<const T *>(tensor_data);
auto iter = this->benchmark_data_.find(nodeName);
if (iter != this->benchmark_data_.end()) {

20
mindspore/lite/tools/benchmark/benchmark_unified_api.cc
View File

@ -528,7 +528,10 @@ int BenchmarkUnifiedApi::CompareOutputForModelPool(std::vector<mindspore::MSTens
MS_LOG(ERROR) << "Get tensor failed, tensor name: " << tensor_name;
return RET_ERROR;
}
int ret = CompareDataGetTotalBiasAndSize(tensor_name, &tensor, &total_bias, &total_size);
constexpr float kParallelRelative = 1e-7;
constexpr float kParallelAbsolute = 1e-10;
int ret = CompareDataGetTotalBiasAndSize(tensor_name, &tensor, &total_bias, &total_size, kParallelRelative,
kParallelAbsolute);
if (ret != RET_OK) {
MS_LOG(ERROR) << "Error in CompareData";
std::cerr << "Error in CompareData" << std::endl;
@ -702,7 +705,8 @@ int BenchmarkUnifiedApi::CompareOutputByCosineDistance(float cosine_distance_thr
}
int BenchmarkUnifiedApi::CompareDataGetTotalBiasAndSize(const std::string &name, mindspore::MSTensor *tensor,
float *total_bias, int *total_size) {
float *total_bias, int *total_size, float relative_tolerance,
float absolute_tolerance) {
float bias = 0;
auto mutableData = tensor->MutableData();
if (mutableData == nullptr) {
@ -712,27 +716,27 @@ int BenchmarkUnifiedApi::CompareDataGetTotalBiasAndSize(const std::string &name,
switch (static_cast<int>(tensor->DataType())) {
case TypeId::kNumberTypeFloat:
case TypeId::kNumberTypeFloat32: {
bias = CompareData<float, int64_t>(name, tensor->Shape(), mutableData);
bias = CompareData<float, int64_t>(name, tensor->Shape(), mutableData, relative_tolerance, absolute_tolerance);
break;
}
case TypeId::kNumberTypeInt8: {
bias = CompareData<int8_t, int64_t>(name, tensor->Shape(), mutableData);
bias = CompareData<int8_t, int64_t>(name, tensor->Shape(), mutableData, relative_tolerance, absolute_tolerance);
break;
}
case TypeId::kNumberTypeUInt8: {
bias = CompareData<uint8_t, int64_t>(name, tensor->Shape(), mutableData);
bias = CompareData<uint8_t, int64_t>(name, tensor->Shape(), mutableData, relative_tolerance, absolute_tolerance);
break;
}
case TypeId::kNumberTypeInt32: {
bias = CompareData<int32_t, int64_t>(name, tensor->Shape(), mutableData);
bias = CompareData<int32_t, int64_t>(name, tensor->Shape(), mutableData, relative_tolerance, absolute_tolerance);
break;
}
case TypeId::kNumberTypeInt16: {
bias = CompareData<int16_t, int64_t>(name, tensor->Shape(), mutableData);
bias = CompareData<int16_t, int64_t>(name, tensor->Shape(), mutableData, relative_tolerance, absolute_tolerance);
break;
}
case TypeId::kNumberTypeBool: {
bias = CompareData<bool, int64_t>(name, tensor->Shape(), mutableData);
bias = CompareData<bool, int64_t>(name, tensor->Shape(), mutableData, relative_tolerance, absolute_tolerance);
break;
}
default:

3
mindspore/lite/tools/benchmark/benchmark_unified_api.h
View File

@ -55,7 +55,8 @@ class MS_API BenchmarkUnifiedApi : public BenchmarkBase {
protected:
int CompareDataGetTotalBiasAndSize(const std::string &name, mindspore::MSTensor *tensor, float *total_bias,
int *total_size);
int *total_size, float relative_tolerance = kRelativeTolerance,
float absolute_tolerance = kAbsoluteTolerance);
int CompareDataGetTotalCosineDistanceAndSize(const std::string &name, mindspore::MSTensor *tensor,
float *total_cosine_distance, int *total_size);
void InitContext(const std::shared_ptr<mindspore::Context> &context);