From f1c1ec721ee79a1db9d128f3798ee1047db9f914 Mon Sep 17 00:00:00 2001 From: cjh9368 Date: Mon, 3 Aug 2020 16:40:10 +0800 Subject: [PATCH] benchmark print model name, calculate benchmark accuracy formula --- mindspore/lite/tools/benchmark/benchmark.cc | 53 +++++++++++++-------- mindspore/lite/tools/benchmark/benchmark.h | 5 +- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/mindspore/lite/tools/benchmark/benchmark.cc b/mindspore/lite/tools/benchmark/benchmark.cc index b04d38ef9d3..3ca5ef590b6 100644 --- a/mindspore/lite/tools/benchmark/benchmark.cc +++ b/mindspore/lite/tools/benchmark/benchmark.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include "src/common/common.h" #include "include/ms_tensor.h" #include "include/context.h" @@ -191,11 +192,16 @@ float Benchmark::CompareData(const std::string &nodeName, std::vector msSha std::cout << msTensorData[j] << " "; } + if (std::isnan(msTensorData[j]) || std::isinf(msTensorData[j])) { + MS_LOG(ERROR) << "Output tensor has nan or inf data, compare fail"; + return RET_ERROR; + } + auto tolerance = absoluteTolerance + relativeTolerance * fabs(calibTensor->data.at(j)); auto absoluteError = std::fabs(msTensorData[j] - calibTensor->data.at(j)); if (absoluteError > tolerance) { // just assume that atol = rtol - meanError += absoluteError / (fabs(calibTensor->data.at(j)) + 1); + meanError += absoluteError / (fabs(calibTensor->data.at(j)) + FLT_MIN); errorCount++; } } @@ -296,16 +302,10 @@ int Benchmark::MarkPerformance() { } if (_flags->loopCount > 0) { timeAvg /= _flags->loopCount; - // MS_LOG(INFO) << "CSV:%s:%d:%f:%f:%f\n", _flags->modelPath.substr(_flags->modelPath.find_last_of(DELIM_SLASH) + - // 1).c_str(), - // _flags->numThreads, timeMin / 1000.0f, timeMax / 1000.0f, timeAvg / 1000.0f); - // MS_LOG(INFO) <<"Modle = %s, numThreads = %d, MinRunTime = %f ms, MaxRuntime = %f ms, AvgRunTime = %f ms", - // _flags->modelPath.substr(_flags->modelPath.find_last_of(DELIM_SLASH) + 1).c_str(), _flags->numThreads, - // timeMin / 1000.0f, timeMax / 1000.0f, timeAvg / 1000.0f); - - printf("CSV:%s:%d:%f:%f:%f\n", _flags->modelPath.substr(_flags->modelPath.find_last_of(DELIM_SLASH) + 1).c_str(), - _flags->numThreads, timeMin / 1000.0f, timeMax / 1000.0f, timeAvg / 1000.0f); - printf("Modle = %s, numThreads = %d, MinRunTime = %f ms, MaxRuntime = %f ms, AvgRunTime = %f ms\n", + MS_LOG(INFO) << "Model = " << _flags->modelPath.substr(_flags->modelPath.find_last_of(DELIM_SLASH) + 1).c_str() + << ", NumThreads = " << _flags->numThreads << ", MinRunTime = " << timeMin / 1000.0f + << ", MaxRuntime = " << timeMax / 1000.0f << ", AvgRunTime = " << timeAvg / 1000.0f; + printf("Model = %s, NumThreads = %d, MinRunTime = %f ms, MaxRuntime = %f ms, AvgRunTime = %f ms\n", _flags->modelPath.substr(_flags->modelPath.find_last_of(DELIM_SLASH) + 1).c_str(), _flags->numThreads, timeMin / 1000.0f, timeMax / 1000.0f, timeAvg / 1000.0f); } @@ -325,13 +325,22 @@ int Benchmark::MarkAccuracy() { std::cout << std::endl; } auto status = session->RunGraph(); - if (status != 0) { - MS_LOG(ERROR) << "Inference error %d" << status; + if (status != RET_OK) { + MS_LOG(ERROR) << "Inference error " << status; return status; } - ReadCalibData(); - CompareOutput(); + status = ReadCalibData(); + if (status != RET_OK) { + MS_LOG(ERROR) << "Read calib data error " << status; + return status; + } + + status = CompareOutput(); + if (status != RET_OK) { + MS_LOG(ERROR) << "Compare output error " << status; + return status; + } return 0; } @@ -371,10 +380,10 @@ int Benchmark::RunBenchmark(const std::string &deviceType) { msInputs = session->GetInputs(); auto endPrepareTime = GetTimeUs(); #if defined(__arm__) - MS_LOG(INFO) << "PrepareTime = %lld ms, " << (endPrepareTime - startPrepareTime) / 1000; + MS_LOG(INFO) << "PrepareTime = " << (endPrepareTime - startPrepareTime) / 1000 << " ms"; printf("PrepareTime = %lld ms, ", (endPrepareTime - startPrepareTime) / 1000); #else - MS_LOG(INFO) << "PrepareTime = %ld ms, " << (endPrepareTime - startPrepareTime) / 1000; + MS_LOG(INFO) << "PrepareTime = " << (endPrepareTime - startPrepareTime) / 1000 << " ms "; printf("PrepareTime = %ld ms, ", (endPrepareTime - startPrepareTime) / 1000); #endif @@ -383,18 +392,21 @@ int Benchmark::RunBenchmark(const std::string &deviceType) { auto status = LoadInput(); if (status != 0) { MS_LOG(ERROR) << "Generate input data error"; + delete graphBuf; return status; } if (!_flags->calibDataPath.empty()) { status = MarkAccuracy(); if (status != 0) { MS_LOG(ERROR) << "Run MarkAccuracy error: %d" << status; + delete graphBuf; return status; } } else { status = MarkPerformance(); if (status != 0) { MS_LOG(ERROR) << "Run MarkPerformance error: %d" << status; + delete graphBuf; return status; } } @@ -509,13 +521,14 @@ int RunBenchmark(int argc, const char **argv) { } if (status != 0) { - MS_LOG(ERROR) << "Run Benchmark Error : " << status; + MS_LOG(ERROR) << "Run Benchmark " << flags.modelPath.substr(flags.modelPath.find_last_of(DELIM_SLASH) + 1).c_str() + << " Failed : " << status; return 1; } - MS_LOG(INFO) << "end of benchmark"; + MS_LOG(INFO) << "Run Benchmark " << flags.modelPath.substr(flags.modelPath.find_last_of(DELIM_SLASH) + 1).c_str() + << " Success."; return 0; } } // namespace lite } // namespace mindspore - diff --git a/mindspore/lite/tools/benchmark/benchmark.h b/mindspore/lite/tools/benchmark/benchmark.h index 787d5f1a287..36aed9f2a24 100644 --- a/mindspore/lite/tools/benchmark/benchmark.h +++ b/mindspore/lite/tools/benchmark/benchmark.h @@ -37,8 +37,8 @@ namespace mindspore::lite { enum MS_API InDataType { kImage = 0, kBinary = 1 }; -constexpr float relativeTolerance = 0.01; -constexpr float absoluteTolerance = 0.01; +constexpr float relativeTolerance = 1e-5; +constexpr float absoluteTolerance = 1e-8; struct MS_API CheckTensor { CheckTensor(const std::vector &shape, const std::vector &data) { @@ -143,4 +143,3 @@ class MS_API Benchmark { int MS_API RunBenchmark(int argc, const char **argv); } // namespace mindspore::lite #endif // MINNIE_BENCHMARK_BENCHMARK_H_ -