p04896132
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!10433 [MS LITE]fix benchamrk output 

From: @YeFeng_24
Reviewed-by: 
Signed-off-by:
This commit is contained in:
mindspore-ci-bot 2020-12-25 14:07:56 +08:00 committed by Gitee
parent ed4ddda060 af16958ae2
commit 2f50b42746
4 changed files with 80 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
mindspore/lite/src/lite_session.cc
View File

@ -290,15 +290,6 @@ void LiteSession::InitGraphOutputNodeMap(const lite::Model *model) {
}
}
void LiteSession::InitGraphOutputTensorNames(const lite::Model *model) {
MS_ASSERT(model != nullptr);
MS_ASSERT(this->output_tensor_names_.empty());
auto out_size = model->sub_graphs_.front()->output_indices_.size();
for (size_t i = 0; i < out_size; ++i) {
this->output_tensor_names_.emplace_back(std::to_string(model->sub_graphs_.front()->output_indices_[i]));
}
}
void LiteSession::InitGraphOutputTensorMap(const lite::Model *model) {
MS_ASSERT(model != nullptr);
MS_ASSERT(this->output_tensor_map_.empty());
@ -311,9 +302,12 @@ void LiteSession::InitGraphOutputTensorMap(const lite::Model *model) {
MS_LOG(ERROR) << "out_tensor is null!";
return;
}
this->output_tensor_map_.insert(std::make_pair(std::to_string(graph_out_index), out_tensor));
if (!out_tensor->tensor_name().empty()) {
this->output_tensor_map_.insert(std::make_pair(out_tensor->tensor_name(), out_tensor));
this->output_tensor_names_.emplace_back(out_tensor->tensor_name());
} else {
this->output_tensor_map_.insert(std::make_pair(std::to_string(graph_out_index), out_tensor));
this->output_tensor_names_.emplace_back(std::to_string(graph_out_index));
}
}
}
@ -339,7 +333,6 @@ void LiteSession::InitGraphInOutTensors(const lite::Model *model) {
InitGraphOutputTensors(model);
InitGraphInputMap(model);
InitGraphOutputNodeMap(model);
InitGraphOutputTensorNames(model);
InitGraphOutputTensorMap(model);
for (auto *tensor : this->inputs_) {
tensor->set_category(Tensor::Category::GRAPH_INPUT);

2
mindspore/lite/src/lite_session.h
View File

@ -88,8 +88,6 @@ class LiteSession : public session::LiteSession {
void InitGraphOutputNodeMap(const lite::Model *model);
void InitGraphOutputTensorNames(const lite::Model *model);
void InitGraphOutputTensorMap(const lite::Model *model);
void AdjustModelOutputTensorInitRefCount(const lite::Model *model);

94
mindspore/lite/tools/benchmark/benchmark.cc
View File

@ -159,8 +159,7 @@ int Benchmark::ReadInputFile() {
}
// calibData is FP32
int Benchmark::ReadCalibData() {
const char *calib_data_path = flags_->benchmark_data_file_.c_str();
int Benchmark::ReadCalibData(bool new_data, const char *calib_data_path) {
// read calib data
std::ifstream in_file(calib_data_path);
if (!in_file.good()) {
@ -232,7 +231,11 @@ int Benchmark::ReadTensorData(std::ifstream &in_file_stream, const std::string &
MS_LOG(ERROR) << "New CheckTensor failed, tensor name: " << tensor_name;
return RET_ERROR;
}
this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor));
if (has_new_data_) {
this->new_benchmark_data_.insert(std::make_pair(tensor_name, check_tensor));
} else {
this->benchmark_data_.insert(std::make_pair(tensor_name, check_tensor));
}
return RET_OK;
}
@ -240,26 +243,51 @@ int Benchmark::CompareOutput() {
std::cout << "================ Comparing Output data ================" << std::endl;
float total_bias = 0;
int total_size = 0;
for (const auto &calib_tensor : benchmark_data_) {
std::string node_or_tensor_name = calib_tensor.first;
tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name);
if (tensor == nullptr) {
MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name;
return RET_ERROR;
if (new_benchmark_data_.size() > benchmark_data_.size()) {
for (const auto &calib_tensor : new_benchmark_data_) {
std::string node_or_tensor_name = calib_tensor.first;
tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name);
if (tensor == nullptr) {
MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name;
return RET_ERROR;
}
int ret;
if (tensor->data_type() == kObjectTypeString) {
ret = CompareStringData(node_or_tensor_name, tensor, new_benchmark_data_);
} else {
ret =
CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size, new_benchmark_data_);
}
if (ret != RET_OK) {
MS_LOG(ERROR) << "Error in CompareData";
std::cerr << "Error in CompareData" << std::endl;
std::cout << "=======================================================" << std::endl << std::endl;
return ret;
}
}
int ret;
if (tensor->data_type() == kObjectTypeString) {
ret = CompareStringData(node_or_tensor_name, tensor);
} else {
ret = CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size);
}
if (ret != RET_OK) {
MS_LOG(ERROR) << "Error in CompareData";
std::cerr << "Error in CompareData" << std::endl;
std::cout << "=======================================================" << std::endl << std::endl;
return ret;
} else {
for (const auto &calib_tensor : benchmark_data_) {
std::string node_or_tensor_name = calib_tensor.first;
tensor::MSTensor *tensor = GetTensorByNodeOrTensorName(node_or_tensor_name);
if (tensor == nullptr) {
MS_LOG(ERROR) << "Get tensor failed, tensor name: " << node_or_tensor_name;
return RET_ERROR;
}
int ret;
if (tensor->data_type() == kObjectTypeString) {
ret = CompareStringData(node_or_tensor_name, tensor, benchmark_data_);
} else {
ret = CompareDataGetTotalBiasAndSize(node_or_tensor_name, tensor, &total_bias, &total_size, benchmark_data_);
}
if (ret != RET_OK) {
MS_LOG(ERROR) << "Error in CompareData";
std::cerr << "Error in CompareData" << std::endl;
std::cout << "=======================================================" << std::endl << std::endl;
return ret;
}
}
}
float mean_bias;
if (total_size != 0) {
mean_bias = total_bias / float_t(total_size) * 100;
@ -291,7 +319,8 @@ tensor::MSTensor *Benchmark::GetTensorByNodeOrTensorName(const std::string &node
return tensor;
}
int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tensor) {
int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tensor,
std::unordered_map<std::string, CheckTensor *> benchmark_data) {
auto iter = this->benchmark_data_.find(name);
if (iter != this->benchmark_data_.end()) {
std::vector<std::string> calib_strings = iter->second->strings_data;
@ -314,7 +343,8 @@ int Benchmark::CompareStringData(const std::string &name, tensor::MSTensor *tens
}
int Benchmark::CompareDataGetTotalBiasAndSize(const std::string &name, tensor::MSTensor *tensor, float *total_bias,
int *total_size) {
int *total_size,
std::unordered_map<std::string, CheckTensor *> benchmark_data) {
float bias = 0;
auto mutableData = tensor->MutableData();
if (mutableData == nullptr) {
@ -323,19 +353,19 @@ int Benchmark::CompareDataGetTotalBiasAndSize(const std::string &name, tensor::M
}
switch (msCalibDataType) {
case TypeId::kNumberTypeFloat: {
bias = CompareData<float>(name, tensor->shape(), mutableData);
bias = CompareData<float>(name, tensor->shape(), mutableData, benchmark_data);
break;
}
case TypeId::kNumberTypeInt8: {
bias = CompareData<int8_t>(name, tensor->shape(), mutableData);
bias = CompareData<int8_t>(name, tensor->shape(), mutableData, benchmark_data);
break;
}
case TypeId::kNumberTypeUInt8: {
bias = CompareData<uint8_t>(name, tensor->shape(), mutableData);
bias = CompareData<uint8_t>(name, tensor->shape(), mutableData, benchmark_data);
break;
}
case TypeId::kNumberTypeInt32: {
bias = CompareData<int32_t>(name, tensor->shape(), mutableData);
bias = CompareData<int32_t>(name, tensor->shape(), mutableData, benchmark_data);
break;
}
default:
@ -423,12 +453,20 @@ int Benchmark::MarkAccuracy() {
std::cerr << "Inference error " << status << std::endl;
return status;
}
status = ReadCalibData();
const char *calib_data_path = flags_->benchmark_data_file_.c_str();
status = ReadCalibData(false, calib_data_path);
if (status != RET_OK) {
MS_LOG(ERROR) << "Read calib data error " << status;
std::cerr << "Read calib data error " << status << std::endl;
return status;
has_new_data_ = true;
status = ReadCalibData(true, flags_->benchmark_data_file_.append("_1").c_str());
if (status != RET_OK) {
MS_LOG(ERROR) << "no new data.";
has_new_data_ = false;
return status;
}
}
status = CompareOutput();
if (status != RET_OK) {
MS_LOG(ERROR) << "Compare output error " << status;

16
mindspore/lite/tools/benchmark/benchmark.h
View File

@ -129,7 +129,7 @@ class MS_API Benchmark {
int ReadInputFile();
int ReadCalibData();
int ReadCalibData(bool new_data, const char *calib_data_path);
int ReadTensorData(std::ifstream &in_file_stream, const std::string &tensor_name, const std::vector<size_t> &dims);
@ -137,10 +137,11 @@ class MS_API Benchmark {
tensor::MSTensor *GetTensorByNodeOrTensorName(const std::string &node_or_tensor_name);
int CompareStringData(const std::string &name, tensor::MSTensor *tensor);
int CompareStringData(const std::string &name, tensor::MSTensor *tensor,
std::unordered_map<std::string, CheckTensor *> benchmark_data);
int CompareDataGetTotalBiasAndSize(const std::string &name, tensor::MSTensor *tensor, float *total_bias,
int *total_size);
int *total_size, std::unordered_map<std::string, CheckTensor *> benchmark_data);
int InitCallbackParameter();
@ -150,10 +151,11 @@ class MS_API Benchmark {
// tensorData need to be converter first
template <typename T>
float CompareData(const std::string &nodeName, const std::vector<int> &msShape, const void *tensor_data) {
float CompareData(const std::string &nodeName, const std::vector<int> &msShape, const void *tensor_data,
std::unordered_map<std::string, CheckTensor *> benchmark_data) {
const T *msTensorData = static_cast<const T *>(tensor_data);
auto iter = this->benchmark_data_.find(nodeName);
if (iter != this->benchmark_data_.end()) {
auto iter = benchmark_data.find(nodeName);
if (iter != benchmark_data.end()) {
std::vector<size_t> castedMSShape;
size_t shapeSize = 1;
for (int64_t dim : msShape) {
@ -238,11 +240,13 @@ class MS_API Benchmark {
int MarkAccuracy();
private:
bool has_new_data_ = false;
BenchmarkFlags *flags_;
session::LiteSession *session_{nullptr};
std::vector<mindspore::tensor::MSTensor *> ms_inputs_;
std::unordered_map<std::string, std::vector<mindspore::tensor::MSTensor *>> ms_outputs_;
std::unordered_map<std::string, CheckTensor *> benchmark_data_;
std::unordered_map<std::string, CheckTensor *> new_benchmark_data_;
std::unordered_map<std::string, TypeId> data_type_map_{{"FLOAT", TypeId::kNumberTypeFloat},
{"INT8", TypeId::kNumberTypeInt8},
{"INT32", TypeId::kNumberTypeInt32},