p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!34158 read calibration directory in alphabetical order 

Merge pull request !34158 from liyan2022/dev_main
This commit is contained in:
i-robot 2022-05-11 09:37:19 +00:00 committed by Gitee
parent 6cb21da318 f0bcd3076a
commit 1588a18999
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
3 changed files with 75 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
mindspore/lite/tools/converter/config_parser/preprocess_parser.cc
View File

@ -24,6 +24,7 @@
#include "src/common/log_adapter.h"
#include "mindspore/lite/tools/common/string_util.h"
#include "include/errorcode.h"
#include "src/common/file_utils.h"
namespace mindspore {
namespace lite {
@ -193,42 +194,29 @@ int PreprocessParser::CollectCalibInputs(const std::map<std::string, std::string
};
for (const auto &image_path : calibrate_data_path) {
DIR *root = opendir(image_path.second.c_str());
if (root == nullptr) {
MS_LOG(ERROR) << "cant open dir: " << image_path.second.c_str();
return RET_PARAM_INVALID;
std::vector<std::string> file_names;
auto ret = ReadDirectory(image_path.second.c_str(), &file_names);
if (ret != RET_OK) {
return ret;
}
struct dirent *image_dir = readdir(root);
size_t count = 0;
while (image_dir != nullptr && count < limited_count) {
std::string file_name(image_dir->d_name);
if (file_name != "." && file_name != "..") {
const std::string file_path = image_path.second + "/" + file_name;
AddImage(file_path, image_path.first);
count++;
MS_ASSERT(file_names.size() >= kDotDirCount);
if (file_names.size() < (limited_count + kDotDirCount)) {
MS_LOG(ERROR) << "file count less than calibrate size, file count: " << file_names.size()
<< " limited_count: " << limited_count << " kDotDirCount: " << kDotDirCount;
return RET_ERROR;
}
for (size_t index = 0; index < (limited_count + kDotDirCount); index++) {
if (file_names[index] == "." || file_names[index] == "..") {
continue;
}
image_dir = readdir(root);
}
auto ret = closedir(root);
if (ret != 0) {
MS_LOG(ERROR) << " close dir failed.";
return RET_ERROR;
}
if (inputs->find(image_path.first) != inputs->end()) {
auto &cur_inputs = inputs->at(image_path.first);
std::sort(cur_inputs.begin(), cur_inputs.end());
} else {
MS_LOG(ERROR) << "cant find " << image_path.first << " at input maps.";
return RET_ERROR;
}
if (count != limited_count) {
MS_LOG(ERROR) << " data path: " << image_path.second << " data count:" << count
<< " < limited_count:" << limited_count;
return RET_ERROR;
const std::string file_path = image_path.second + "/" + file_names[index];
MS_LOG(DEBUG) << "calibrate file_path: " << file_path;
AddImage(file_path, image_path.first);
}
}
return RET_OK;
}
int PreprocessParser::ParseImageNormalize(const DataPreProcessString &data_pre_process_str,
preprocess::ImagePreProcessParam *image_pre_process) {
if (!data_pre_process_str.normalize_mean.empty() &&
@ -245,6 +233,7 @@ int PreprocessParser::ParseImageNormalize(const DataPreProcessString &data_pre_p
return RET_OK;
}
int PreprocessParser::ParseImageResize(const DataPreProcessString &data_pre_process_str,
preprocess::ImagePreProcessParam *image_pre_process) {
if (!data_pre_process_str.resize_width.empty()) {
@ -277,6 +266,7 @@ int PreprocessParser::ParseImageResize(const DataPreProcessString &data_pre_proc
}
return RET_OK;
}
int PreprocessParser::ParseImageCenterCrop(const DataPreProcessString &data_pre_process_str,
preprocess::ImagePreProcessParam *image_pre_process) {
if (!data_pre_process_str.center_crop_width.empty()) {
@ -301,5 +291,43 @@ int PreprocessParser::ParseImageCenterCrop(const DataPreProcessString &data_pre_
}
return RET_OK;
}
int PreprocessParser::ReadDirectory(const std::string &path, std::vector<std::string> *file_names) {
if (file_names == nullptr) {
MS_LOG(ERROR) << "file_names is null";
return RET_ERROR;
}
DIR *dp = opendir(path.empty() ? "." : RealPath(path.c_str()).c_str());
if (dp == nullptr) {
MS_LOG(ERROR) << "cant open dir: " << path;
return RET_PARAM_INVALID;
}
size_t file_count = 0;
while (true) {
if (file_count >= kFileCountLimit) {
break;
}
struct dirent *de = readdir(dp);
if (de == NULL) {
break;
}
file_names->push_back(std::string(de->d_name));
file_count++;
}
auto ret = closedir(dp);
if (ret != 0) {
MS_LOG(ERROR) << " close dir failed.";
return RET_ERROR;
}
if (file_count >= kFileCountLimit) {
MS_LOG(ERROR) << " read calibrate directory failed, files count exceed limit: " << kFileCountLimit;
return RET_ERROR;
}
std::sort(file_names->begin(), file_names->end());
return RET_OK;
}
} // namespace lite
} // namespace mindspore

4
mindspore/lite/tools/converter/config_parser/preprocess_parser.h
View File

@ -24,6 +24,8 @@
namespace mindspore {
namespace lite {
constexpr int kDotDirCount = 2;
constexpr int kFileCountLimit = 100000;
class PreprocessParser {
public:
static int ParsePreprocess(const DataPreProcessString &data_pre_process_str,
@ -50,6 +52,8 @@ class PreprocessParser {
static int CollectCalibInputs(const std::map<std::string, std::string> &calibrate_data_path, size_t limited_count,
std::map<std::string, std::vector<std::string>> *inputs);
static int ReadDirectory(const std::string &path, std::vector<std::string> *file_names);
};
} // namespace lite
} // namespace mindspore

30
mindspore/lite/tools/converter/quantizer/debug_info_manager.cc
View File

@ -171,11 +171,12 @@ int DebugInfoManager::SaveInfo(const std::string &file_path) {
int DebugInfoManager::SetOriginStaticInfo(QuantDebugInfo *quant_debug_info, const mindspore::lite::Tensor &tensor,
const quant::DebugMode &debug_mode) {
if (debug_mode == quant::DETAIL) {
if (tensor.data_type() == kNumberTypeFloat32) {
TypeId data_type = tensor.data_type();
if (data_type == kNumberTypeFloat32) {
GetStatByTensor<float>(static_cast<float *>(tensor.data()), tensor.ElementsNum(), quant_debug_info);
} else if (tensor.data_type() == kNumberTypeInt32) {
} else if (data_type == kNumberTypeInt32) {
GetStatByTensor<int>(static_cast<int *>(tensor.data()), tensor.ElementsNum(), quant_debug_info);
} else if (tensor.data_type() == kNumberTypeInt8) {
} else if (data_type == kNumberTypeInt8) {
GetStatByTensor<int8_t>(static_cast<int8_t *>(tensor.data()), tensor.ElementsNum(), quant_debug_info);
} else {
MS_LOG(ERROR) << tensor.tensor_name() << " unsupported data type " << tensor.data_type();
@ -312,8 +313,8 @@ int DebugInfoManager::AddComparedInfo(const mindspore::MSCallBackParam &call_bac
std::map<std::string, mindspore::schema::Tensor *> DebugInfoManager::ParseInputTensors(
const mindspore::lite::LiteModel &model) {
std::map<std::string, mindspore::schema::Tensor *> maps;
for (auto node : model.all_nodes_) {
for (auto index : node->input_indices_) {
for (auto &node : model.all_nodes_) {
for (auto &index : node->input_indices_) {
auto tensor_name = model.all_tensors_[index]->name()->str();
maps[tensor_name] = model.all_tensors_[index];
}
@ -323,8 +324,8 @@ std::map<std::string, mindspore::schema::Tensor *> DebugInfoManager::ParseInputT
std::map<std::string, mindspore::schema::Tensor *> DebugInfoManager::ParseOutputTensorFromModel(const Model &model) {
std::map<std::string, mindspore::schema::Tensor *> maps;
for (auto node : model.all_nodes_) {
for (auto index : node->output_indices_) {
for (auto &node : model.all_nodes_) {
for (auto &index : node->output_indices_) {
auto tensor_name = model.all_tensors_[index]->name()->str();
maps[tensor_name] = model.all_tensors_[index];
}
@ -549,7 +550,7 @@ void DebugInfoManager::PrintQuantParam() {
std::cout << quant_param.node_type << ",";
std::cout << quant_param.tensor_name << ",";
std::cout << quant_param.element_num << ",";
for (auto dim : quant_param.dims) {
for (auto &dim : quant_param.dims) {
std::cout << dim << " ";
}
std::cout << ",";
@ -581,7 +582,7 @@ int DebugInfoManager::SaveQuantParam(const std::string &file_path) {
out_file << quant_param.node_type << ",";
out_file << quant_param.tensor_name << ",";
out_file << quant_param.element_num << ",";
for (auto dim : quant_param.dims) {
for (auto &dim : quant_param.dims) {
out_file << dim << " ";
}
out_file << ",";
@ -692,7 +693,7 @@ int DebugInfoManager::StatisticsDataPerRound(
auto quant_before_callBack =
GetBeforeCallBack(quant_input_tensor_map, op_parameters, false, config.commonQuantParam.debug_mode);
auto quant_after_callBack = GetAfterCallBack(op_parameters, false, config.commonQuantParam.debug_mode);
for (auto tensor : quant->GetInputs()) {
for (auto &tensor : quant->GetInputs()) {
auto tensor_data = tensor.MutableData();
CHECK_NULL_RETURN(tensor_data);
ret = memcpy_s(tensor_data, tensor.DataSize(), origin->GetInputByTensorName(tensor.Name()).Data().get(),
@ -713,12 +714,7 @@ int DebugInfoManager::StatisticsDataPerRound(
std::string DebugInfoManager::CreateFilePath(const std::string &dir_path, const std::string &file_name) {
auto real_path = RealPath(dir_path.c_str());
std::string file_path{};
#ifdef _WIN32
file_path = real_path + "\\" + file_name;
#else
file_path = real_path + "/" + file_name;
#endif
std::string file_path = real_path + FILE_SEPARATOR + file_name;
return file_path;
}
@ -731,7 +727,7 @@ int DebugInfoManager::CompareOriginWithQuant(const std::shared_ptr<mindspore::Mo
auto begin = GetTimeUs();
auto origin_input_tensor_map = ParseInputTensors(origin_lite_model);
auto quant_input_tensor_map = ParseInputTensors(quant_lite_model);
for (auto tensor : origin->GetOutputs()) {
for (auto &tensor : origin->GetOutputs()) {
origin_outputs_[tensor.Name()] = tensor;
}
int ret;