Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!11040 mixed quant 

From: @xutianchun
Reviewed-by: 
Signed-off-by:
This commit is contained in:
mindspore-ci-bot 2021-01-11 17:26:02 +08:00 committed by Gitee
parent 9d5bdf1ddc bb1c4e3c6a
commit 1908d6b8c4
19 changed files with 740 additions and 345 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/lite/src/ops/adder.cc
View File

@ -21,7 +21,7 @@
#include "include/errorcode.h"
#include "src/common/log_adapter.h"
#ifdef PRIMITIVE_WRITEABLE
#include "tools/converter/quantizer/quantize_util.h"
#include "src/param_value_lite.h"
#endif
#ifndef PRIMITIVE_WRITEABLE

2
mindspore/lite/src/ops/conv2d.cc
View File

@ -24,7 +24,7 @@
#include "src/common/log_adapter.h"
#ifdef PRIMITIVE_WRITEABLE
#include <float.h>
#include "tools/converter/quantizer/quantize_util.h"
#include "src/param_value_lite.h"
#endif
#ifndef PRIMITIVE_WRITEABLE

3
mindspore/lite/src/ops/deconv2d.cc
View File

@ -21,8 +21,7 @@
#include "src/common/log_adapter.h"
#ifdef PRIMITIVE_WRITEABLE
#include <float.h>
#include "tools/converter/quantizer/quantize_util.h"
#include "src/param_value_lite.h"
#endif
#ifndef PRIMITIVE_WRITEABLE

2
mindspore/lite/src/ops/depthwise_conv2d.cc
View File

@ -19,7 +19,7 @@
#include <memory>
#include <string>
#ifdef PRIMITIVE_WRITEABLE
#include "tools/converter/quantizer/quantize_util.h"
#include "src/param_value_lite.h"
#endif
#ifndef PRIMITIVE_WRITEABLE
#include "src/ops/ops_register.h"

2
mindspore/lite/src/ops/matmul.cc
View File

@ -18,7 +18,7 @@
#include <memory>
#include <utility>
#ifdef PRIMITIVE_WRITEABLE
#include "tools/converter/quantizer/quantize_util.h"
#include "src/param_value_lite.h"
#endif
#ifndef PRIMITIVE_WRITEABLE

3
mindspore/lite/src/ops/maximum.cc
View File

@ -19,8 +19,7 @@
#include "src/common/log_adapter.h"
#ifdef PRIMITIVE_WRITEABLE
#include <float.h>
#include "tools/converter/quantizer/quantize_util.h"
#include "src/param_value_lite.h"
#endif
#ifndef PRIMITIVE_WRITEABLE

2
mindspore/lite/src/ops/maximum_grad.cc
View File

@ -19,7 +19,7 @@
#include "src/common/log_adapter.h"
#ifdef PRIMITIVE_WRITEABLE
#include <float.h>
#include "tools/converter/quantizer/quantize_util.h"
#include "src/param_value_lite.h"
#endif
#ifndef PRIMITIVE_WRITEABLE

2
mindspore/lite/src/ops/minimum_grad.cc
View File

@ -19,7 +19,7 @@
#include "src/common/log_adapter.h"
#ifdef PRIMITIVE_WRITEABLE
#include <float.h>
#include "tools/converter/quantizer/quantize_util.h"
#include "src/param_value_lite.h"
#endif
#ifndef PRIMITIVE_WRITEABLE

6
mindspore/lite/src/ops/primitive_c.cc
View File

@ -389,7 +389,9 @@ void PrimitiveC::set_input_quant_params(const std::vector<std::vector<schema::Qu
}
void PrimitiveC::set_input_quant_param(const size_t &index, const std::vector<schema::QuantParamT> &input_quant_param) {
MS_ASSERT(index < this->input_quant_param_.size());
if (index >= this->input_quant_param_.size()) {
this->input_quant_param_.resize(index + 1);
}
this->input_quant_param_.at(index) = input_quant_param;
}
@ -495,7 +497,7 @@ std::shared_ptr<PrimitiveC> GetTupleGetItemPrim() {
}
template <typename T, typename = std::enable_if<std::is_base_of<PrimitiveC, T>::value>>
std::shared_ptr<PrimitiveC> NewPrimitiveC(const Primitive &prim, const std::vector<AnfNodePtr> &inputs,
std::shared_ptr<PrimitiveC> NewPrimitiveC(const mindspore::Primitive &prim, const std::vector<AnfNodePtr> &inputs,
const schema::QuantType &quantType) {
auto primc = std::make_shared<T>();
if (primc == nullptr) {

2
mindspore/lite/tools/converter/anf_transform.cc
View File

@ -204,7 +204,7 @@ FuncGraphPtr AnfTransform::TransformSingleFuncGraph(const FuncGraphPtr &old_grap
ReturnCode::GetSingleReturnCode()->UpdateReturnCode(RET_ERROR);
return nullptr;
}
this->mQuantizer = std::make_unique<quant::WeightQuantizer>(new_graph, config->quantWeightSize,
this->mQuantizer = std::make_unique<quant::WeightQuantizer>(new_graph, config->configFile, config->quantWeightSize,
config->quantWeightChannel, config->bitNum);
if (mQuantizer == nullptr) {
MS_LOG(ERROR) << "New WeightQuantizer failed";

2
mindspore/lite/tools/converter/converter.cc
View File

@ -32,8 +32,6 @@
#include "tools/anf_exporter/anf_exporter.h"
#include "tools/anf_importer/import_from_mindir.h"
#include "proto/onnx.pb.h"
#include "tools/converter/quantizer/post_training_quantizer.h"
#include "tools/converter/quantizer/quant_cast.h"
#include "include/version.h"
namespace mindspore {

1
mindspore/lite/tools/converter/legacy_optimizer/graph/tensor_name_pass.cc
View File

@ -16,7 +16,6 @@
#include "tools/converter/legacy_optimizer/graph/tensor_name_pass.h"
#include "tools/converter/converter_context.h"
#include "tools/converter/quantizer/quantize_util.h"
#include "tools/common/tensor_util.h"
namespace mindspore::lite {

1
mindspore/lite/tools/converter/quantizer/CMakeLists.txt
View File

@ -6,7 +6,6 @@ include_directories(${3RD_DIR}/opencv/build/include/opencv4)
file(GLOB QUANTIZER
${CMAKE_CURRENT_SOURCE_DIR}/calc_quant_param.cc
${CMAKE_CURRENT_SOURCE_DIR}/quantizer.cc
${CMAKE_CURRENT_SOURCE_DIR}/aware_quantizer.cc
${CMAKE_CURRENT_SOURCE_DIR}/quantize_util.cc
${CMAKE_CURRENT_SOURCE_DIR}/post_training_quantizer.cc
${CMAKE_CURRENT_SOURCE_DIR}/quant_cast.cc

318
mindspore/lite/tools/converter/quantizer/post_training_quantizer.cc
View File

@ -39,6 +39,7 @@
#include "tools/common/tensor_util.h"
#include "src/common/file_utils.h"
#include "src/common/utils.h"
#include "tools/converter/quantizer/weight_quantizer.h"
using std::string;
using std::vector;
@ -380,182 +381,16 @@ STATUS Calibrator::AddQuantizedOp(const CNodePtr &node) {
return RET_OK;
}
void Calibrator::AddImage(const string &file, size_t index) {
if (index >= images_.size()) {
MS_LOG(ERROR) << "images_ size: " << images_.size() << " but index: " << index;
return;
}
auto exist = [](const string &file) {
struct stat buf {};
return stat(file.c_str(), &buf) == 0;
};
if (exist(file)) {
this->images_[index].push_back(file);
} else {
MS_LOG(WARNING) << "invalid image file path: " << file;
}
}
STATUS Calibrator::GenerateInputData(size_t input_index, size_t image_index,
mindspore::tensor::MSTensor *tensor) const {
MS_ASSERT(tensor != nullptr);
if (input_index >= images_.size()) {
MS_LOG(ERROR) << "images_ size: " << images_.size() << " but input_index: " << input_index;
return RET_ERROR;
}
if (image_index >= images_[input_index].size()) {
MS_LOG(ERROR) << "images_[input_index] size: " << images_[input_index].size()
<< " but image_index: " << image_index;
return RET_ERROR;
}
string path = images_[input_index][image_index];
MS_LOG(INFO) << "read image: " << path;
size_t size;
char *bin_buf = ReadFile(path.c_str(), &size);
if (bin_buf == nullptr) {
MS_LOG(ERROR) << "ReadFile return nullptr";
return RET_NULL_PTR;
}
auto data = tensor->MutableData();
if (data == nullptr) {
MS_LOG(ERROR) << "Get tensor MutableData return nullptr";
return RET_NULL_PTR;
}
if (size != tensor->Size()) {
MS_LOG(ERROR) << "the input data is not consistent with model input, file_size: " << size
<< " input tensor size: " << tensor->Size();
return RET_ERROR;
}
auto ret = memcpy_s(data, tensor->Size(), bin_buf, size);
if (ret != EOK) {
MS_LOG(ERROR) << "memcpy_s error: " << ret;
delete[] bin_buf;
return RET_ERROR;
}
delete[] bin_buf;
return RET_OK;
return CopyInputDataToTensor(input_index, image_index, images_, tensor);
}
STATUS Calibrator::CollectImages() {
this->images_.resize(config_param_.image_paths.size());
auto input_i = 0;
bool multi_input = config_param_.image_paths.size() > 1;
for (const auto &image_path : config_param_.image_paths) {
DIR *root = opendir(image_path.c_str());
if (root == nullptr) {
MS_LOG(ERROR) << "invalid image path: " << image_path;
return RET_PARAM_INVALID;
}
struct dirent *image_dir = readdir(root);
size_t count = 0;
while (image_dir != nullptr) {
string file_name(image_dir->d_name);
if (file_name != "." && file_name != "..") {
const std::string file_path = image_path + "/" + file_name;
if (multi_input || config_param_.batch_count == 0) {
this->AddImage(file_path, input_i);
count++;
} else if (count < config_param_.batch_count) {
this->AddImage(file_path, input_i);
count++;
} else {
break;
}
}
image_dir = readdir(root);
}
std::sort(images_[input_i].begin(), images_[input_i].end());
if (config_param_.batch_count != 0 && config_param_.batch_count < images_[input_i].size()) {
images_[input_i].resize(config_param_.batch_count);
}
closedir(root);
input_i++;
}
return RET_OK;
return CollectCalibInputs(config_param_.image_paths, config_param_.batch_count, &images_);
}
STATUS Calibrator::ReadConfig() {
if (config_path_.empty() || config_path_.length() > PATH_MAX) {
MS_LOG(ERROR) << "invalid config path!";
return RET_PARAM_INVALID;
}
// check whether config file path is valid
char *resolved_path = new (std::nothrow) char[PATH_MAX]{0};
if (resolved_path == nullptr) {
MS_LOG(ERROR) << "New an object failed.";
return RET_ERROR;
}
#ifdef _WIN32
if (_fullpath(resolved_path, config_path_.c_str(), 1024) != nullptr) {
config_path_ = string(resolved_path);
}
#else
if (realpath(config_path_.c_str(), resolved_path) != nullptr) {
config_path_ = string(resolved_path);
}
#endif
std::ifstream fs(config_path_.c_str(), std::ifstream::in);
if (!fs.is_open()) {
MS_LOG(ERROR) << "config proto file %s open failed: " << config_path_;
delete[] resolved_path;
return RET_PARAM_INVALID;
}
std::string line;
while (std::getline(fs, line)) {
auto index = line.find('=');
if (index == std::string::npos) {
MS_LOG(ERROR) << "the config file is invalid, can not find '=', please check";
delete[] resolved_path;
return RET_PARAM_INVALID;
}
auto key = line.substr(0, index);
auto value = line.substr(index + 1);
Trim(&key);
Trim(&value);
if (key == "image_path") {
auto &raw_image_paths = value;
auto ind = raw_image_paths.find(',');
while (ind != std::string::npos) {
auto image_path = raw_image_paths.substr(0, ind);
Trim(&image_path);
config_param_.image_paths.push_back(image_path);
raw_image_paths = raw_image_paths.substr(ind + 1);
Trim(&raw_image_paths);
ind = raw_image_paths.find(',');
}
config_param_.image_paths.push_back(raw_image_paths);
} else if (key == "batch_count") {
config_param_.batch_count = std::stoul(value);
} else if (key == "thread_num") {
config_param_.thread_num = std::stoul(value);
} else if (key == "method_x") {
if (value != kMethodKL && value != kMethodMaxMin && value != kMethodOutlier) {
MS_LOG(WARNING) << "unsupported method_x: " << value << ". Use default value.";
} else {
config_param_.method_x = value;
}
} else if (key == "bias_correction") {
std::for_each(value.begin(), value.end(), ::tolower);
if (value == "true") {
config_param_.bias_correction = true;
}
} else {
MS_LOG(WARNING) << "unsupported parameter";
}
}
for (const auto &path : config_param_.image_paths) {
MS_LOG(DEBUG) << "calibration data_path: " << path;
}
MS_LOG(DEBUG) << "batch_count: " << config_param_.batch_count << " "
<< "method_x: " << config_param_.method_x << " "
<< "thread_num: " << config_param_.thread_num << " "
<< "bias_correction: " << config_param_.bias_correction;
delete[] resolved_path;
fs.close();
return RET_OK;
}
STATUS Calibrator::ReadConfig() { return ParseConfigFile(config_path_, &config_param_); }
Calibrator::Calibrator(string path, size_t bit_num, int quant_max, int quant_min)
: config_path_(std::move(path)), bit_num_(bit_num), quant_max_(quant_max), quant_min_(quant_min) {}
@ -621,8 +456,8 @@ STATUS PostTrainingQuantizer::DoQuantOutput(double scale, int zeropoint, struct
return RET_OK;
}
STATUS PostTrainingQuantizer::DoWeightQuant(const AnfNodePtr &weight, std::shared_ptr<PrimitiveC> primitive_c,
bool perchanel) const {
STATUS PostTrainingQuantizer::DoWeightQuant(const std::string &op_name, const AnfNodePtr &weight,
std::shared_ptr<PrimitiveC> primitive_c, bool perchanel) const {
MS_ASSERT(weight != nullptr);
MS_ASSERT(lite_primitive != nullptr);
// perlayer
@ -640,8 +475,21 @@ STATUS PostTrainingQuantizer::DoWeightQuant(const AnfNodePtr &weight, std::share
MS_LOG(ERROR) << weight->fullname_with_scope() << " can not get value";
return RET_NULL_PTR;
}
auto status = QuantFilter<int8_t>(paramValue, std::move(primitive_c), QuantType_PostTraining, quant_max, quant_min,
bit_num, perchanel);
auto bit_num_t = bit_num;
auto quant_max_t = quant_max;
auto quant_min_t = quant_min;
if (calibrator_->config_param_.mixed) {
auto opname_iter = opname_bit_.find(op_name);
if (opname_iter == opname_bit_.end()) {
MS_LOG(WARNING) << op_name << " not in the opname_bit_ map";
} else {
bit_num_t = opname_iter->second;
quant_max_t = (1 << (unsigned int)(bit_num_t - 1)) - 1;
quant_min_t = -(1 << (unsigned int)(bit_num_t - 1));
}
}
auto status = QuantFilter<int8_t>(paramValue, std::move(primitive_c), QuantType_PostTraining, quant_max_t,
quant_min_t, bit_num_t, perchanel);
if (status != RET_OK) {
MS_LOG(ERROR) << "QuantFilter failed: " << status;
return status;
@ -921,7 +769,7 @@ STATUS PostTrainingQuantizer::QuantNode() {
}
if (abstractTensor->element()->GetTypeTrack()->type_id() == kNumberTypeFloat32) {
MS_LOG(DEBUG) << "this parameter do quant";
DoWeightQuant(input_node, primitive_c, false);
DoWeightQuant(op_name, input_node, primitive_c, false);
} else {
MS_LOG(DEBUG) << "this parameter no need to do quant";
}
@ -943,7 +791,7 @@ STATUS PostTrainingQuantizer::QuantNode() {
op_type == PrimitiveType_FullConnection) {
perchannel = true;
}
DoWeightQuant(weight, primitive_c, perchannel);
DoWeightQuant(op_name, weight, primitive_c, perchannel);
// do bias quant
if (cnode->inputs().size() == 4) {
auto bias = cnode->input(3);
@ -982,18 +830,8 @@ STATUS PostTrainingQuantizer::UpdateDivergInverval() {
* 3. save quantied node
**/
STATUS PostTrainingQuantizer::PreProcess() {
if (this->calibrator_ == nullptr) {
MS_LOG(ERROR) << "calibrator is null!";
return RET_ERROR;
}
// 1. generate config param
STATUS status = calibrator_->ReadConfig();
if (status != RET_OK) {
MS_LOG(ERROR) << "read proto text failed!";
return status;
}
// 2. collect image files
status = calibrator_->CollectImages();
auto status = calibrator_->CollectImages();
if (status != RET_OK) {
MS_LOG(ERROR) << "collect images failed!";
return status;
@ -1560,55 +1398,49 @@ STATUS PostTrainingQuantizer::ComputeThreshold() { return this->calibrator_->Com
STATUS PostTrainingQuantizer::DoQuantize(FuncGraphPtr func_graph) {
MS_LOG(INFO) << "start to parse config file";
STATUS status = PreProcess();
if (this->calibrator_ == nullptr) {
MS_LOG(ERROR) << "calibrator is null!";
return RET_ERROR;
}
// 1. generate config param
STATUS status = calibrator_->ReadConfig();
if (status != RET_OK) {
MS_LOG(ERROR) << "read proto text failed!";
return status;
}
if (calibrator_->config_param_.mixed) {
// get opname_bit map
auto weight_quant_func_graph = CopyFuncGraph(func_graph);
if (weight_quant_func_graph == nullptr) {
MS_LOG(ERROR) << "CopyFuncGraph error";
return RET_ERROR;
}
WeightQuantizer weight_quantizer(weight_quant_func_graph, calibrator_->config_param_);
weight_quantizer.flags = flags;
status = weight_quantizer.DoQuantize(weight_quant_func_graph);
if (status != RET_OK) {
MS_LOG(ERROR) << "Do mix weight quant error";
return RET_ERROR;
}
opname_bit_ = weight_quantizer.opname_bit_;
}
status = PreProcess();
if (status != RET_OK) {
MS_LOG(ERROR) << "do pre process failed!";
return status;
}
// anf -- fb
auto meta_graph = Export(func_graph, true, true);
if (meta_graph == nullptr) {
MS_LOG(ERROR) << "Export to meta_graph return nullptr";
return RET_ERROR;
}
// transform
GraphDefTransform transform;
transform.SetGraphDef(meta_graph);
flags.quantType = schema::QuantType_QUANT_NONE;
status = transform.Transform(flags);
if (status != RET_OK) {
MS_LOG(ERROR) << "FBTransform model failed " << status;
return RET_ERROR;
}
MS_LOG(INFO) << "start create session";
flatbuffers::FlatBufferBuilder builder(1024);
auto offset = schema::MetaGraph::Pack(builder, meta_graph);
builder.Finish(offset);
schema::FinishMetaGraphBuffer(builder, offset);
size_t size = builder.GetSize();
auto *content = reinterpret_cast<const char *>(builder.GetBufferPointer());
if (content == nullptr) {
MS_LOG(ERROR) << "GetBufferPointer nullptr";
return RET_ERROR;
}
auto model = lite::Model::Import(content, size);
Context ctx;
ctx.thread_num_ = calibrator_->GetThreadNum();
fp32_session_ = dynamic_cast<mindspore::lite::LiteSession *>(session::LiteSession::CreateSession(&ctx));
fp32_session_ = CreateSessionByFuncGraph(func_graph, flags, calibrator_->GetThreadNum());
if (fp32_session_ == nullptr) {
MS_LOG(ERROR) << "create session failed!";
return RET_ERROR;
}
auto ret = fp32_session_->CompileGraph(model);
if (ret != lite::RET_OK) {
MS_LOG(ERROR) << "compile graph error";
return RET_ERROR;
}
MS_LOG(INFO) << "start to update divergence's max value";
status = DoInference();
if (status != RET_OK) {
@ -1647,49 +1479,13 @@ STATUS PostTrainingQuantizer::DoQuantize(FuncGraphPtr func_graph) {
if (calibrator_->GetBiasCorrection()) {
// init in8 session
// anf -- fb
auto int8_meta_graph = Export(func_graph, true, true);
if (int8_meta_graph == nullptr) {
MS_LOG(ERROR) << "Export to int8_meta_graph return nullptr";
return RET_ERROR;
}
// transform
GraphDefTransform fb_transform;
fb_transform.SetGraphDef(int8_meta_graph);
MS_LOG(INFO) << "create quant session";
flags.quantType = schema::QuantType_PostTraining;
status = fb_transform.Transform(flags);
if (status != RET_OK) {
MS_LOG(ERROR) << "FBTransform model failed " << status;
return RET_ERROR;
}
MS_LOG(INFO) << "start create quantized session";
flatbuffers::FlatBufferBuilder int8_builder(1024);
auto int8_offset = schema::MetaGraph::Pack(int8_builder, int8_meta_graph);
int8_builder.Finish(int8_offset);
schema::FinishMetaGraphBuffer(int8_builder, int8_offset);
size = int8_builder.GetSize();
auto *int8_content = reinterpret_cast<const char *>(int8_builder.GetBufferPointer());
if (int8_content == nullptr) {
MS_LOG(ERROR) << "GetBufferPointer nullptr";
return RET_ERROR;
}
auto int8_model = lite::Model::Import(int8_content, size);
Context int8_ctx;
int8_ctx.thread_num_ = calibrator_->GetThreadNum();
int8_ctx.device_list_[0].device_info_.cpu_device_info_.cpu_bind_mode_ = HIGHER_CPU;
int8_session_ = dynamic_cast<mindspore::lite::LiteSession *>(session::LiteSession::CreateSession(&int8_ctx));
int8_session_ = CreateSessionByFuncGraph(func_graph, flags, calibrator_->GetThreadNum());
if (int8_session_ == nullptr) {
MS_LOG(ERROR) << "create session failed!";
return RET_ERROR;
}
ret = int8_session_->CompileGraph(int8_model);
if (ret != lite::RET_OK) {
MS_LOG(ERROR) << "compile graph error";
return RET_ERROR;
}
MS_LOG(INFO) << "do bias correction";
status = BiasCorrection(func_graph);

28
mindspore/lite/tools/converter/quantizer/post_training_quantizer.h
View File

@ -28,6 +28,8 @@
#include "tools/converter/quantizer/quantizer.h"
#include "tools/converter/converter.h"
#include "include/ms_tensor.h"
#include "tools/converter/quantizer/quantize_util.h"
#include "tools/converter/quantizer/weight_quantizer.h"
namespace mindspore::lite::quant {
class Calibrator;
@ -38,19 +40,8 @@ struct MaxMin {
float max;
};
const char kMethodMaxMin[] = "MAX_MIN";
const char kMethodKL[] = "KL";
const char kMethodOutlier[] = "RemovalOutlier";
constexpr int kDefaultBinNumber = 2048;
struct ConfigParam {
std::vector<std::string> image_paths;
uint32_t batch_count{100};
std::string method_x{kMethodKL};
uint32_t thread_num{1};
bool bias_correction{false};
};
class PostTrainingQuantizer : public Quantizer {
public:
PostTrainingQuantizer(FuncGraphPtr graph, std::string path, int bit_num, TypeId target_type = kNumberTypeInt8,
@ -64,14 +55,16 @@ class PostTrainingQuantizer : public Quantizer {
int quant_min{INT8_MIN};
private:
std::map<std::string, int> opname_bit_;
bool per_channel_{true};
TypeId target_type_{kNumberTypeInt8};
std::unique_ptr<Calibrator> calibrator_;
mindspore::lite::LiteSession *fp32_session_;
mindspore::lite::LiteSession *int8_session_;
session::LiteSession *fp32_session_{nullptr};
session::LiteSession *int8_session_{nullptr};
std::map<std::string, std::vector<float>> fp32_op_input_map; // concurency
std::map<std::string, std::vector<float>> fp32_op_output_ch_mean_map; // concurency
@ -112,7 +105,8 @@ class PostTrainingQuantizer : public Quantizer {
STATUS DoQuantOutput(double scale, int32_t zeropoint, struct MaxMin *max_min,
const std::shared_ptr<PrimitiveC> &) const;
STATUS DoWeightQuant(const AnfNodePtr &weight, std::shared_ptr<PrimitiveC> primitive_c, bool perchannel) const;
STATUS DoWeightQuant(const std::string &op_name, const AnfNodePtr &weight, std::shared_ptr<PrimitiveC> primitive_c,
bool perchannel) const;
STATUS DoBiasQuant(const AnfNodePtr &bias, const std::shared_ptr<PrimitiveC> &primitive_c);
STATUS Int8Inference();
@ -213,13 +207,13 @@ class Calibrator {
std::unordered_map<std::string, std::vector<std::unique_ptr<DivergInfo>>> *GetOutputDivergInfo();
PostQuantConfig config_param_;
private:
std::vector<std::vector<std::string>> images_; // multi_input, echo input has multi input data
std::string config_path_;
ConfigParam config_param_;
std::unordered_map<std::string, std::vector<std::unique_ptr<DivergInfo>>> inputs_diverg_info_;
std::unordered_map<std::string, std::vector<std::unique_ptr<DivergInfo>>> outputs_diverg_info_;
@ -227,8 +221,6 @@ class Calibrator {
size_t bit_num_;
int quant_max_;
int quant_min_;
void AddImage(const std::string &file, size_t index);
};
} // namespace mindspore::lite::quant
#endif // MINDSPORE_LITE_TOOLS_CONVERTER_QUANTIZER_POSTRAINING_QUANTIZER_H

311
mindspore/lite/tools/converter/quantizer/quantize_util.cc
View File

@ -17,6 +17,8 @@
#include "mindspore/lite/tools/converter/quantizer/quantize_util.h"
#include <cmath>
#include <string>
#include <map>
#include <fstream>
#include <algorithm>
#include <memory>
#include <vector>
@ -26,6 +28,8 @@
#include "src/common/utils.h"
#include "abstract/abstract_value.h"
#include "securec/include/securec.h"
#include "tools/anf_exporter/anf_exporter.h"
#include "mindspore/lite/include/version.h"
using std::string;
using std::vector;
@ -83,10 +87,10 @@ bool QuantStrategy::CanConvOpQuantized(const CNodePtr &node) const {
bool QuantStrategy::CanOpPostQuantized(AnfNodePtr &node) const {
MS_ASSERT(node != nullptr);
if (!node->isa<CNode>()) {
if (!node->isa<mindspore::CNode>()) {
return false;
}
auto cnode = std::dynamic_pointer_cast<CNode>(node);
auto cnode = std::dynamic_pointer_cast<mindspore::CNode>(node);
auto type = NodePrimitiveType(cnode);
static const std::vector<schema::PrimitiveType> int8OpList = {
schema::PrimitiveType_Conv2D,
@ -475,4 +479,307 @@ schema::PrimitiveType NodePrimitiveType(const CNodePtr &cnode) {
}
return (schema::PrimitiveType)primitive_c->Type();
}
STATUS ParseConfigFile(std::string config_file, PostQuantConfig *post_quant_config) {
if (post_quant_config == nullptr) {
MS_LOG(ERROR) << "post_quant_config is null.";
return RET_PARAM_INVALID;
}
if (config_file.empty() || config_file.length() > PATH_MAX) {
MS_LOG(ERROR) << "invalid config path!";
return RET_PARAM_INVALID;
}
// check whether config file path is valid
auto resolved_path = std::make_unique<char[]>(PATH_MAX);
if (resolved_path == nullptr) {
MS_LOG(ERROR) << "New an object failed.";
return RET_ERROR;
}
#ifdef _WIN32
if (_fullpath(resolved_path.get(), config_file.c_str(), 1024) != nullptr) {
config_file = string(resolved_path.get());
}
#else
if (realpath(config_file.c_str(), resolved_path.get()) != nullptr) {
config_file = string(resolved_path.get());
}
#endif
std::ifstream fs(config_file.c_str(), std::ifstream::in);
if (!fs.is_open()) {
MS_LOG(ERROR) << "config file open failed: " << config_file;
return RET_PARAM_INVALID;
}
std::string line;
while (std::getline(fs, line)) {
Trim(&line);
if (line.empty()) {
continue;
}
auto index = line.find('=');
if (index == std::string::npos) {
MS_LOG(ERROR) << "the config file is invalid, can not find '=', please check";
return RET_PARAM_INVALID;
}
auto key = line.substr(0, index);
auto value = line.substr(index + 1);
Trim(&key);
Trim(&value);
if (key == "image_path") {
auto &raw_image_paths = value;
auto ind = raw_image_paths.find(',');
while (ind != std::string::npos) {
auto image_path = raw_image_paths.substr(0, ind);
Trim(&image_path);
post_quant_config->image_paths.push_back(image_path);
raw_image_paths = raw_image_paths.substr(ind + 1);
Trim(&raw_image_paths);
ind = raw_image_paths.find(',');
}
post_quant_config->image_paths.push_back(raw_image_paths);
} else if (key == "batch_count") {
post_quant_config->batch_count = std::stoul(value);
} else if (key == "thread_num") {
post_quant_config->thread_num = std::stoul(value);
} else if (key == "method_x") {
if (value != kMethodKL && value != kMethodMaxMin && value != kMethodOutlier) {
MS_LOG(WARNING) << "unsupported method_x: " << value << ". Use default value.";
} else {
post_quant_config->method_x = value;
}
} else if (key == "bias_correction") {
std::for_each(value.begin(), value.end(), ::tolower);
if (value == "true") {
post_quant_config->bias_correction = true;
}
} else if (key == "mixed") {
std::for_each(value.begin(), value.end(), ::tolower);
if (value == "true") {
post_quant_config->mixed = true;
}
} else if (key == "mean_error_threshold") {
post_quant_config->mean_error_threshold = std::stof(value);
} else {
MS_LOG(WARNING) << "unsupported parameter: " << key;
}
}
for (const auto &path : post_quant_config->image_paths) {
MS_LOG(DEBUG) << "calibration data_path: " << path;
}
MS_LOG(DEBUG) << "batch_count: " << post_quant_config->batch_count << "\n"
<< "method_x: " << post_quant_config->method_x << "\n"
<< "thread_num: " << post_quant_config->thread_num << "\n"
<< "bias_correction: " << post_quant_config->bias_correction << "\n"
<< "mixed: " << post_quant_config->mixed << "\n"
<< "mean_error_threshold: " << post_quant_config->mean_error_threshold;
post_quant_config->inited = true;
fs.close();
return RET_OK;
}
session::LiteSession *CreateSessionByFuncGraph(const FuncGraphPtr &func_graph, const converter::Flags &flags,
int thread_num) {
auto meta_graph = Export(func_graph, true, true);
if (meta_graph == nullptr) {
MS_LOG(ERROR) << "Export to meta_graph failed";
return nullptr;
}
// transform
GraphDefTransform fb_transform;
fb_transform.SetGraphDef(meta_graph);
auto status = fb_transform.Transform(flags);
if (status != RET_OK) {
MS_LOG(ERROR) << "FBTransform model failed";
return nullptr;
}
meta_graph->version = Version();
flatbuffers::FlatBufferBuilder builder(1024);
auto offset = schema::MetaGraph::Pack(builder, meta_graph);
builder.Finish(offset);
schema::FinishMetaGraphBuffer(builder, offset);
auto size = builder.GetSize();
auto *content = reinterpret_cast<const char *>(builder.GetBufferPointer());
if (content == nullptr) {
MS_LOG(ERROR) << "GetBufferPointer return null";
return nullptr;
}
auto model = lite::Model::Import(content, size);
if (model == nullptr) {
MS_LOG(ERROR) << "Import model failed";
return nullptr;
}
Context ctx;
ctx.thread_num_ = thread_num;
auto session = session::LiteSession::CreateSession(&ctx);
if (session == nullptr) {
MS_LOG(ERROR) << "create session failed.";
return nullptr;
}
status = session->CompileGraph(model);
if (status != RET_OK) {
MS_LOG(ERROR) << "CompileGraph error";
return nullptr;
}
model->Free();
return session;
}
STATUS CollectCalibInputs(const std::vector<std::string> &input_dirs, size_t count_limited,
std::vector<std::vector<std::string>> *inputs) {
if (inputs == nullptr) {
MS_LOG(ERROR) << "inputs is null";
return RET_ERROR;
}
auto AddImage = [&inputs](const std::string &file, size_t index) {
if (index >= inputs->size()) {
MS_LOG(ERROR) << "images_ size: " << inputs->size() << " but input index: " << index;
return;
}
struct stat buf {};
if (stat(file.c_str(), &buf) == 0) {
inputs->at(index).push_back(file);
} else {
MS_LOG(WARNING) << "invalid image file path: " << file;
}
};
inputs->resize(input_dirs.size());
auto input_i = 0;
bool multi_input = input_dirs.size() > 1;
for (const auto &image_path : input_dirs) {
DIR *root = opendir(image_path.c_str());
if (root == nullptr) {
MS_LOG(ERROR) << "invalid image path: " << image_path;
return RET_PARAM_INVALID;
}
struct dirent *image_dir = readdir(root);
size_t count = 0;
while (image_dir != nullptr) {
string file_name(image_dir->d_name);
if (file_name != "." && file_name != "..") {
const std::string file_path = image_path + "/" + file_name;
if (multi_input || count == 0) {
AddImage(file_path, input_i);
count++;
} else if (count < count_limited) {
AddImage(file_path, input_i);
count++;
} else {
break;
}
}
image_dir = readdir(root);
}
std::sort(inputs->at(input_i).begin(), inputs->at(input_i).end());
if (count_limited != 0 && count_limited < inputs->at(input_i).size()) {
inputs->at(input_i).resize(count_limited);
}
closedir(root);
input_i++;
}
return RET_OK;
}
STATUS CopyInputDataToTensor(size_t input_index, size_t image_index,
const std::vector<std::vector<std::string>> &images, mindspore::tensor::MSTensor *tensor) {
MS_ASSERT(tensor != nullptr);
if (input_index >= images.size()) {
MS_LOG(ERROR) << "images_ size: " << images.size() << " but input_index: " << input_index;
return RET_ERROR;
}
if (image_index >= images[input_index].size()) {
MS_LOG(ERROR) << "images_[input_index] size: " << images[input_index].size() << " but image_index: " << image_index;
return RET_ERROR;
}
string path = images[input_index][image_index];
MS_LOG(INFO) << "read image: " << path;
size_t size;
char *bin_buf = ReadFile(path.c_str(), &size);
if (bin_buf == nullptr) {
MS_LOG(ERROR) << "ReadFile return nullptr";
return RET_NULL_PTR;
}
auto data = tensor->MutableData();
if (data == nullptr) {
MS_LOG(ERROR) << "Get tensor MutableData return nullptr";
return RET_NULL_PTR;
}
if (size != tensor->Size()) {
MS_LOG(ERROR) << "the input data is not consistent with model input, file_size: " << size
<< " input tensor size: " << tensor->Size();
return RET_ERROR;
}
auto ret = memcpy_s(data, tensor->Size(), bin_buf, size);
if (ret != EOK) {
MS_LOG(ERROR) << "memcpy_s error: " << ret;
delete[] bin_buf;
return RET_ERROR;
}
delete[] bin_buf;
return RET_OK;
}
FuncGraphPtr CopyFuncGraph(const FuncGraphPtr &func_graph) {
Cloner cloner({func_graph}, true, true, true, std::make_shared<TraceCopy>(), nullptr);
auto new_func_graph = cloner[func_graph];
std::map<std::string, CNodePtr> old_cnode_map;
for (const auto &cnode : func_graph->GetOrderedCnodes()) {
old_cnode_map[cnode->fullname_with_scope()] = cnode;
}
for (auto &cnode : new_func_graph->GetOrderedCnodes()) {
auto cnode_name = cnode->fullname_with_scope();
auto old_cnode_iter = old_cnode_map.find(cnode_name);
if (old_cnode_iter == old_cnode_map.end()) {
MS_LOG(ERROR) << "can not find node: " << cnode_name;
return nullptr;
}
auto old_cnode = old_cnode_iter->second;
auto inputs = cnode->inputs();
for (size_t i = 0; i < inputs.size(); i++) {
auto input_node = inputs[i];
if (input_node->isa<Parameter>()) {
auto param_node = input_node->cast<ParameterPtr>();
if (param_node->has_default()) {
ParamValueLitePtr old_param_value = std::static_pointer_cast<ParamValueLite>(param_node->default_param());
auto new_param_value = std::make_shared<ParamValueLite>();
auto copyed_data = malloc(old_param_value->tensor_size());
if (copyed_data == nullptr) {
MS_LOG(ERROR) << "malloc data error, size: " << old_param_value->tensor_size();
return nullptr;
}
memcpy(copyed_data, old_param_value->tensor_addr(), old_param_value->tensor_size());
new_param_value->set_tensor_size(old_param_value->tensor_size());
new_param_value->set_tensor_addr(copyed_data);
new_param_value->set_tensor_shape(old_param_value->tensor_shape());
new_param_value->set_format(old_param_value->format());
new_param_value->set_tensor_type(old_param_value->tensor_type());
param_node->set_default_param(new_param_value);
}
auto old_abstract_base = param_node->abstract();
if (!utils::isa<abstract::AbstractTensorPtr>(old_abstract_base)) {
MS_LOG(ERROR) << "Abstract of parameter should be abstract tensor, " << param_node->name();
return nullptr;
}
auto old_abstract = utils::cast<abstract::AbstractTensorPtr>(old_abstract_base);
auto new_abstract = std::make_shared<abstract::AbstractTensor>(old_abstract->element()->GetTypeTrack(),
old_abstract->GetShapeTrack());
param_node->set_abstract(new_abstract);
}
} // end inputs loop
} // end cnodes loop
return new_func_graph;
}
} // namespace mindspore::lite::quant

35
mindspore/lite/tools/converter/quantizer/quantize_util.h
View File

@ -17,6 +17,8 @@
#ifndef MINDSPORE_LITE_TOOLS_CONVERTER_QUANTIZER_QUANTIZER_UTIL_H
#define MINDSPORE_LITE_TOOLS_CONVERTER_QUANTIZER_QUANTIZER_UTIL_H
#include <dirent.h>
#include <sys/stat.h>
#include <memory>
#include <string>
#include <cmath>
@ -35,11 +37,29 @@
#include "ir/primitive.h"
#include "abstract/dshape.h"
#include "tools/converter/quantizer/bitpacking.h"
#include "src/lite_session.h"
#include "tools/converter/graphdef_transform.h"
#include "src/common/file_utils.h"
namespace mindspore::lite::quant {
static constexpr size_t UINT8_QUANTIZATION = 8;
static constexpr size_t WEIGHT_INDEX = 1;
const char kMethodMaxMin[] = "MAX_MIN";
const char kMethodKL[] = "KL";
const char kMethodOutlier[] = "RemovalOutlier";
struct PostQuantConfig {
std::vector<std::string> image_paths;
uint32_t batch_count{100};
std::string method_x{kMethodKL};
uint32_t thread_num{1};
bool bias_correction{false};
bool mixed{false};
float mean_error_threshold{0.04};
bool inited{false};
};
/**
* 1. when op's weight size > mWeightSize just skip
* 2. only do conv/deconv/convdepthwise/deconvdepthwise/mul/matmul/batchmatmul quantization
@ -320,6 +340,21 @@ STATUS QuantFilter(const ParamValueLitePtr &weight, const std::shared_ptr<Primit
return RET_OK;
}
// utils
schema::PrimitiveType NodePrimitiveType(const CNodePtr &cnode);
STATUS ParseConfigFile(std::string config_file, PostQuantConfig *post_quant_config);
session::LiteSession *CreateSessionByFuncGraph(const FuncGraphPtr &func_graph, const converter::Flags &flags,
int thread_num);
STATUS CollectCalibInputs(const std::vector<std::string> &input_dirs, size_t count_limited,
std::vector<std::vector<std::string>> *inputs);
STATUS CopyInputDataToTensor(size_t input_index, size_t image_index,
const std::vector<std::vector<std::string>> &images, mindspore::tensor::MSTensor *tensor);
FuncGraphPtr CopyFuncGraph(const FuncGraphPtr &);
} // namespace mindspore::lite::quant
#endif

336
mindspore/lite/tools/converter/quantizer/weight_quantizer.cc
View File

@ -18,6 +18,7 @@
#include <list>
#include <string>
#include <vector>
#include <unordered_map>
#include "src/common/common.h"
#include "ir/dtype/type_id.h"
@ -36,6 +37,7 @@ bool WeightQuantizer::IsPosNum(const std::string &str) {
}
return true;
}
STATUS WeightQuantizer::WeightQuantInputCheck(const converter::Flags *config) {
MS_ASSERT(config != nullptr);
if (!WeightQuantizer::IsPosNum(config->quantWeightChannel)) {
@ -57,28 +59,57 @@ STATUS WeightQuantizer::WeightQuantInputCheck(const converter::Flags *config) {
}
return RET_OK;
}
WeightQuantizer::WeightQuantizer(FuncGraphPtr graph, const string &weightSize,
WeightQuantizer::WeightQuantizer(FuncGraphPtr graph, const PostQuantConfig &config) : Quantizer(graph) {
quant_strategy_ = std::make_unique<QuantStrategy>(0, 0);
config_param_ = config;
}
WeightQuantizer::WeightQuantizer(FuncGraphPtr graph, const std::string &config_file, const string &weightSize,
const std::string &convWeightChannelThreshold, const std::string &bitNum)
: Quantizer(graph) {
this->config_file_ = config_file;
auto quantSize = static_cast<size_t>(std::stoull(weightSize));
this->bitNum = static_cast<size_t>(std::stoull(bitNum));
this->bit_num_ = static_cast<size_t>(std::stoull(bitNum));
auto convQuantWeightChannelThreshold = static_cast<size_t>(std::stoull(convWeightChannelThreshold));
mStrategy = std::make_unique<QuantStrategy>(quantSize, convQuantWeightChannelThreshold);
quant_max = (1 << (unsigned int)(this->bitNum - 1)) - 1;
quant_min = -(1 << (unsigned int)(this->bitNum - 1));
quant_strategy_ = std::make_unique<QuantStrategy>(quantSize, convQuantWeightChannelThreshold);
quant_max = (1 << (unsigned int)(this->bit_num_ - 1)) - 1;
quant_min = -(1 << (unsigned int)(this->bit_num_ - 1));
// parse type_id
if (this->bitNum > 0 && this->bitNum <= 8) {
if (this->bit_num_ > 0 && this->bit_num_ <= 8) {
type_id = kNumberTypeInt8;
} else if (this->bitNum <= 16) {
} else if (this->bit_num_ <= 16) {
type_id = kNumberTypeInt16;
} else {
MS_LOG(ERROR) << "invalid input bits";
}
}
WeightQuantizer::~WeightQuantizer() { delete fp32_session_; }
STATUS WeightQuantizer::SetAbstract(ParamValueLitePtr param_value, ParameterPtr param_node,
std::shared_ptr<PrimitiveC> primitive_c) {
// set dtype
param_value->set_tensor_type(type_id);
auto abstract_base = param_node->abstract();
if (abstract_base == nullptr) {
MS_LOG(ERROR) << "Abstract of parameter is nullptr, " << param_node->name();
return RET_ERROR;
}
if (!utils::isa<abstract::AbstractTensorPtr>(abstract_base)) {
MS_LOG(ERROR) << "Abstract of parameter should be anstract tensor, " << param_node->name();
return RET_ERROR;
}
auto abstract_tensor = utils::cast<abstract::AbstractTensorPtr>(abstract_base);
abstract_tensor->element()->set_type(TypeIdToType(type_id));
primitive_c->set_quant_type(schema::QuantType_WeightQuant);
return RET_OK;
}
STATUS WeightQuantizer::DoConvQuantize(const std::list<CNodePtr> &nodes) {
for (auto &cnode : nodes) {
if (!mStrategy->CanConvOpQuantized(cnode)) {
if (!quant_strategy_->CanConvOpQuantized(cnode)) {
continue;
}
@ -108,36 +139,28 @@ STATUS WeightQuantizer::DoConvQuantize(const std::list<CNodePtr> &nodes) {
}
auto status = RET_ERROR;
if (type_id == kNumberTypeInt8) {
status = QuantFilter<int8_t>(param_value, primitive_c, QuantType_WeightQuant, quant_max, quant_min, bitNum, true);
status =
QuantFilter<int8_t>(param_value, primitive_c, QuantType_WeightQuant, quant_max, quant_min, bit_num_, true);
} else if (type_id == kNumberTypeInt16) {
status =
QuantFilter<int16_t>(param_value, primitive_c, QuantType_WeightQuant, quant_max, quant_min, bitNum, true);
QuantFilter<int16_t>(param_value, primitive_c, QuantType_WeightQuant, quant_max, quant_min, bit_num_, true);
}
if (status != RET_OK) {
MS_LOG(ERROR) << "QuantFilter failed : " << status;
return status;
}
// set dtype
param_value->set_tensor_type(type_id);
auto abstractBase = param_node->abstract();
if (abstractBase == nullptr) {
MS_LOG(ERROR) << "Abstract of parameter is nullptr, " << param_node->name();
status = SetAbstract(param_value, param_node, primitive_c);
if (status != RET_OK) {
MS_LOG(ERROR) << "SetAbstract failed : " << status;
return RET_ERROR;
}
if (!utils::isa<abstract::AbstractTensorPtr>(abstractBase)) {
MS_LOG(ERROR) << "Abstract of parameter should be anstract tensor, " << param_node->name();
return RET_ERROR;
}
auto abstractTensor = utils::cast<abstract::AbstractTensorPtr>(abstractBase);
abstractTensor->element()->set_type(TypeIdToType(type_id));
primitive_c->set_quant_type(schema::QuantType_WeightQuant);
}
return RET_OK;
}
STATUS WeightQuantizer::DoMulQuantize(const std::list<CNodePtr> &nodes) {
for (auto &node : nodes) {
if (!mStrategy->CanMulOpQuantized(node)) {
if (!quant_strategy_->CanMulOpQuantized(node)) {
continue;
}
auto already_quant = false;
@ -186,38 +209,271 @@ STATUS WeightQuantizer::DoMulQuantize(const std::list<CNodePtr> &nodes) {
auto status = RET_ERROR;
if (type_id == kNumberTypeInt8) {
status = QuantFilter<int8_t>(param_value, primitive_c, QuantType_WeightQuant, quant_max, quant_min, bitNum, true);
status =
QuantFilter<int8_t>(param_value, primitive_c, QuantType_WeightQuant, quant_max, quant_min, bit_num_, true);
} else if (type_id == kNumberTypeInt16) {
status =
QuantFilter<int16_t>(param_value, primitive_c, QuantType_WeightQuant, quant_max, quant_min, bitNum, true);
QuantFilter<int16_t>(param_value, primitive_c, QuantType_WeightQuant, quant_max, quant_min, bit_num_, true);
}
if (status != RET_OK) {
MS_LOG(ERROR) << "QuantFilter failed : " << status;
return status;
}
param_value->set_tensor_type(type_id);
// set dtype
auto abstractBase = param_node->abstract();
if (abstractBase == nullptr) {
MS_LOG(ERROR) << "Abstract of parameter is nullptr, " << param_node->name();
status = SetAbstract(param_value, param_node, primitive_c);
if (status != RET_OK) {
MS_LOG(ERROR) << "SetAbstract failed : " << status;
return RET_ERROR;
}
if (!utils::isa<abstract::AbstractTensorPtr>(abstractBase)) {
MS_LOG(ERROR) << "Abstract of parameter should be anstract tensor, " << param_node->name();
return RET_ERROR;
}
auto abstractTensor = utils::cast<abstract::AbstractTensorPtr>(abstractBase);
abstractTensor->element()->set_type(TypeIdToType(type_id));
primitive_c->set_quant_type(schema::QuantType_WeightQuant);
}
return RET_OK;
}
STATUS WeightQuantizer::DoQuantize(FuncGraphPtr funcGraph) {
MS_ASSERT(funcGraph != nullptr);
constexpr float relative_tolerance = 1e-5;
constexpr float abs_tolerance = 1e-4;
template <typename T>
float CompareOutputData(const std::unordered_map<std::string, mindspore::tensor::MSTensor *> &expected_tensor,
const std::unordered_map<std::string, mindspore::tensor::MSTensor *> &compare_tensor) {
auto valid_data = [](T data) -> bool { return (!std::isnan(data) && !std::isinf(data)); };
float total_mean_error = 0.0f;
int tensor_cnt = expected_tensor.size();
if (tensor_cnt <= 0) {
MS_LOG(ERROR) << "unexpected tensor_cnt: " << tensor_cnt;
return RET_ERROR;
}
for (const auto &exp_tensor_pair : expected_tensor) {
float mean_error = 0.0f;
int error_cnt = 0;
auto exp_tensor_name = exp_tensor_pair.first;
auto exp_tensor = exp_tensor_pair.second;
auto cmp_tensor_find_iter = compare_tensor.find(exp_tensor_name);
if (cmp_tensor_find_iter == compare_tensor.end()) {
MS_LOG(ERROR) << "can not find: " << exp_tensor_name;
return RET_ERROR;
}
auto cmp_tensor = cmp_tensor_find_iter->second;
auto exp_tensor_shape = exp_tensor->shape();
auto cmp_tensor_shape = cmp_tensor->shape();
if (exp_tensor_shape != cmp_tensor_shape) {
MS_LOG(ERROR) << "exp tensor shape not equal to cmp. exp_tensor_elem_cnt: " << exp_tensor->ElementsNum()
<< " cmp_tensor_elem_cnt: " << cmp_tensor->ElementsNum();
return RET_ERROR;
}
auto exp_data = static_cast<T *>(exp_tensor->MutableData());
auto cmp_data = static_cast<T *>(cmp_tensor->MutableData());
auto elem_cnt = exp_tensor->ElementsNum();
for (int i = 0; i < elem_cnt; i++) {
if (!valid_data(exp_data[i]) || !valid_data(cmp_data[i])) {
MS_LOG(ERROR) << "data is not valid. exp: " << exp_data[i] << " cmp: " << cmp_data[i] << " index: " << i;
return RET_ERROR;
}
auto tolerance = abs_tolerance + relative_tolerance * fabs(exp_data[i]);
auto abs_error = std::fabs(exp_data[i] - cmp_data[i]);
if (abs_error > tolerance) {
if (fabs(exp_data[i] == 0)) {
if (abs_error > 1e-5) {
mean_error += abs_error;
error_cnt++;
} else {
// it is ok, very close to 0
continue;
}
} else {
mean_error += abs_error / (fabs(exp_data[i]) + FLT_MIN);
error_cnt++;
}
} else {
// it is ok, no error
continue;
}
} // end one tensor data loop
total_mean_error += mean_error / elem_cnt;
} // end tensor loop
return total_mean_error / tensor_cnt;
}
STATUS WeightQuantizer::DoMiexedQuant(FuncGraphPtr func_graph) {
// 0.1 Create Fp32 Session
flags.quantType = schema::QuantType_QUANT_NONE;
fp32_session_ = CreateSessionByFuncGraph(func_graph, flags, config_param_.thread_num);
if (fp32_session_ == nullptr) {
MS_LOG(ERROR) << "CreateSessoin fail";
return RET_ERROR;
}
auto fp32_inputs = fp32_session_->GetInputs();
// 0.2 Parse input calib files
auto status = CollectCalibInputs(config_param_.image_paths, config_param_.batch_count, &images_);
if (status != RET_OK) {
MS_LOG(ERROR) << "CollectCalibInputs fail";
return RET_ERROR;
}
auto cnodes = func_graph->GetOrderedCnodes();
for (auto iter = cnodes.end(); iter != cnodes.begin();) {
auto cnode = *(--iter);
auto primitive_c = GetValueNode<std::shared_ptr<PrimitiveC>>(cnode->input(0));
if (primitive_c == nullptr) {
MS_LOG(ERROR) << "primitive_c is null.";
return RET_ERROR;
}
auto op_name = cnode->fullname_with_scope();
MS_LOG(DEBUG) << "process node: " << op_name
<< " type: " << schema::EnumNamePrimitiveType((schema::PrimitiveType)primitive_c->Type());
if (quant_strategy_->CanConvOpQuantized(cnode) || quant_strategy_->CanMulOpQuantized(cnode)) {
auto input_node = cnode->input(2);
if (!input_node->isa<Parameter>()) {
MS_LOG(WARNING) << op_name << " the second input is not parameter";
continue;
}
auto param_node = input_node->cast<ParameterPtr>();
if (!param_node->has_default()) {
MS_LOG(WARNING) << op_name << " the second input can not convert to parameter";
continue;
}
auto param_value = std::static_pointer_cast<ParamValueLite>(param_node->default_param());
if (param_value == nullptr) {
MS_LOG(WARNING) << op_name << " the second input can not convert to parameter";
continue;
}
if (param_value->tensor_type() != TypeId::kNumberTypeFloat32) {
MS_LOG(WARNING) << op_name << " the second input type is not float";
continue;
}
// copy origin data in case to recover
auto *raw_data = static_cast<float *>(param_value->tensor_addr());
auto elem_count = param_value->tensor_shape_size();
auto origin_data = malloc(sizeof(float) * elem_count);
auto ret = memcpy_s(origin_data, sizeof(float) * elem_count, raw_data, param_value->tensor_size());
if (ret != EOK) {
MS_LOG(ERROR) << "memcpy fail: "
<< " dst size: " << sizeof(float) * elem_count << " src size: " << param_value->tensor_size();
return RET_ERROR;
}
// 1. try quant
for (int bit_num_t = 2; bit_num_t <= 8; bit_num_t++) {
type_id = TypeId::kNumberTypeInt8;
int quant_max_t = (1 << (unsigned int)(bit_num_t - 1)) - 1;
int quant_min_t = -(1 << (unsigned int)(bit_num_t - 1));
if (type_id == TypeId::kNumberTypeInt8) {
status = QuantFilter<int8_t>(param_value, primitive_c, QuantType::QuantType_WeightQuant, quant_max_t,
quant_min_t, bit_num_t, true);
} else if (type_id == TypeId::kNumberTypeInt16) {
status = QuantFilter<int16_t>(param_value, primitive_c, QuantType::QuantType_WeightQuant, quant_max_t,
quant_min_t, bit_num_t, true);
} else {
MS_LOG(ERROR) << "unexpected type_id: " << type_id;
return RET_ERROR;
}
if (status != RET_OK) {
MS_LOG(ERROR) << "quant filter fail.";
return RET_ERROR;
}
status = SetAbstract(param_value, param_node, primitive_c);
if (status != RET_OK) {
MS_LOG(ERROR) << "SetAbstract failed : " << status;
return RET_ERROR;
}
// 2. evaluate the quant
// 2.1 create quant session, get input, output tensor
flags.quantType = schema::QuantType_WeightQuant;
auto quant_session =
std::unique_ptr<session::LiteSession>(CreateSessionByFuncGraph(func_graph, flags, config_param_.thread_num));
if (quant_session == nullptr) {
MS_LOG(ERROR) << "create session error: " << status;
return RET_ERROR;
}
auto quant_inputs = quant_session->GetInputs();
auto mean_error = 0.0f;
if (fp32_inputs.size() != images_.size()) {
MS_LOG(ERROR) << "model's input tensor cnt: " << fp32_inputs.size() << " != " << images_.size();
return RET_ERROR;
}
auto image_cnt = images_.at(0).size();
for (size_t i = 0; i < image_cnt; i++) {
// set multi-input data
for (size_t input_index = 0; input_index < fp32_inputs.size(); input_index++) {
status = CopyInputDataToTensor(input_index, i, images_, fp32_inputs[input_index]);
if (status != RET_OK) {
MS_LOG(ERROR) << "generate input data from images failed!";
return RET_ERROR;
}
status = CopyInputDataToTensor(input_index, i, images_, quant_inputs[input_index]);
if (status != RET_OK) {
MS_LOG(ERROR) << "generate input data from images failed!";
return RET_ERROR;
}
}
std::future<STATUS> fp32_inference = std::async(
std::launch::async, [](session::LiteSession *fp32_session) -> STATUS { return fp32_session->RunGraph(); },
fp32_session_);
status = quant_session->RunGraph();
if (status != RET_OK) {
MS_LOG(ERROR) << "quant session run error";
return RET_ERROR;
}
status = fp32_inference.get();
if (status != RET_OK) {
MS_LOG(ERROR) << "fp32 session run error";
return RET_ERROR;
}
// 3. compare betwen quant and fp32
auto fp32_outputs = fp32_session_->GetOutputs();
auto quant_outputs = quant_session->GetOutputs();
mean_error += CompareOutputData<float>(fp32_outputs, quant_outputs);
} // end_for: calib data loop
mean_error = mean_error / image_cnt;
if (mean_error <= config_param_.mean_error_threshold) {
MS_LOG(DEBUG) << "op: " << op_name << " got mixed bit: " << bit_num_t << " mean_error: " << mean_error;
opname_bit_[op_name] = bit_num_t;
break;
} else if (bit_num_t != 8) {
// recover
param_value->set_tensor_size(sizeof(float) * elem_count);
ret = memcpy_s(raw_data, param_value->tensor_size(), origin_data, sizeof(float) * elem_count);
if (ret != EOK) {
MS_LOG(ERROR) << "memcpy fail: "
<< " src size: " << sizeof(float) * elem_count << " dst size: " << param_value->tensor_size();
return RET_ERROR;
}
} else {
MS_LOG(DEBUG) << "op: " << op_name << " set bit: " << bit_num_t << " mean_error: " << mean_error;
opname_bit_[op_name] = bit_num_t;
}
} // end bit loop
free(origin_data);
} // if: conv and matmul
} // end loop: all cnode
return RET_OK;
}
STATUS WeightQuantizer::DoQuantize(FuncGraphPtr func_graph) {
MS_ASSERT(func_graph != nullptr);
STATUS ret;
auto cnodes = funcGraph->GetOrderedCnodes();
auto cnodes = func_graph->GetOrderedCnodes();
if (!config_file_.empty()) {
ret = ParseConfigFile(config_file_, &config_param_);
if (ret != RET_OK) {
MS_LOG(ERROR) << "ReadConfig error.";
return RET_ERROR;
}
}
if (config_param_.mixed) {
MS_LOG(INFO) << "Do mixed bit quantization";
return DoMiexedQuant(func_graph);
}
ret = DoConvQuantize(cnodes);
if (ret != RET_OK) {
MS_LOG(ERROR) << "DoConvQuantize failed :" << ret;

27
mindspore/lite/tools/converter/quantizer/weight_quantizer.h
View File

@ -17,9 +17,12 @@
#ifndef MINDSPORE_LITE_TOOLS_CONVERTER_QUANTIZER_WEIGHT_QUANTIZER_H
#define MINDSPORE_LITE_TOOLS_CONVERTER_QUANTIZER_WEIGHT_QUANTIZER_H
#include <future>
#include <memory>
#include <map>
#include <list>
#include <string>
#include <vector>
#include "tools/converter/quantizer/quantizer.h"
#include "tools/converter/quantizer/quantize_util.h"
#include "ir/func_graph.h"
@ -27,27 +30,37 @@
#include "include/model.h"
#include "base/base.h"
#include "abstract/dshape.h"
#include "src/lite_session.h"
namespace mindspore::lite::quant {
class WeightQuantizer : public Quantizer {
public:
WeightQuantizer(FuncGraphPtr graph, const std::string &weightSize, const std::string &covWeightChannelThreshold,
const std::string &bitNum);
WeightQuantizer(FuncGraphPtr graph, const std::string &config_file, const std::string &weightSize,
const std::string &covWeightChannelThreshold, const std::string &bitNum);
WeightQuantizer(FuncGraphPtr graph, const PostQuantConfig &config);
~WeightQuantizer();
~WeightQuantizer() = default;
STATUS DoQuantize(FuncGraphPtr funcGraph) override;
STATUS DoQuantize(FuncGraphPtr func_graph) override;
STATUS DoConvQuantize(const std::list<CNodePtr> &nodes);
STATUS DoMulQuantize(const std::list<CNodePtr> &nodes);
static STATUS WeightQuantInputCheck(const converter::Flags *config);
static bool IsPosNum(const std::string &str);
int quant_max;
int quant_min;
TypeId type_id{kTypeUnknown};
std::map<std::string, int> opname_bit_;
private:
std::unique_ptr<QuantStrategy> mStrategy;
size_t bitNum;
std::unique_ptr<QuantStrategy> quant_strategy_;
size_t bit_num_;
std::string config_file_;
PostQuantConfig config_param_;
std::vector<std::vector<std::string>> images_; // multi_input, [[mode_input_0], [model_input_1]...]
session::LiteSession *fp32_session_ = nullptr;
STATUS DoMiexedQuant(FuncGraphPtr);
STATUS SetAbstract(ParamValueLitePtr param_value, ParameterPtr param_node, std::shared_ptr<PrimitiveC> primitive_c);
};
} // namespace mindspore::lite::quant
#endif