sfqcyy
/
mindspore
forked from OSSInnovation/mindspore
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

!6089 Extend watchpoint support in debugger to all tensor types 

Merge pull request !6089 from HarshvardhanGupta/add-more-dtypes-wp
This commit is contained in:
mindspore-ci-bot 2020-09-12 13:17:05 +08:00 committed by Gitee
parent 3ec9214d0a 18d1e8795a
commit f16ad7aa27
2 changed files with 81 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
mindspore/ccsrc/debug/debug_services.cc
View File

@ -58,13 +58,14 @@ void DebugServices::RemoveWatchpoint(unsigned int id) {
watchpoint_table.erase(id); watchpoint_table.erase(id);
} }
DebugServices::tensor_stats DebugServices::SummarizeTensor(const float *start, unsigned int n, bool need_min_max, template <typename T>
DebugServices::tensor_stats DebugServices::SummarizeTensor(const T *start, unsigned int n, bool need_min_max,
bool need_mean_sd) { bool need_mean_sd) {
tensor_stats stats; tensor_stats stats;
for (unsigned int i = 0; i < n; ++i) { for (unsigned int i = 0; i < n; ++i) {
float val = start[i]; auto val = static_cast<double>(start[i]);
stats.has_nan = stats.has_nan || isnan(val); stats.has_nan = stats.has_nan || std::isnan(val);
stats.has_inf = stats.has_inf || isinf(val); stats.has_inf = stats.has_inf || std::isinf(val);
if (stats.has_inf && stats.has_nan) { if (stats.has_inf && stats.has_nan) {
// other statistics don't make sense in this case // other statistics don't make sense in this case
break; break;
@ -76,9 +77,7 @@ DebugServices::tensor_stats DebugServices::SummarizeTensor(const float *start, u
} }
if (need_mean_sd) { if (need_mean_sd) {
// for mean and sd calculation see double delta = val - stats.mean;
// https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm
float delta = val - stats.mean;
stats.mean += delta / (i + 1); stats.mean += delta / (i + 1);
stats.m2 += delta * (val - stats.mean); stats.m2 += delta * (val - stats.mean);
} }
@ -109,13 +108,7 @@ void DebugServices::CheckWatchpoints(std::vector<std::string> *name, std::vector
bool inf_nan_enabled = false; bool inf_nan_enabled = false;
for (auto w_table_item : watchpoint_table) { for (auto w_table_item : watchpoint_table) {
auto wp = std::get<1>(w_table_item); auto wp = std::get<1>(w_table_item);
if (wp.condition.type != IS_OVERFLOW && tensor_dtype == kNumberTypeBool) continue;
// if (!wp.conditions.condition_list[IS_OVERFLOW].enabled) {
if (wp.condition.type != IS_OVERFLOW) {
// only overflow condition supports all data types
if (tensor_dtype != kNumberTypeFloat && tensor_dtype != kNumberTypeFloat32) continue;
}
if (wp.IsNodeIncluded(tensor_name_no_slot)) { if (wp.IsNodeIncluded(tensor_name_no_slot)) {
min_max_enabled |= wp.min_max_enabled(); min_max_enabled |= wp.min_max_enabled();
mean_sd_enabled |= wp.mean_sd_enabled(); mean_sd_enabled |= wp.mean_sd_enabled();
@ -124,11 +117,70 @@ void DebugServices::CheckWatchpoints(std::vector<std::string> *name, std::vector
} }
} }
tensor_stats stats; tensor_stats stats;
uint num_elements = tensor_ptr->DataSize();
if (min_max_enabled || mean_sd_enabled || inf_nan_enabled) { if (min_max_enabled || mean_sd_enabled || inf_nan_enabled) {
auto *start_addr = reinterpret_cast<float *>(tensor_ptr->data_c()); switch (tensor_dtype) {
unsigned int num_elements = (tensor_ptr->data().nbytes()) / sizeof(float); case kNumberTypeUInt8: {
stats = SummarizeTensor(start_addr, num_elements, min_max_enabled, mean_sd_enabled); auto start_addr = reinterpret_cast<uint8_t *>(tensor_ptr->data_c());
stats = SummarizeTensor(start_addr, num_elements, min_max_enabled, mean_sd_enabled);
break;
}
case kNumberTypeInt8: {
auto start_addr = reinterpret_cast<int8_t *>(tensor_ptr->data_c());
stats = SummarizeTensor(start_addr, num_elements, min_max_enabled, mean_sd_enabled);
break;
}
case kNumberTypeUInt16: {
auto start_addr = reinterpret_cast<uint16_t *>(tensor_ptr->data_c());
stats = SummarizeTensor(start_addr, num_elements, min_max_enabled, mean_sd_enabled);
break;
}
case kNumberTypeInt16: {
auto start_addr = reinterpret_cast<int16_t *>(tensor_ptr->data_c());
stats = SummarizeTensor(start_addr, num_elements, min_max_enabled, mean_sd_enabled);
break;
}
case kNumberTypeUInt32: {
auto start_addr = reinterpret_cast<uint32_t *>(tensor_ptr->data_c());
stats = SummarizeTensor(start_addr, num_elements, min_max_enabled, mean_sd_enabled);
break;
}
case kNumberTypeInt32:
case kNumberTypeInt: {
auto start_addr = reinterpret_cast<int32_t *>(tensor_ptr->data_c());
stats = SummarizeTensor(start_addr, num_elements, min_max_enabled, mean_sd_enabled);
break;
}
case kNumberTypeUInt64: {
auto start_addr = reinterpret_cast<uint64_t *>(tensor_ptr->data_c());
stats = SummarizeTensor(start_addr, num_elements, min_max_enabled, mean_sd_enabled);
break;
}
case kNumberTypeInt64: {
auto start_addr = reinterpret_cast<int64_t *>(tensor_ptr->data_c());
stats = SummarizeTensor(start_addr, num_elements, min_max_enabled, mean_sd_enabled);
break;
}
case kNumberTypeFloat16: {
auto start_addr = reinterpret_cast<float16 *>(tensor_ptr->data_c());
stats = SummarizeTensor(start_addr, num_elements, min_max_enabled, mean_sd_enabled);
break;
}
case kNumberTypeFloat32:
case kNumberTypeFloat: {
auto start_addr = reinterpret_cast<float *>(tensor_ptr->data_c());
stats = SummarizeTensor(start_addr, num_elements, min_max_enabled, mean_sd_enabled);
break;
}
case kNumberTypeFloat64: {
auto start_addr = reinterpret_cast<double *>(tensor_ptr->data_c());
stats = SummarizeTensor(start_addr, num_elements, min_max_enabled, mean_sd_enabled);
break;
}
default:
MS_LOG(INFO) << "Unsupported tensor type";
break;
}
} }
for (auto &it : watchpoints_to_check_table) { for (auto &it : watchpoints_to_check_table) {

21
mindspore/ccsrc/debug/debug_services.h
View File

@ -93,26 +93,26 @@ class DebugServices {
} watchpoint_t; } watchpoint_t;
struct tensor_stats { struct tensor_stats {
float min = std::numeric_limits<float>::max(); double min = std::numeric_limits<double>::max();
float max = std::numeric_limits<float>::lowest(); double max = std::numeric_limits<double>::lowest();
bool has_inf = false; bool has_inf = false;
bool has_nan = false; bool has_nan = false;
unsigned int n = 0; unsigned int n = 0;
float mean = 0.0; double mean = 0.0;
float m2 = 0.0; double m2 = 0.0;
float statLookup(CONDITION_TYPE type) const { double statLookup(CONDITION_TYPE type) const {
if (type == MAX_GT || type == MAX_LT) return max; if (type == MAX_GT || type == MAX_LT) return max;
if (type == MIN_GT || type == MIN_LT) return min; if (type == MIN_GT || type == MIN_LT) return min;
if (type == MAX_MIN_GT || type == MAX_MIN_LT) return (max - min); if (type == MAX_MIN_GT || type == MAX_MIN_LT) return (max - min);
if (type == MEAN_GT || type == MEAN_LT) return mean; if (type == MEAN_GT || type == MEAN_LT) return mean;
if (type == SD_GT || type == SD_LT) return getStandardDeviation(); if (type == SD_GT || type == SD_LT) return getStandardDeviation();
return std::numeric_limits<float>::quiet_NaN(); return std::numeric_limits<double>::quiet_NaN();
} }
float getMean() const { return mean; } double getMean() const { return mean; }
float getVariance() const { double getVariance() const {
if (n > 1) { if (n > 1) {
return m2 / (n - 1); return m2 / (n - 1);
} else { } else {
@ -120,7 +120,7 @@ class DebugServices {
} }
} }
float getStandardDeviation() const { return sqrt(getVariance()); } double getStandardDeviation() const { return sqrt(getVariance()); }
}; };
void AddWatchpoint(unsigned int id, unsigned int watch_condition, float parameter, void AddWatchpoint(unsigned int id, unsigned int watch_condition, float parameter,
@ -152,7 +152,8 @@ class DebugServices {
TensorLoader *tensor_loader_; TensorLoader *tensor_loader_;
static tensor_stats SummarizeTensor(const float *start, unsigned int n, bool need_min_max, bool need_mean_sd); template <typename T>
static tensor_stats SummarizeTensor(const T *start, unsigned int n, bool need_min_max, bool need_mean_sd);
}; };
} // namespace mindspore } // namespace mindspore