qicosmos
/
yalantinglibs
mirror of https://github.com/alibaba/yalantinglibs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[metric] improve json serialize performance (#785) 

* [metric] improve json serialize performance

* remove

* fix
This commit is contained in:
saipubw 2024-10-09 11:08:21 +08:00 committed by GitHub
parent a52ac9129d
commit f65d56710f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 76 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
include/ylt/metric/counter.hpp
View File

@ -2,6 +2,7 @@
#include <algorithm>
#include <array>
#include <atomic>
#include <memory>
#include <thread>
#include <variant>
@ -14,17 +15,18 @@ enum class op_type_t { INC, DEC, SET };
#ifdef CINATRA_ENABLE_METRIC_JSON
struct json_counter_metric_t {
std::map<std::string, std::string> labels;
std::vector<std::string_view> labels;
std::variant<int64_t, double> value;
};
YLT_REFL(json_counter_metric_t, labels, value);
struct json_counter_t {
std::string name;
std::string help;
std::string type;
std::string_view name;
std::string_view help;
std::string_view type;
std::vector<std::string_view> labels_name;
std::vector<json_counter_metric_t> metrics;
};
YLT_REFL(json_counter_t, name, help, type, metrics);
YLT_REFL(json_counter_t, name, help, type, labels_name, metrics);
#endif
template <typename value_type>
@ -78,13 +80,23 @@ class basic_static_counter : public static_metric {
#ifdef CINATRA_ENABLE_METRIC_JSON
void serialize_to_json(std::string &str) override {
if (default_label_value_.value() == 0) {
auto value = default_label_value_.value();
if (value == 0 && !has_change_) {
return;
}
json_counter_t counter{name_, help_, std::string(metric_name())};
auto value = default_label_value_.value();
counter.metrics.push_back({static_labels_, value});
json_counter_t counter{name_, help_, metric_name()};
counter.labels_name.reserve(static_labels_.size());
for (auto &[k, _] : static_labels_) {
counter.labels_name.emplace_back(k);
}
counter.metrics.resize(1);
counter.metrics[0].labels.reserve(static_labels_.size());
for (auto &[k, _] : static_labels_) {
counter.metrics[0].labels.emplace_back(k);
}
counter.metrics[0].value = value;
iguana::to_json(counter, str);
}
#endif
@ -248,10 +260,12 @@ class basic_dynamic_counter
#ifdef CINATRA_ENABLE_METRIC_JSON
void serialize_to_json(std::string &str) override {
std::string s;
auto map = Base::copy();
json_counter_t counter{Base::name_, Base::help_,
std::string(Base::metric_name())};
json_counter_t counter{Base::name_, Base::help_, Base::metric_name()};
counter.labels_name.reserve(Base::labels_name().size());
for (auto &e : Base::labels_name()) {
counter.labels_name.emplace_back(e);
}
to_json(counter, map, str);
}
@ -262,10 +276,11 @@ class basic_dynamic_counter
auto &val = e->value;
json_counter_metric_t metric;
size_t index = 0;
metric.labels.reserve(k.size());
for (auto &label_value : k) {
metric.labels.emplace(Base::labels_name_[index++], label_value);
metric.labels.emplace_back(label_value);
}
metric.value = (int64_t)val;
metric.value = val.load(std::memory_order::relaxed);
counter.metrics.push_back(std::move(metric));
}
if (!counter.metrics.empty()) {
@ -279,7 +294,7 @@ class basic_dynamic_counter
void serialize_map(T &value_map, std::string &str) {
for (auto &e : value_map) {
auto &labels_value = e->label;
auto &val = e->value;
auto val = e->value.load(std::memory_order::relaxed);
str.append(Base::name_);
if (Base::labels_name_.empty()) {
str.append(" ");

73
include/ylt/metric/summary.hpp
View File

@ -17,19 +17,21 @@
namespace ylt::metric {
#ifdef CINATRA_ENABLE_METRIC_JSON
struct json_summary_metric_t {
std::map<std::string, std::string> labels;
std::map<double, double> quantiles;
int64_t count;
std::vector<std::string_view> labels;
std::vector<float> quantiles_value;
uint64_t count;
double sum;
};
YLT_REFL(json_summary_metric_t, labels, quantiles, count, sum);
YLT_REFL(json_summary_metric_t, labels, quantiles_value, count, sum);
struct json_summary_t {
std::string name;
std::string help;
std::string type;
std::string_view name;
std::string_view help;
std::string_view type;
const std::vector<std::string>& labels_name;
const std::vector<double>& quantiles_key;
std::vector<json_summary_metric_t> metrics;
};
YLT_REFL(json_summary_t, name, help, type, metrics);
YLT_REFL(json_summary_t, name, help, type, labels_name, quantiles_key, metrics);
#endif
class summary_t : public static_metric {
@ -114,29 +116,17 @@ class summary_t : public static_metric {
return;
}
double sum = 0;
uint64_t count = 0;
auto rates = get_rates(sum, count);
if (count == 0) {
return;
}
json_summary_t summary{name_, help_, std::string(metric_name())};
json_summary_t summary{name_, help_, metric_name(), labels_name(),
quantiles_};
json_summary_metric_t metric;
for (size_t i = 0; i < quantiles_.size(); i++) {
for (size_t i = 0; i < labels_name_.size(); i++) {
metric.labels[labels_name_[i]] = labels_value_[i];
}
metric.quantiles.emplace(quantiles_[i], rates[i]);
metric.quantiles_value = get_rates(metric.sum, metric.count);
if (metric.count == 0) {
return;
}
metric.sum = sum;
metric.count = count;
metric.labels.reserve(labels_value_.size());
for (auto& e : labels_value_) metric.labels.emplace_back(e);
summary.metrics.push_back(std::move(metric));
iguana::to_json(summary, str);
}
#endif
@ -228,25 +218,28 @@ class basic_dynamic_summary
#ifdef CINATRA_ENABLE_METRIC_JSON
virtual void serialize_to_json(std::string& str) override {
json_summary_t summary{Base::name_, Base::help_,
std::string(Base::metric_name())};
auto map = Base::copy();
for (auto& e : map) {
auto& labels_value = e->label;
auto& summary_value = e->value;
json_summary_metric_t metric;
if (map.empty()) {
return;
}
json_summary_t summary{Base::name_, Base::help_, Base::metric_name(),
Base::labels_name(), quantiles_};
summary.metrics.reserve(map.size());
for (size_t i = 0; i < map.size(); ++i) {
auto& labels_value = map[i]->label;
auto& summary_value = map[i]->value;
double sum = 0;
uint64_t count = 0;
auto rates = summary_value.stat(sum, count);
if (count == 0)
continue;
summary.metrics.emplace_back();
json_summary_metric_t& metric = summary.metrics.back();
metric.count = count;
metric.sum = sum;
for (size_t i = 0; i < quantiles_.size(); i++) {
for (size_t i = 0; i < labels_value.size(); i++) {
metric.labels[Base::labels_name_[i]] = labels_value[i];
}
metric.quantiles.emplace(quantiles_[i], rates[i]);
}
summary.metrics.push_back(std::move(metric));
metric.quantiles_value = std::move(rates);
metric.labels.reserve(labels_value.size());
for (auto& e : labels_value) metric.labels.emplace_back(e);
}
iguana::to_json(summary, str);
}

2
src/metric/tests/parallel_test.cpp
View File

@ -6,8 +6,10 @@
#include "ylt/metric.hpp"
TEST_CASE("test high parallel perform test") {
#ifndef _MSC_VER
bench_static_summary_mixed(std::thread::hardware_concurrency() * 4, 3s);
bench_dynamic_summary_mixed(std::thread::hardware_concurrency() * 4, 2s);
bench_static_counter_mixed(std::thread::hardware_concurrency() * 4, 2s);
bench_static_counter_mixed(std::thread::hardware_concurrency() * 4, 2s);
#endif
}

18
src/metric/tests/test_metric.cpp
View File

@ -631,7 +631,8 @@ TEST_CASE("test gauge") {
std::string str_json;
g.serialize_to_json(str_json);
std::cout << str_json << "\n";
CHECK(str_json.find("\"code\":\"200\"") != std::string::npos);
std::cout << str_json.size() << std::endl;
CHECK(str_json.size() == 185);
#endif
std::string str;
@ -702,7 +703,6 @@ TEST_CASE("test summary") {
std::string str_json;
summary.serialize_to_json(str_json);
std::cout << str_json << "\n";
CHECK(str_json.find("\"0.9\":") != std::string::npos);
#endif
}
@ -731,7 +731,8 @@ TEST_CASE("test summary with INF") {
std::string str_json;
summary.serialize_to_json(str_json);
std::cout << str_json << "\n";
CHECK(str_json.find("\"0.9\":") != std::string::npos);
std::cout << str_json.size() << std::endl;
CHECK(str_json.size() == 238);
#endif
}
@ -760,7 +761,8 @@ TEST_CASE("test summary with NAN") {
std::string str_json;
summary.serialize_to_json(str_json);
std::cout << str_json << "\n";
CHECK(str_json.find("\"0.9\":") != std::string::npos);
std::cout << str_json.size() << std::endl;
CHECK(str_json.size() == 238);
#endif
}
@ -793,7 +795,8 @@ TEST_CASE("test summary with illegal quantities") {
std::string str_json;
summary.serialize_to_json(str_json);
std::cout << str_json << "\n";
CHECK(str_json.find("\"0.9\":") != std::string::npos);
std::cout << str_json.size() << std::endl;
CHECK(str_json.size() == 233);
#endif
}
@ -811,7 +814,7 @@ TEST_CASE("test summary with many quantities") {
}
std::string str;
summary.serialize(str);
std::cout << str;
// std::cout << str;
double sum;
uint64_t cnt;
auto result = summary.get_rates(sum, cnt);
@ -828,7 +831,8 @@ TEST_CASE("test summary with many quantities") {
std::string str_json;
summary.serialize_to_json(str_json);
std::cout << str_json << "\n";
CHECK(str_json.find("\"0.9\":") != std::string::npos);
std::cout << str_json.size() << std::endl;
CHECK(str_json.size() == 8868);
#endif
}