Add ErrorKind field to SevError trace events

flow/Error.cpp

@@ -40,6 +40,10 @@ Error Error::fromUnvalidatedCode(int code) {
 		return Error::fromCode(code);
 }
 
+bool Error::isDiskError() const {
+	return (error_code == error_code_io_error || error_code == error_code_io_timeout);
+}
+
 Error internal_error_impl(const char* file, int line) {
 	fprintf(stderr, "Internal Error @ %s %d:\n  %s\n", file, line, platform::get_backtrace().c_str());
 
@@ -47,6 +51,7 @@ Error internal_error_impl(const char* file, int line) {
 	    .error(Error::fromCode(error_code_internal_error))
 	    .detail("File", file)
 	    .detail("Line", line)
+	    .setErrorKind(ErrorKind::BugDetected)
 	    .backtrace();
 	flushTraceFileVoid();
 	return Error(error_code_internal_error);
@@ -60,6 +65,7 @@ Error internal_error_impl(const char* msg, const char* file, int line) {
 	    .detail("FailedAssertion", msg)
 	    .detail("File", file)
 	    .detail("Line", line)
+	    .setErrorKind(ErrorKind::BugDetected)
 	    .backtrace();
 	flushTraceFileVoid();
 	return Error(error_code_internal_error);
@@ -86,6 +92,7 @@ Error internal_error_impl(const char* a_nm,
 	    .detail("RightValue", b)
 	    .detail("File", file)
 	    .detail("Line", line)
+	    .setErrorKind(ErrorKind::BugDetected)
 	    .backtrace();
 	flushTraceFileVoid();
 	return Error(error_code_internal_error);

flow/Error.h

@@ -50,6 +50,7 @@ public:
 		return flags & FLAG_INJECTED_FAULT;
 	} // Use as little as possible, so injected faults effectively test real faults!
 	bool isValid() const { return error_code != invalid_error_code; }
+	bool isDiskError() const;
 
 	template <class Ar>
 	void serialize(Ar& ar) {

flow/Trace.cpp

@@ -355,7 +355,6 @@ public:
 		if (localAddress.present()) {
 			fields.addField("Machine", formatIpPort(localAddress.get().ip, localAddress.get().port));
 		}
-
 		fields.addField("LogGroup", logGroup);
 
 		RoleInfo const& r = mutateRoleInfo();
@@ -653,6 +652,21 @@ bool traceClockSource(std::string& source) {
 		return false;
 	}
 }
+
+std::string toString(ErrorKind errorKind) {
+	switch (errorKind) {
+		case ErrorKind::Unset:
+			return "Unset";
+		case ErrorKind::DiskIssue:
+			return "DiskIssue";
+		case ErrorKind::BugDetected:
+			return "BugDetected";
+		default:
+			UNSTOPPABLE_ASSERT(false);
+			return "";
+	}
+}
+
 } // namespace
 
 bool selectTraceFormatter(std::string format) {
@@ -900,6 +914,10 @@ bool TraceEvent::init() {
 		}
 
 		detail("Severity", int(severity));
+		if (severity >= SevError) {
+			detail("ErrorKind", errorKind);
+			errorKindIndex = fields.size()-1;
+		}
 		detail("Time", "0.000000");
 		timeIndex = fields.size() - 1;
 		if (FLOW_KNOBS && FLOW_KNOBS->TRACE_DATETIME_ENABLED) {
@@ -942,6 +960,9 @@ TraceEvent& TraceEvent::errorImpl(class Error const& error, bool includeCancelle
 			detail("ErrorDescription", error.what());
 			detail("ErrorCode", error.code());
 		}
+		if (err.isDiskError()) {
+			setErrorKind(ErrorKind::DiskIssue);
+		}
 	} else {
 		if (initialized) {
 			TraceEvent(g_network && g_network->isSimulated() ? SevError : SevWarnAlways,
@@ -1152,6 +1173,9 @@ void TraceEvent::log() {
 					severity = SevInfo;
 					backtrace();
 					severity = SevError;
+					if (errorKindIndex != -1) {
+						fields.mutate(errorKindIndex).second = toString(errorKind);
+					}
 				}
 
 				if (isNetworkThread()) {
@@ -1365,12 +1389,12 @@ TraceEventFields::TraceEventFields() : bytes(0), annotated(false) {}
 
 void TraceEventFields::addField(const std::string& key, const std::string& value) {
 	bytes += key.size() + value.size();
-	fields.push_back(std::make_pair(key, value));
+	fields.emplace_back(key, value);
 }
 
 void TraceEventFields::addField(std::string&& key, std::string&& value) {
 	bytes += key.size() + value.size();
-	fields.push_back(std::make_pair(std::move(key), std::move(value)));
+	fields.emplace_back(std::move(key), std::move(value));
 }
 
 size_t TraceEventFields::size() const {

flow/Trace.h

@@ -64,6 +64,12 @@ enum Severity {
 	SevMax = 1000000
 };
 
+enum class ErrorKind : uint8_t {
+	Unset,
+	DiskIssue,
+	BugDetected,
+};
+
 const int NUM_MAJOR_LEVELS_OF_EVENTS = SevMaxUsed / 10 + 1;
 
 class TraceEventFields {
@@ -457,6 +463,11 @@ public:
 
 	bool isEnabled() const { return enabled; }
 
+	TraceEvent &setErrorKind(ErrorKind errorKind) {
+		this->errorKind = errorKind;
+		return *this;
+	}
+
 	explicit operator bool() const { return enabled; }
 
 	void log();
@@ -475,6 +486,7 @@ private:
 	std::string trackingKey;
 	TraceEventFields fields;
 	Severity severity;
+	ErrorKind errorKind;
 	const char* type;
 	UID id;
 	Error err;
@@ -482,6 +494,7 @@ private:
 	int maxFieldLength;
 	int maxEventLength;
 	int timeIndex;
+	int errorKindIndex { -1 };
 
 	void setSizeLimits();