Add knob to control trace recording percentage

documentation/sphinx/source/request-tracing.rst

@@ -95,3 +95,13 @@ Tracing can be enabled or disabled for individual transactions. The special key
 space exposes an API to set a custom trace ID for a transaction, or to disable
 tracing for the transaction. See the special key space :ref:`tracing module
 documentation <special-key-space-tracing-module>` to learn more.
+
+^^^^^^^^^^^^^^
+Trace sampling
+^^^^^^^^^^^^^^
+
+By default, all traces are recorded. If tracing is producing too much data,
+adjust the trace sample rate with the ``TRACING_SAMPLE_RATE`` knob. Set the
+knob to 0.0 to record no traces, to 1.0 to record all traces, or somewhere in
+the middle. Traces are sampled as a unit. All individual spans in the trace
+will be included in the sample.

fdbclient/NativeAPI.actor.cpp

@@ -4130,11 +4130,14 @@ void debugAddTags(Transaction* tr) {
 }
 
 SpanID generateSpanID(int transactionTracingEnabled) {
-	uint64_t tid = deterministicRandom()->randomUInt64();
+	uint64_t txnId = deterministicRandom()->randomUInt64();
 	if (transactionTracingEnabled > 0) {
-		return SpanID(tid, deterministicRandom()->randomUInt64());
+		uint64_t tokenId = deterministicRandom()->random01() <= FLOW_KNOBS->TRACING_SAMPLE_RATE
+		                       ? deterministicRandom()->randomUInt64()
+		                       : 0;
+		return SpanID(txnId, tokenId);
 	} else {
-		return SpanID(tid, 0);
+		return SpanID(txnId, 0);
 	}
 }
 
@@ -5766,7 +5769,7 @@ Future<Version> Transaction::getReadVersion(uint32_t flags) {
 		}
 
 		Location location = "NAPI:getReadVersion"_loc;
-		UID spanContext = deterministicRandom()->randomUniqueID();
+		UID spanContext = generateSpanID(cx->transactionTracingEnabled);
 		auto const req = DatabaseContext::VersionRequest(spanContext, options.tags, info.debugID);
 		batcher.stream.send(req);
 		startTime = now();
@@ -6129,7 +6132,7 @@ ACTOR Future<std::pair<Optional<StorageMetrics>, int>> waitStorageMetrics(Databa
                                                                           StorageMetrics permittedError,
                                                                           int shardLimit,
                                                                           int expectedShardCount) {
-	state Span span("NAPI:WaitStorageMetrics"_loc);
+	state Span span("NAPI:WaitStorageMetrics"_loc, generateSpanID(cx->transactionTracingEnabled));
 	loop {
 		std::vector<std::pair<KeyRange, Reference<LocationInfo>>> locations =
 		    wait(getKeyRangeLocations(cx,

flow/Knobs.cpp

@@ -67,7 +67,8 @@ void FlowKnobs::initialize(Randomize randomize, IsSimulated isSimulated) {
 	init( HUGE_ARENA_LOGGING_INTERVAL,                         5.0 );
 
 	init( WRITE_TRACING_ENABLED,                              true ); if( randomize && BUGGIFY ) WRITE_TRACING_ENABLED = false;
-	init( TRACING_UDP_LISTENER_PORT,                          8889 ); // Only applicable if TracerType is set to a network option.
+	init( TRACING_SAMPLE_RATE,                                 1.0 ); // Fraction of traces (not spans) to sample (0 means ignore all traces)
+	init( TRACING_UDP_LISTENER_PORT,                          8889 ); // Only applicable if TracerType is set to a network option
 
 	//connectionMonitor
 	init( CONNECTION_MONITOR_LOOP_TIME,   isSimulated ? 0.75 : 1.0 ); if( randomize && BUGGIFY ) CONNECTION_MONITOR_LOOP_TIME = 6.0;

flow/Knobs.h

@@ -117,6 +117,7 @@ public:
 	double HUGE_ARENA_LOGGING_INTERVAL;
 
 	bool WRITE_TRACING_ENABLED;
+	double TRACING_SAMPLE_RATE;
 	int TRACING_UDP_LISTENER_PORT;
 
 	// run loop profiling

flow/Tracing.actor.cpp

@@ -373,7 +373,7 @@ void openTracer(TracerType type) {
 ITracer::~ITracer() {}
 
 Span& Span::operator=(Span&& o) {
-	if (begin > 0.0) {
+	if (begin > 0.0 && context.second() > 0) {
 		end = g_network->now();
 		g_tracer->trace(*this);
 	}
@@ -388,7 +388,7 @@ Span& Span::operator=(Span&& o) {
 }
 
 Span::~Span() {
-	if (begin > 0.0) {
+	if (begin > 0.0 && context.second() > 0) {
 		end = g_network->now();
 		g_tracer->trace(*this);
 	}

flow/Tracing.h

@@ -37,11 +37,19 @@ struct Span {
 	Span(SpanID context, Location location, std::initializer_list<SpanID> const& parents = {})
 	  : context(context), begin(g_network->now()), location(location), parents(arena, parents.begin(), parents.end()) {
 		if (parents.size() > 0) {
-			this->context = SpanID((*parents.begin()).first(), context.second());
+			// If the parents' token is 0 (meaning the trace should not be
+			// recorded), set the child token to 0 as well. Otherwise, use the
+			// existing (likely randomly generated) value.
+			uint64_t traceId = (*parents.begin()).second() > 0 ? context.second() : 0;
+			this->context = SpanID((*parents.begin()).first(), traceId);
 		}
 	}
-	Span(Location location, std::initializer_list<SpanID> const& parents = {})
-	  : Span(deterministicRandom()->randomUniqueID(), location, parents) {}
+	Span(Location location, std::initializer_list<SpanID> const& parents = {}) {
+		uint64_t tokenId = deterministicRandom()->random01() < FLOW_KNOBS->TRACING_SAMPLE_RATE
+		                       ? deterministicRandom()->randomUInt64()
+		                       : 0;
+		Span(UID(deterministicRandom()->randomUInt64(), tokenId), location, parents);
+	}
 	Span(Location location, SpanID context) : Span(location, { context }) {}
 	Span(const Span&) = delete;
 	Span(Span&& o) {
@@ -70,12 +78,13 @@ struct Span {
 
 	void addParent(SpanID span) {
 		if (parents.size() == 0) {
+			uint64_t traceId = (*parents.begin()).second() > 0 ? context.second() : 0;
 			// Use first parent to set trace ID. This is non-ideal for spans
 			// with multiple parents, because the trace ID will associate the
 			// span with only one trace. A workaround is to look at the parent
 			// relationships instead of the trace ID. Another option in the
 			// future is to keep a list of trace IDs.
-			context = SpanID(span.first(), context.second());
+			context = SpanID(span.first(), traceId);
 		}
 		parents.push_back(arena, span);
 	}
@@ -112,7 +121,7 @@ void openTracer(TracerType type);
 template <class T>
 struct SpannedDeque : Deque<T> {
 	Span span;
-	explicit SpannedDeque(Location loc) : span(deterministicRandom()->randomUniqueID(), loc) {}
+	explicit SpannedDeque(Location loc) : span(loc) {}
 	SpannedDeque(SpannedDeque&& other) : Deque<T>(std::move(other)), span(std::move(other.span)) {}
 	SpannedDeque(SpannedDeque const&) = delete;
 	SpannedDeque& operator=(SpannedDeque const&) = delete;