Introduce a simulate EncryptKeyVaultProxy interface (#6576)

Description

Major changes proposed are:
1. Rename ServerKnob->ENABLE_ENCRYPT_KEY_PROXY to
   ServerKnob->ENABLE_ENCRYPTION. Approach simplifies enabling
   controlling encyrption code change using a single knob (desirable)
2. Implement EncyrptKeyVaultProxy simulated interface to assist
   validating encyrption workflows in simulation runs. The interface
   is leveraged to satisfy "encryption keys" lookup which otherwise
   gets satisfied by integrating organization preferred Encryption
   Key Management solution.

Testing

Unit test to validate the newly added code

fdbclient/ServerKnobs.cpp

@@ -816,7 +816,7 @@ void ServerKnobs::initialize(Randomize randomize, ClientKnobs* clientKnobs, IsSi
 	init ( CLUSTER_RECOVERY_EVENT_NAME_PREFIX,               "Master");
 
     // encrypt key proxy
-    init( ENABLE_ENCRYPT_KEY_PROXY,                            false );
+    init( ENABLE_ENCRYPTION,                                   false );
 
 	// Blob granlues
 	init( BG_URL,                                                 "" ); // TODO: store in system key space, eventually

fdbclient/ServerKnobs.h

@@ -765,7 +765,7 @@ public:
 	std::string CLUSTER_RECOVERY_EVENT_NAME_PREFIX;
 
 	// encrypt key proxy
-	bool ENABLE_ENCRYPT_KEY_PROXY;
+	bool ENABLE_ENCRYPTION;
 
 	// blob granule stuff
 	// FIXME: configure url with database configuration instead of knob eventually

fdbserver/CMakeLists.txt

@@ -117,6 +117,8 @@ set(FDBSERVER_SRCS
   ServerDBInfo.actor.h
   ServerDBInfo.h
   SigStack.cpp
+  SimEncryptVaultProxy.actor.h
+  SimEncryptVaultProxy.actor.cpp
   SimpleConfigConsumer.actor.cpp
   SimpleConfigConsumer.h
   SimulatedCluster.actor.cpp

fdbserver/ClusterController.actor.cpp

@@ -599,7 +599,7 @@ void checkBetterSingletons(ClusterControllerData* self) {
 	}
 
 	WorkerDetails newEKPWorker;
-	if (SERVER_KNOBS->ENABLE_ENCRYPT_KEY_PROXY) {
+	if (SERVER_KNOBS->ENABLE_ENCRYPTION) {
 		newEKPWorker = findNewProcessForSingleton(self, ProcessClass::EncryptKeyProxy, id_used);
 	}
 
@@ -613,7 +613,7 @@ void checkBetterSingletons(ClusterControllerData* self) {
 	}
 
 	ProcessClass::Fitness bestFitnessForEKP;
-	if (SERVER_KNOBS->ENABLE_ENCRYPT_KEY_PROXY) {
+	if (SERVER_KNOBS->ENABLE_ENCRYPTION) {
 		bestFitnessForEKP = findBestFitnessForSingleton(self, newEKPWorker, ProcessClass::EncryptKeyProxy);
 	}
 
@@ -638,7 +638,7 @@ void checkBetterSingletons(ClusterControllerData* self) {
 	}
 
 	bool ekpHealthy = true;
-	if (SERVER_KNOBS->ENABLE_ENCRYPT_KEY_PROXY) {
+	if (SERVER_KNOBS->ENABLE_ENCRYPTION) {
 		ekpHealthy = isHealthySingleton<EncryptKeyProxyInterface>(
 		    self, newEKPWorker, ekpSingleton, bestFitnessForEKP, self->recruitingEncryptKeyProxyID);
 	}
@@ -662,7 +662,7 @@ void checkBetterSingletons(ClusterControllerData* self) {
 	}
 
 	Optional<Standalone<StringRef>> currEKPProcessId, newEKPProcessId;
-	if (SERVER_KNOBS->ENABLE_ENCRYPT_KEY_PROXY) {
+	if (SERVER_KNOBS->ENABLE_ENCRYPTION) {
 		currEKPProcessId = ekpSingleton.interface.get().locality.processId();
 		newEKPProcessId = newEKPWorker.interf.locality.processId();
 	}
@@ -674,7 +674,7 @@ void checkBetterSingletons(ClusterControllerData* self) {
 		newPids.emplace_back(newBMProcessId);
 	}
 
-	if (SERVER_KNOBS->ENABLE_ENCRYPT_KEY_PROXY) {
+	if (SERVER_KNOBS->ENABLE_ENCRYPTION) {
 		currPids.emplace_back(currEKPProcessId);
 		newPids.emplace_back(newEKPProcessId);
 	}
@@ -689,7 +689,7 @@ void checkBetterSingletons(ClusterControllerData* self) {
 	}
 
 	// if the knob is disabled, the EKP coloc counts should have no affect on the coloc counts check below
-	if (!SERVER_KNOBS->ENABLE_ENCRYPT_KEY_PROXY) {
+	if (!SERVER_KNOBS->ENABLE_ENCRYPTION) {
 		ASSERT(currColocMap[currEKPProcessId] == 0);
 		ASSERT(newColocMap[newEKPProcessId] == 0);
 	}
@@ -706,8 +706,7 @@ void checkBetterSingletons(ClusterControllerData* self) {
 			ddSingleton.recruit(self);
 		} else if (CLIENT_KNOBS->ENABLE_BLOB_GRANULES && newColocMap[newBMProcessId] < currColocMap[currBMProcessId]) {
 			bmSingleton.recruit(self);
-		} else if (SERVER_KNOBS->ENABLE_ENCRYPT_KEY_PROXY &&
-		           newColocMap[newEKPProcessId] < currColocMap[currEKPProcessId]) {
+		} else if (SERVER_KNOBS->ENABLE_ENCRYPTION && newColocMap[newEKPProcessId] < currColocMap[currEKPProcessId]) {
 			ekpSingleton.recruit(self);
 		}
 	}
@@ -1240,7 +1239,7 @@ void registerWorker(RegisterWorkerRequest req,
 		    self, w, currSingleton, registeringSingleton, self->recruitingBlobManagerID);
 	}
 
-	if (SERVER_KNOBS->ENABLE_ENCRYPT_KEY_PROXY && req.encryptKeyProxyInterf.present()) {
+	if (SERVER_KNOBS->ENABLE_ENCRYPTION && req.encryptKeyProxyInterf.present()) {
 		auto currSingleton = EncryptKeyProxySingleton(self->db.serverInfo->get().encryptKeyProxy);
 		auto registeringSingleton = EncryptKeyProxySingleton(req.encryptKeyProxyInterf);
 		haltRegisteringOrCurrentSingleton<EncryptKeyProxyInterface>(
@@ -2416,7 +2415,7 @@ ACTOR Future<Void> clusterControllerCore(ClusterControllerFullInterface interf,
 	state Future<ErrorOr<Void>> error = errorOr(actorCollection(self.addActor.getFuture()));
 
 	// EncryptKeyProxy is necessary for TLog recovery, recruit it as the first process
-	if (SERVER_KNOBS->ENABLE_ENCRYPT_KEY_PROXY) {
+	if (SERVER_KNOBS->ENABLE_ENCRYPTION) {
 		self.addActor.send(monitorEncryptKeyProxy(&self));
 	}
 	self.addActor.send(clusterWatchDatabase(&self, &self.db, coordinators, leaderFail)); // Start the master database

fdbserver/EncryptKeyProxy.actor.cpp

@@ -19,14 +19,18 @@
  */
 
 #include "fdbserver/EncryptKeyProxyInterface.h"
+#include "fdbserver/SimEncryptVaultProxy.actor.h"
 #include "fdbserver/WorkerInterface.actor.h"
 #include "fdbserver/ServerDBInfo.h"
 #include "flow/Arena.h"
 #include "flow/Error.h"
 #include "flow/EventTypes.actor.h"
 #include "flow/FastRef.h"
+#include "flow/IRandom.h"
 #include "flow/Trace.h"
 #include "flow/genericactors.actor.h"
+#include "flow/network.h"
+
 #include "flow/actorcompiler.h" // This must be the last #include.
 
 struct EncryptKeyProxyData : NonCopyable, ReferenceCounted<EncryptKeyProxyData> {
@@ -42,6 +46,17 @@ ACTOR Future<Void> encryptKeyProxyServer(EncryptKeyProxyInterface ekpInterface,
 	state Future<Void> collection = actorCollection(self->addActor.getFuture());
 	self->addActor.send(traceRole(Role::ENCRYPT_KEY_PROXY, ekpInterface.id()));
 
+	SimEncryptVaultProxyInterface simEncryptVaultProxyInf;
+	if (g_network->isSimulated()) {
+
+		// In simulation construct an EncryptVaultProxy actor to satisfy encryption keys lookups otherwise satisfied by
+		// integrating external Encryption Key Management solutions.
+
+		const uint32_t maxEncryptKeys = deterministicRandom()->randomInt(1024, 2048);
+		simEncryptVaultProxyInf.initEndpoints();
+		self->addActor.send(simEncryptVaultProxyCore(simEncryptVaultProxyInf, maxEncryptKeys));
+	}
+
 	TraceEvent("EKP_Start", self->myId).log();
 
 	// TODO(ahusain): skeleton implementation, more to come

fdbserver/EncryptKeyProxyInterface.h

@@ -20,6 +20,7 @@
 
 #ifndef FDBSERVER_ENCRYPTKEYPROXYINTERFACE_H
 #define FDBSERVER_ENCRYPTKEYPROXYINTERFACE_H
+#include "flow/FileIdentifier.h"
 #include "flow/network.h"
 #pragma once
 

fdbserver/SimEncryptVaultProxy.actor.cpp

@@ -0,0 +1,153 @@
+/*
+ * SimEncryptVaulProxy.actor.cpp
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <memory>
+#include <unordered_map>
+
+#include "fdbrpc/sim_validation.h"
+#include "fdbserver/SimEncryptVaultProxy.actor.h"
+#include "flow/ActorCollection.h"
+#include "flow/Error.h"
+#include "flow/IRandom.h"
+#include "flow/ITrace.h"
+#include "flow/StreamCipher.h"
+#include "flow/UnitTest.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+struct SimEncryptKeyCtx {
+	SimEncryptKeyId id;
+	SimEncryptKey key;
+
+	SimEncryptKeyCtx() : id(0) {}
+	explicit SimEncryptKeyCtx(SimEncryptKeyId kId, const char* data) : id(kId), key(data) {}
+};
+
+struct SimEncyrptVaultProxyContext {
+	uint32_t maxEncryptionKeys;
+	std::unordered_map<SimEncryptKeyId, std::unique_ptr<SimEncryptKeyCtx>> simEncryptKeyStore;
+
+	SimEncyrptVaultProxyContext() : maxEncryptionKeys(0) {}
+	explicit SimEncyrptVaultProxyContext(uint32_t keyCount) : maxEncryptionKeys(keyCount) {
+		uint8_t buffer[AES_256_KEY_LENGTH];
+
+		// Construct encryption keyStore.
+		for (int i = 0; i < maxEncryptionKeys; i++) {
+			generateRandomData(&buffer[0], AES_256_KEY_LENGTH);
+			SimEncryptKeyCtx ctx(i, reinterpret_cast<const char*>(buffer));
+			simEncryptKeyStore[i] = std::make_unique<SimEncryptKeyCtx>(i, reinterpret_cast<const char*>(buffer));
+		}
+	}
+};
+
+ACTOR Future<Void> simEncryptVaultProxyCore(SimEncryptVaultProxyInterface interf, uint32_t maxEncryptKeys) {
+	state SimEncyrptVaultProxyContext vaultProxyCtx(maxEncryptKeys);
+
+	ASSERT(vaultProxyCtx.simEncryptKeyStore.size() == maxEncryptKeys);
+
+	TraceEvent("SimEncryptVaultProxy_Init", interf.id()).detail("MaxEncrptKeys", maxEncryptKeys);
+
+	loop {
+		choose {
+			when(SimGetEncryptKeyByKeyIdRequest req = waitNext(interf.encryptKeyLookupByKeyId.getFuture())) {
+				SimGetEncryptKeyByKeyIdReply reply;
+
+				// Lookup corresponding EncryptKeyCtx for input keyId
+				if (vaultProxyCtx.simEncryptKeyStore.find(req.encryptKeyId) != vaultProxyCtx.simEncryptKeyStore.end()) {
+					reply.encryptKey = StringRef(vaultProxyCtx.simEncryptKeyStore[req.encryptKeyId].get()->key);
+					req.reply.send(reply);
+				} else {
+					req.reply.sendError(key_not_found());
+				}
+			}
+			when(SimGetEncryptKeyByDomainIdRequest req = waitNext(interf.encryptKeyLookupByDomainId.getFuture())) {
+				SimGetEncryptKeyByDomainIdReply reply;
+
+				// Map encryptionDomainId to corresponding EncryptKeyCtx element using a modulo operation. This would
+				// mean multiple domains gets mapped to the same encryption key which is fine, the EncryptKeyStore
+				// guarantees that keyId -> plaintext encryptKey mapping is idempotent.
+
+				reply.encryptKeyId = req.encryptDomainId % maxEncryptKeys;
+				reply.encryptKey = StringRef(vaultProxyCtx.simEncryptKeyStore[reply.encryptKeyId].get()->key);
+				req.reply.send(reply);
+			}
+		}
+	}
+}
+
+void forceLinkSimEncryptVaultProxyTests() {}
+
+namespace {
+
+ACTOR Future<Void> testRunWorkload(SimEncryptVaultProxyInterface inf, uint32_t nEncryptionKeys) {
+	state uint32_t maxEncryptionKeys = nEncryptionKeys;
+	state int maxDomainIds = deterministicRandom()->randomInt(121, 295);
+	state int maxIterations = deterministicRandom()->randomInt(786, 1786);
+	state std::unordered_map<SimEncryptDomainId, std::unique_ptr<SimEncryptKeyCtx>> domainIdKeyMap;
+	state int i = 0;
+
+	TraceEvent("RunWorkloadStart").detail("MaxDomainIds", maxDomainIds).detail("MaxIterations", maxIterations);
+
+	{
+		// construct domainId to EncryptKeyCtx map
+		for (i = 0; i < maxDomainIds; i++) {
+			SimGetEncryptKeyByDomainIdRequest req;
+			req.encryptDomainId = i;
+			SimGetEncryptKeyByDomainIdReply reply = wait(inf.encryptKeyLookupByDomainId.getReply(req));
+			domainIdKeyMap[i] =
+			    std::make_unique<SimEncryptKeyCtx>(reply.encryptKeyId, reply.encryptKey.toString().c_str());
+		}
+
+		// randomly pick any domainId and validate if lookupByKeyId result matches
+		for (i = 0; i < maxIterations; i++) {
+			state int idx = deterministicRandom()->randomInt(0, maxDomainIds);
+			state SimEncryptKeyCtx* ctx = domainIdKeyMap[idx].get();
+			SimGetEncryptKeyByKeyIdRequest req(ctx->id);
+			SimGetEncryptKeyByKeyIdReply reply = wait(inf.encryptKeyLookupByKeyId.getReply(req));
+			ASSERT(reply.encryptKey.compare(ctx->key) == 0);
+		}
+	}
+
+	{
+		// Verify unknown key access returns the error
+		state SimGetEncryptKeyByKeyIdRequest req;
+		req.encryptKeyId = maxEncryptionKeys + 1;
+		try {
+			SimGetEncryptKeyByKeyIdReply reply = wait(inf.encryptKeyLookupByKeyId.getReply(req));
+		} catch (Error& e) {
+			ASSERT(e.code() == error_code_key_not_found);
+		}
+	}
+
+	TraceEvent("RunWorkloadDone").log();
+	return Void();
+}
+
+} // namespace
+
+TEST_CASE("fdbserver/SimEncryptVaultProxy") {
+	state SimEncryptVaultProxyInterface inf;
+	state uint32_t maxEncryptKeys = 64;
+
+	loop choose {
+		when(wait(simEncryptVaultProxyCore(inf, maxEncryptKeys))) { throw internal_error(); }
+		when(wait(testRunWorkload(inf, maxEncryptKeys))) { break; }
+	}
+	return Void();
+}

fdbserver/SimEncryptVaultProxy.actor.h

@@ -0,0 +1,132 @@
+/*
+ * SimEncryptVaultProxy.actor.h
+ *
+ * This source file is part of the FoundationDB open source project
+ *
+ * Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#if defined(NO_INTELLISENSE) && !defined(FDBSERVER_SIMENCRYPTVAULTPROXY_ACTOR_G_H)
+#define FDBSERVER_SIMENCRYPTVAULTPROXY_ACTOR_G_H
+#include "fdbserver/SimEncryptVaultProxy.actor.g.h"
+#elif !defined(FDBSERVER_SIMENCRYPTVAULTPROXY_ACTOR_H)
+#define FDBSERVER_SIMENCRYPTVAULTPROXY_ACTOR_H
+
+#include "fdbclient/FDBTypes.h"
+#include "fdbrpc/fdbrpc.h"
+#include "flow/FileIdentifier.h"
+#include "flow/Trace.h"
+#include "flow/flow.h"
+#include "flow/network.h"
+#include "flow/actorcompiler.h" // This must be the last #include.
+
+using SimEncryptKeyId = uint64_t;
+using SimEncryptDomainId = uint64_t;
+using SimEncryptKey = std::string;
+
+struct SimEncryptVaultProxyInterface {
+	constexpr static FileIdentifier file_identifier = 2416711;
+	RequestStream<ReplyPromise<Void>> waitFailure;
+	RequestStream<struct SimGetEncryptKeyByKeyIdRequest> encryptKeyLookupByKeyId;
+	RequestStream<struct SimGetEncryptKeyByDomainIdRequest> encryptKeyLookupByDomainId;
+
+	SimEncryptVaultProxyInterface() {}
+
+	UID id() const { return encryptKeyLookupByKeyId.getEndpoint().token; }
+	template <class Archive>
+	void serialize(Archive& ar) {
+		if constexpr (!is_fb_function<Archive>) {
+			ASSERT(ar.protocolVersion().isValid());
+		}
+		serializer(ar, waitFailure);
+		if (Archive::isDeserializing) {
+			encryptKeyLookupByKeyId =
+			    RequestStream<struct GetCommitVersionRequest>(waitFailure.getEndpoint().getAdjustedEndpoint(1));
+			encryptKeyLookupByDomainId =
+			    RequestStream<struct GetRawCommittedVersionRequest>(waitFailure.getEndpoint().getAdjustedEndpoint(2));
+		}
+	}
+
+	void initEndpoints() {
+		std::vector<std::pair<FlowReceiver*, TaskPriority>> streams;
+		streams.push_back(waitFailure.getReceiver());
+		streams.push_back(encryptKeyLookupByKeyId.getReceiver(TaskPriority::DefaultPromiseEndpoint));
+		streams.push_back(encryptKeyLookupByDomainId.getReceiver(TaskPriority::DefaultPromiseEndpoint));
+		FlowTransport::transport().addEndpoints(streams);
+	}
+};
+
+struct SimGetEncryptKeyByKeyIdReply {
+	constexpr static FileIdentifier file_identifier = 2313778;
+	Standalone<StringRef> encryptKey;
+
+	SimGetEncryptKeyByKeyIdReply() : encryptKey(StringRef()) {}
+	explicit SimGetEncryptKeyByKeyIdReply(Standalone<StringRef> key) : encryptKey(key) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, encryptKey);
+	}
+};
+
+struct SimGetEncryptKeyByKeyIdRequest {
+	constexpr static FileIdentifier file_identifier = 6913396;
+	SimEncryptKeyId encryptKeyId;
+	ReplyPromise<SimGetEncryptKeyByKeyIdReply> reply;
+
+	SimGetEncryptKeyByKeyIdRequest() : encryptKeyId(0) {}
+	explicit SimGetEncryptKeyByKeyIdRequest(SimEncryptKeyId keyId) : encryptKeyId(keyId) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, encryptKeyId, reply);
+	}
+};
+
+struct SimGetEncryptKeyByDomainIdReply {
+	constexpr static FileIdentifier file_identifier = 3009025;
+	SimEncryptDomainId encryptKeyId;
+	Standalone<StringRef> encryptKey;
+
+	SimGetEncryptKeyByDomainIdReply() : encryptKeyId(0), encryptKey(StringRef()) {}
+	explicit SimGetEncryptKeyByDomainIdReply(SimEncryptKeyId keyId, Standalone<StringRef> key)
+	  : encryptKeyId(keyId), encryptKey(key) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, encryptKeyId, encryptKey);
+	}
+};
+
+struct SimGetEncryptKeyByDomainIdRequest {
+	constexpr static FileIdentifier file_identifier = 9918682;
+	SimEncryptDomainId encryptDomainId;
+	ReplyPromise<SimGetEncryptKeyByDomainIdReply> reply;
+
+	SimGetEncryptKeyByDomainIdRequest() : encryptDomainId(0) {}
+	explicit SimGetEncryptKeyByDomainIdRequest(SimEncryptDomainId domainId) : encryptDomainId(domainId) {}
+
+	template <class Ar>
+	void serialize(Ar& ar) {
+		serializer(ar, encryptDomainId, reply);
+	}
+};
+
+ACTOR Future<Void> simEncryptVaultProxyCore(struct SimEncryptVaultProxyInterface interf, uint32_t maxEncryptKeys);
+
+#include "flow/unactorcompiler.h"
+#endif // FDBSERVER_SIMENCRYPTVAULTPROXY_ACTOR_H

fdbserver/Status.actor.cpp

@@ -813,7 +813,7 @@ ACTOR static Future<JsonBuilderObject> processStatusFetcher(
 		roles.addRole("blob_manager", db->get().blobManager.get());
 	}
 
-	if (SERVER_KNOBS->ENABLE_ENCRYPT_KEY_PROXY && db->get().encryptKeyProxy.present()) {
+	if (SERVER_KNOBS->ENABLE_ENCRYPTION && db->get().encryptKeyProxy.present()) {
 		roles.addRole("encrypt_key_proxy", db->get().encryptKeyProxy.get());
 	}
 

fdbserver/workloads/ConsistencyCheck.actor.cpp

@@ -2374,7 +2374,7 @@ struct ConsistencyCheckWorkload : TestWorkload {
 		}
 
 		// Check EncryptKeyProxy
-		if (SERVER_KNOBS->ENABLE_ENCRYPT_KEY_PROXY && db.encryptKeyProxy.present() &&
+		if (SERVER_KNOBS->ENABLE_ENCRYPTION && db.encryptKeyProxy.present() &&
 		    (!nonExcludedWorkerProcessMap.count(db.encryptKeyProxy.get().address()) ||
 		     nonExcludedWorkerProcessMap[db.encryptKeyProxy.get().address()].processClass.machineClassFitness(
 		         ProcessClass::EncryptKeyProxy) > fitnessLowerBound)) {

fdbserver/workloads/UnitTests.actor.cpp

@@ -34,6 +34,7 @@ void forceLinkStreamCipherTests();
 void forceLinkParallelStreamTests();
 void forceLinkSimExternalConnectionTests();
 void forceLinkMutationLogReaderTests();
+void forceLinkSimEncryptVaultProxyTests();
 void forceLinkIThreadPoolTests();
 
 struct UnitTestWorkload : TestWorkload {
@@ -79,6 +80,7 @@ struct UnitTestWorkload : TestWorkload {
 		forceLinkParallelStreamTests();
 		forceLinkSimExternalConnectionTests();
 		forceLinkMutationLogReaderTests();
+		forceLinkSimEncryptVaultProxyTests();
 		forceLinkIThreadPoolTests();
 	}