Test more C API in upgrade tests (#7853)

* Test more C API in upgrade tests

* Fix saving results in blob granule API correctness test

* Enable multitenancy in upgrade test with API version >=720

* Disable the upgrade test with blob granule workload

bindings/c/CMakeLists.txt

@@ -341,6 +341,17 @@ if(NOT WIN32)
       )
   set_tests_properties("fdb_c_upgrade_to_future_version" PROPERTIES ENVIRONMENT "${SANITIZER_OPTIONS}")
 
+if (0) # reenable after stabilizing the test
+  add_test(NAME fdb_c_upgrade_to_future_version_blob_granules
+    COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
+        --build-dir ${CMAKE_BINARY_DIR}
+        --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/ApiBlobGranulesCorrectness.toml
+        --upgrade-path "7.2.0" "7.3.0" "7.2.0"
+        --blob-granules-enabled
+        --process-number 3
+      )
+endif()
+
   if(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT USE_SANITIZER)
     add_test(NAME fdb_c_upgrade_single_threaded_630api
       COMMAND ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py

bindings/c/test/apitester/TesterBlobGranuleCorrectnessWorkload.cpp

@@ -124,8 +124,10 @@ private:
 			    } else if (err.code() != error_code_success) {
 				    ctx->onError(err);
 			    } else {
-				    auto& [out_kv, out_count, out_more] = out;
+				    auto resCopy = copyKeyValueArray(out);
+				    auto& [resVector, out_more] = resCopy;
 				    ASSERT(!out_more);
+				    results.get()->assign(resVector.begin(), resVector.end());
 				    if (!seenReadSuccess) {
 					    info("BlobGranuleCorrectness::randomReadOp first success\n");
 				    }

bindings/c/test/apitester/tests/upgrade/ApiBlobGranulesCorrectness.toml

@@ -0,0 +1,23 @@
+[[test]]
+title = 'Mixed Workload for Upgrade Tests with a Multi-Threaded Client'
+multiThreaded = true
+buggify = true
+databasePerTransaction = false
+minFdbThreads = 2
+maxFdbThreads = 8
+minDatabases = 2
+maxDatabases = 8
+minClientThreads = 2
+maxClientThreads = 8
+minClients = 2
+maxClients = 8
+
+    [[test.workload]]
+    name = 'ApiBlobGranuleCorrectness'
+    minKeyLength = 1
+	maxKeyLength = 64
+	minValueLength = 1
+	maxValueLength = 1000
+	maxKeysPerTransaction = 50
+	initialSize = 100
+	runUntilStop = true

bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml

@@ -32,4 +32,14 @@ maxClients = 8
     maxKeysPerTransaction = 50
     initialSize = 100
     runUntilStop = true
-    readExistingKeysRatio = 0.9
+    readExistingKeysRatio = 0.9
+
+    [[test.workload]]
+    name = 'AtomicOpsCorrectness'
+    initialSize = 0
+    runUntilStop = true
+
+    [[test.workload]]
+    name = 'WatchAndWait'
+    initialSize = 0
+    runUntilStop = true

bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadSingleThr.toml

@@ -30,4 +30,14 @@ maxClients = 8
     maxKeysPerTransaction = 50
     initialSize = 100
     runUntilStop = true
-    readExistingKeysRatio = 0.9
+    readExistingKeysRatio = 0.9
+
+    [[test.workload]]
+    name = 'AtomicOpsCorrectness'
+    initialSize = 0
+    runUntilStop = true
+
+    [[test.workload]]
+    name = 'WatchAndWait'
+    initialSize = 0
+    runUntilStop = true

tests/TestRunner/upgrade_test.py

@@ -72,6 +72,7 @@ class UpgradeTest:
             args.process_number,
             create_config=False,
             redundancy=args.redundancy,
+            blob_granules_enabled=args.blob_granules_enabled,
         )
         self.cluster.create_cluster_file()
         self.configure_version(init_version)
@@ -88,6 +89,7 @@ class UpgradeTest:
         self.tester_proc = None
         self.output_pipe = None
         self.ctrl_pipe = None
+        self.determine_api_version()
 
     # Download all old binaries required for testing the specified upgrade path
     def download_old_binaries(self):
@@ -159,7 +161,7 @@ class UpgradeTest:
     def __enter__(self):
         print("Starting cluster version {}".format(self.cluster_version))
         self.cluster.start_cluster()
-        self.cluster.create_database(enable_tenants=False)
+        self.cluster.create_database(enable_tenants=(self.api_version >= 720))
         return self
 
     def __exit__(self, xc_type, exc_value, traceback):
@@ -176,7 +178,6 @@ class UpgradeTest:
     def exec_workload(self, test_file):
         self.tester_retcode = 1
         try:
-            self.determine_api_version()
             cmd_args = [
                 self.tester_bin,
                 "--cluster-file",
@@ -206,6 +207,8 @@ class UpgradeTest:
                 cmd_args = ["gdb", "-ex", "run", "--args"] + cmd_args
             if FUTURE_VERSION in self.upgrade_path:
                 cmd_args += ["--future-version-client-library", self.future_version_client_lib_path]
+            if self.cluster.blob_granules_enabled:
+                cmd_args += ["--blob-granule-local-file-path", str(self.cluster.data.joinpath("fdbblob")) + "/"]
             print("Executing test command: {}".format(" ".join([str(c) for c in cmd_args])))
 
             self.tester_proc = subprocess.Popen(cmd_args, stdout=sys.stdout, stderr=sys.stderr)
@@ -422,6 +425,7 @@ if __name__ == "__main__":
         help="Do not dump cluster log on error",
         action="store_true",
     )
+    parser.add_argument("--blob-granules-enabled", help="Enable blob granules", action="store_true")
     parser.add_argument("--run-with-gdb", help="Execute the tester binary from gdb", action="store_true")
     args = parser.parse_args()
     if args.process_number == 0: