Harness to spawn multiple FDB clusters, and Vishesh's python multithreaded client test

tests/loopback_cluster/run_cluster.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+set -euo pipefail
+trap "kill 0" EXIT
+ROOT=`pwd`
+
+function usage {
+	echo "Usage 'cd working-directory; ${0} path-to-build-root number-of-clusters-to-start' "
+	exit 1
+}
+
+if (( $# != 2 )) ; then
+	echo Wrong number of arguments
+	usage
+fi
+
+BUILD=$1
+
+FDB=${BUILD}/bin/fdbserver
+if [ ! -f ${FDB} ]; then
+	echo "Error: ${FDB} not found!"
+	usage
+fi
+
+for i in `seq 1 $2` ; do
+	DIR=./loopback-cluster-$i
+	PORT_PREFIX=${i}50
+	rm -rf ${DIR}
+	mkdir -p ${DIR}
+	CLUSTER_FILE="test$i:testdb$i@127.0.0.1:${PORT_PREFIX}1"
+	CLUSTER=${DIR}/fdb.cluster
+	echo $CLUSTER_FILE > $CLUSTER
+	echo "Starting Cluster: " $CLUSTER_FILE
+
+	for j in 1 2 3; do
+		LOG=${DIR}/${j}/log
+		DATA=${DIR}/${j}/data
+		mkdir -p ${LOG} ${DATA}
+		${FDB} -p auto:${PORT_PREFIX}${j} -d $DATA -L $LOG -C $CLUSTER &
+	done
+	
+	CLI="$ROOT/bin/fdbcli -C ${CLUSTER} --exec"
+	( sleep 2 ; $CLI "configure new ssd single" ; sleep 1 ; $CLI "status minimal" ) &
+done;
+sleep 10000000000

tests/python_tests/multithreaded_client.py

@@ -0,0 +1,48 @@
+#!/bin/env python2
+import sys
+
+### sample usage (from inside your FDB build output directory):
+
+## Spawn three test clusters:
+# ../tests/loopback_cluster/run_cluster.sh . 3 &
+
+## Run this test
+# ../tests/python_tests/multithreaded_client.py . loopback-cluster-*/fdb.cluster
+
+## Cleanup
+# pkill run_cluster.sh
+# rm -rf loopback-cluster-* client-logs
+
+builddir = sys.argv[1]
+clusters = sys.argv[2:]
+
+sys.path.append(builddir + '/bindings/python')
+
+import fdb
+import os
+import random
+import time
+fdb.api_version(630)
+
+if not os.path.exists("client-logs"):
+    os.mkdir("client-logs")
+
+fdb.options.set_trace_enable("client-logs/")
+fdb.options.set_external_client_directory(builddir + '/lib')
+fdb.options.set_knob("min_trace_severity=5")
+
+
+fdb.options.set_client_threads_per_version(len(clusters))
+
+dbs = []
+for v in clusters:
+    dbs.append(fdb.open(cluster_file=v))
+
+counter = 0
+for i in range(100):
+    key = b"test_%d" % random.randrange(0, 100000000)
+    val = b"value_%d" % random.randrange(0, 10000000)
+    db = dbs[i % len(dbs)]
+    print ("Writing: ", key, val, db)
+    db[key] = val
+    assert (val == db[key])