Support HA in loopback cluster test script (#7953)

* update tag client cache on getKeyLocation

* Support HA for custom cluster test script

tests/loopback_cluster/run_custom_cluster.sh

@@ -14,6 +14,7 @@ KNOBS=""
 LOGS_TASKSET=""
 STATELESS_TASKSET=""
 STORAGE_TASKSET=""
+LOGROUTER_COUNT=0
 
 function usage {
 	echo "Usage"
@@ -38,12 +39,36 @@ function start_servers {
 		DATA=${DIR}/${SERVER_COUNT}/data
 		mkdir -p ${LOG} ${DATA}
 		PORT=$(( $PORT_PREFIX + $SERVER_COUNT ))
-		ZONE=$(( $j % $REPLICATION_COUNT ))
-		$2 ${FDB} -p auto:${PORT} "$KNOBS" -c $3 -d $DATA -L $LOG -C $CLUSTER --locality-zoneid Z-$ZONE --locality-machineid M-$SERVER_COUNT &
+		ZONE=$4-Z-$(( $j % $REPLICATION_COUNT ))
+		$2 ${FDB} -p auto:${PORT} "$KNOBS" -c $3 -d $DATA -L $LOG -C $CLUSTER --datacenter_id=$4 --locality-zoneid $ZONE --locality-machineid M-$SERVER_COUNT &
 		SERVER_COUNT=$(( $SERVER_COUNT + 1 ))
 	done
 }
 
+function create_fileconfig {
+	cat > /tmp/fdbfileconfig.json <<EOF
+{
+	"regions": [{
+		"datacenters": [{
+			"id": "DC1",
+			"priority": 1
+		}, {
+			"id": "DC2",
+			"priority": 0,
+			"satellite": 1,
+			"satellite_logs": 8
+		}],
+		"satellite_redundancy_mode": "one_satellite_double"
+	}, {
+		"datacenters": [{
+			"id": "DC3",
+			"priority": -1
+		}]
+	}]
+}
+EOF
+}
+
 if (( $# < 1 )) ; then
 	echo Wrong number of arguments
 	usage
@@ -82,6 +107,9 @@ while [[ $# -gt 0 ]]; do
 		--replication_count)
 			REPLICATION_COUNT=$2
 			;;
+		--logrouter_count)
+			LOGROUTER_COUNT=$2
+			;;
 	esac
 	shift; shift
 done
@@ -112,12 +140,23 @@ echo $CLUSTER_FILE > $CLUSTER
 
 echo "Starting Cluster: " $CLUSTER_FILE
 
-start_servers $STATELESS_COUNT "$STATELESS_TASKSET" stateless
-start_servers $LOGS_COUNT "$LOGS_TASKSET" log
-start_servers $STORAGE_COUNT "$STORAGE_TASKSET" storage
+start_servers $STATELESS_COUNT "$STATELESS_TASKSET" stateless DC1
+start_servers $LOGS_COUNT "$LOGS_TASKSET" log DC1
+start_servers $STORAGE_COUNT "$STORAGE_TASKSET" storage DC1
 
 CLI="$BUILD/bin/fdbcli -C ${CLUSTER} --exec"
 echo "configure new ssd $replication - stand by"
 
 # sleep 2 seconds to wait for workers to join cluster, then configure database and coordinators
 ( sleep 2 ; $CLI "configure new ssd $replication" ; $CLI "coordinators auto")
+
+if [ $LOGROUTER_COUNT -gt 0 ]; then
+	start_servers $LOGROUTER_COUNT "$STORAGE_TASKSET" router DC3
+	# Same number remote/satellite logs and ss as primary
+	start_servers $LOGS_COUNT "$LOGS_TASKSET" log DC2
+	start_servers $LOGS_COUNT "$LOGS_TASKSET" log DC3
+	start_servers $STORAGE_COUNT "$STORAGE_TASKSET" storage DC3
+	create_fileconfig
+	$CLI "fileconfigure /tmp/fdbfileconfig.json"
+	echo "Wait for data to be fully replicated (Healthy), then issue: $CLI configure usable_regions=2"
+fi