Fix flaky ctest tests (#6310)

* Use localhost cluster for trace_partial_file_suffix_test

This way we get a predictable 127.0.0.1 in the trace file name

* Skip suspend test of pidof is not available

* Avoid writing to closed trace log

calling fdb_network_stop sends a "close" message to the trace thread,
but the network thread might can still be running and sending "flush"
messages to the network thread. This change basically ignores any
flushes that come after a close.

* Ensure unique ports for multi-process tests

bindings/c/test/unit/trace_partial_file_suffix_test.cpp

@@ -72,7 +72,7 @@ int main(int argc, char** argv) {
 
 	// Apparently you need to open a database to initialize logging
 	FDBDatabase* out;
-	fdb_check(fdb_create_database(nullptr, &out));
+	fdb_check(fdb_create_database(argv[1], &out));
 	fdb_database_destroy(out);
 
 	// Eventually there's a new trace file for this test ending in .tmp

bindings/python/tests/fdbcli_tests.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import sys
+import shutil
 import os
 import subprocess
 import logging
@@ -216,6 +217,9 @@ def kill(logger):
 
 @enable_logging()
 def suspend(logger):
+    if not shutil.which("pidof"):
+        logger.debug("Skipping suspend test. Pidof not available")
+        return
     output1 = run_fdbcli_command('suspend')
     lines = output1.split('\n')
     assert len(lines) == 2

flow/FileTraceLogWriter.cpp

@@ -122,6 +122,9 @@ void FileTraceLogWriter::write(const StringRef& str) {
 }
 
 void FileTraceLogWriter::write(const char* str, size_t len) {
+	if (traceFileFD < 0) {
+		return;
+	}
 	auto ptr = str;
 	int remaining = len;
 	bool needsResolve = false;

tests/TestRunner/local_cluster.py

@@ -7,12 +7,24 @@ import sys
 import socket
 
 
-def get_free_port():
+def _get_free_port_internal():
     with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
         s.bind(('0.0.0.0', 0))
         return s.getsockname()[1]
 
 
+_used_ports = set()
+
+
+def get_free_port():
+    global _used_ports
+    port = _get_free_port_internal()
+    while port in _used_ports:
+        port = _get_free_port_internal()
+    _used_ports.add(port)
+    return port
+
+
 class LocalCluster:
     configuration_template = """
 ## foundationdb.conf

tests/TestRunner/tmp_cluster.py

@@ -129,6 +129,10 @@ if __name__ == "__main__":
             break
 
         if errcode:
+            for etc_file in glob.glob(os.path.join(cluster.etc, "*")):
+                print(">>>>>>>>>>>>>>>>>>>> Contents of {}:".format(etc_file))
+                with open(etc_file, "r") as f:
+                    print(f.read())
             for log_file in glob.glob(os.path.join(cluster.log, "*")):
                 print(">>>>>>>>>>>>>>>>>>>> Contents of {}:".format(log_file))
                 with open(log_file, "r") as f: