diff --git a/contrib/Joshua/scripts/bindingTest.sh b/contrib/Joshua/scripts/bindingTest.sh index 8e2fde1f7d..4a0d7c70da 100755 --- a/contrib/Joshua/scripts/bindingTest.sh +++ b/contrib/Joshua/scripts/bindingTest.sh @@ -1,6 +1,5 @@ #!/bin/bash SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -pkill fdbserver ulimit -S -c unlimited unset FDB_NETWORK_OPTION_EXTERNAL_CLIENT_DIRECTORY @@ -8,4 +7,4 @@ WORKDIR="$(pwd)/tmp/$$" if [ ! -d "${WORKDIR}" ] ; then mkdir -p "${WORKDIR}" fi -DEBUGLEVEL=0 DISPLAYERROR=1 RANDOMTEST=1 WORKDIR="${WORKDIR}" FDBSERVERPORT="${PORT_FDBSERVER:-4500}" ${SCRIPTDIR}/bindingTestScript.sh 1 +DEBUGLEVEL=0 DISPLAYERROR=1 RANDOMTEST=1 WORKDIR="${WORKDIR}" ${SCRIPTDIR}/bindingTestScript.sh 1 diff --git a/contrib/Joshua/scripts/bindingTestScript.sh b/contrib/Joshua/scripts/bindingTestScript.sh index 9ef19ab1a6..c9cd5f1a80 100755 --- a/contrib/Joshua/scripts/bindingTestScript.sh +++ b/contrib/Joshua/scripts/bindingTestScript.sh @@ -7,7 +7,7 @@ SCRIPTID="${$}" SAVEONERROR="${SAVEONERROR:-1}" PYTHONDIR="${BINDIR}/tests/python" testScript="${BINDIR}/tests/bindingtester/run_binding_tester.sh" -VERSION="1.6" +VERSION="1.8" source ${SCRIPTDIR}/localClusterStart.sh @@ -23,19 +23,22 @@ cycles="${1}" if [ "${DEBUGLEVEL}" -gt 0 ] then - echo "Work dir: ${WORKDIR}" - echo "Bin dir: ${BINDIR}" - echo "Log dir: ${LOGDIR}" - echo "Python path: ${PYTHONDIR}" - echo "Lib dir: ${LIBDIR}" - echo "Server port: ${FDBSERVERPORT}" - echo "Script Id: ${SCRIPTID}" - echo "Version: ${VERSION}" + echo "Work dir: ${WORKDIR}" + echo "Bin dir: ${BINDIR}" + echo "Log dir: ${LOGDIR}" + echo "Python path: ${PYTHONDIR}" + echo "Lib dir: ${LIBDIR}" + echo "Cluster String: ${CLUSTERSTRING}" + echo "Script Id: ${SCRIPTID}" + echo "Version: ${VERSION}" fi # Begin the cluster using the logic in localClusterStart.sh. startCluster +# Stop the cluster on exit +trap "stopCluster" EXIT + # Display user message if [ "${status}" -ne 0 ]; then : @@ -58,8 +61,8 @@ fi # Display directory and log information, if an error occurred if [ "${status}" -ne 0 ] then - ls "${WORKDIR}" > "${LOGDIR}/dir.log" - ps -eafw > "${LOGDIR}/process-preclean.log" + ls "${WORKDIR}" &> "${LOGDIR}/dir.log" + ps -eafwH &> "${LOGDIR}/process-preclean.log" if [ -f "${FDBCONF}" ]; then cp -f "${FDBCONF}" "${LOGDIR}/" fi @@ -71,10 +74,15 @@ fi # Save debug information files, environment, and log information, if an error occurred if [ "${status}" -ne 0 ] && [ "${SAVEONERROR}" -gt 0 ]; then - ps -eafw > "${LOGDIR}/process-exit.log" - netstat -na > "${LOGDIR}/netstat.log" - df -h > "${LOGDIR}/disk.log" - env > "${LOGDIR}/env.log" + ps -eafwH &> "${LOGDIR}/process-exit.log" + netstat -na &> "${LOGDIR}/netstat.log" + df -h &> "${LOGDIR}/disk.log" + env &> "${LOGDIR}/env.log" +fi + +# Stop the cluster +if stopCluster; then + unset FDBSERVERID fi exit "${status}" diff --git a/contrib/Joshua/scripts/localClusterStart.sh b/contrib/Joshua/scripts/localClusterStart.sh index 3ba4cb9dcb..c79946053f 100644 --- a/contrib/Joshua/scripts/localClusterStart.sh +++ b/contrib/Joshua/scripts/localClusterStart.sh @@ -5,14 +5,26 @@ WORKDIR="${WORKDIR:-${SCRIPTDIR}/tmp/fdb.work}" LOGDIR="${WORKDIR}/log" ETCDIR="${WORKDIR}/etc" BINDIR="${BINDIR:-${SCRIPTDIR}}" -FDBSERVERPORT="${FDBSERVERPORT:-4500}" +FDBPORTSTART="${FDBPORTSTART:-4000}" +SERVERCHECKS="${SERVERCHECKS:-10}" +CONFIGUREWAIT="${CONFIGUREWAIT:-240}" FDBCONF="${ETCDIR}/fdb.cluster" LOGFILE="${LOGFILE:-${LOGDIR}/startcluster.log}" +AUDITCLUSTER="${AUDITCLUSTER:-0}" +AUDITLOG="${AUDITLOG:-/tmp/audit-cluster.log}" # Initialize the variables status=0 messagetime=0 messagecount=0 +let index2="${RANDOM} % 256" +let index3="${RANDOM} % 256" +let index4="(${RANDOM} % 255) + 1" +let FDBPORT="(${RANDOM} % 1000) + ${FDBPORTSTART}" +# Define a random ip address and port on localhost +IPADDRESS="127.${index2}.${index3}.${index4}" +CLUSTERSTRING="${IPADDRESS}:${FDBPORT}" + function log { @@ -92,29 +104,32 @@ function displayMessage } # Create the directories used by the server. -function createDirectories { +function createDirectories +{ + local status=0 + # Display user message if ! displayMessage "Creating directories" then echo 'Failed to display user message' let status="${status} + 1" - + elif ! mkdir -p "${LOGDIR}" "${ETCDIR}" then log "Failed to create directories" let status="${status} + 1" - + # Display user message elif ! displayMessage "Setting file permissions" then log 'Failed to display user message' let status="${status} + 1" - + elif ! chmod 755 "${BINDIR}/fdbserver" "${BINDIR}/fdbcli" then log "Failed to set file permissions" let status="${status} + 1" - + else while read filepath do @@ -137,7 +152,10 @@ function createDirectories { } # Create a cluster file for the local cluster. -function createClusterFile { +function createClusterFile +{ + local status=0 + if [ "${status}" -ne 0 ]; then : # Display user message @@ -148,7 +166,7 @@ function createClusterFile { else description=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8) random_str=$(LC_CTYPE=C tr -dc A-Za-z0-9 < /dev/urandom 2> /dev/null | head -c 8) - echo "$description:$random_str@127.0.0.1:${FDBSERVERPORT}" > "${FDBCONF}" + echo "${description}:${random_str}@${CLUSTERSTRING}" > "${FDBCONF}" fi if [ "${status}" -ne 0 ]; then @@ -161,8 +179,51 @@ function createClusterFile { return ${status} } +# Stop the Cluster from running. +function stopCluster +{ + local status=0 + + # Add an audit entry, if enabled + if [ "${AUDITCLUSTER}" -gt 0 ]; then + printf '%-15s (%6s) Stopping cluster %-20s (%6s): %s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" "${CLUSTERSTRING}" "${FDBSERVERID}" >> "${AUDITLOG}" + fi + if [ -z "${FDBSERVERID}" ]; then + log 'FDB Server process is not defined' + let status="${status} + 1" + elif ! kill -0 "${FDBSERVERID}"; then + log "Failed to locate FDB Server process (${FDBSERVERID})" + let status="${status} + 1" + elif "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec "kill; kill ${CLUSTERSTRING}; sleep 3" --timeout 120 &>> "${LOGDIR}/fdbcli-kill.log" + then + # Ensure that process is dead + if ! kill -0 "${FDBSERVERID}" 2> /dev/null; then + log "Killed cluster (${FDBSERVERID}) via cli" + elif ! kill -9 "${FDBSERVERID}"; then + log "Failed to kill FDB Server process (${FDBSERVERID}) via cli or kill command" + let status="${status} + 1" + else + log "Forcibly killed FDB Server process (${FDBSERVERID}) since cli failed" + fi + elif ! kill -9 "${FDBSERVERID}"; then + log "Failed to forcibly kill FDB Server process (${FDBSERVERID})" + let status="${status} + 1" + else + log "Forcibly killed FDB Server process (${FDBSERVERID})" + fi + return "${status}" +} + # Start the server running. -function startFdbServer { +function startFdbServer +{ + local status=0 + + # Add an audit entry, if enabled + if [ "${AUDITCLUSTER}" -gt 0 ]; then + printf '%-15s (%6s) Starting cluster %-20s\n' "$(date +'%Y-%m-%d %H:%M:%S')" "${$}" "${CLUSTERSTRING}" >> "${AUDITLOG}" + fi + if [ "${status}" -ne 0 ]; then : elif ! displayMessage "Starting Fdb Server" @@ -170,25 +231,29 @@ function startFdbServer { log 'Failed to display user message' let status="${status} + 1" - elif ! "${BINDIR}/fdbserver" -C "${FDBCONF}" -p "auto:${FDBSERVERPORT}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/$$" &> "${LOGDIR}/fdbserver.log" & + elif ! "${BINDIR}/fdbserver" --knob_disable_posix_kernel_aio=1 -C "${FDBCONF}" -p "${CLUSTERSTRING}" -L "${LOGDIR}" -d "${WORKDIR}/fdb/${$}" &> "${LOGDIR}/fdbserver.log" & then log "Failed to start FDB Server" - # Maybe the server is already running - FDBSERVERID="$(pidof fdbserver)" let status="${status} + 1" else FDBSERVERID="${!}" fi - if ! kill -0 ${FDBSERVERID} ; then - log "FDB Server start failed." + if [ -z "${FDBSERVERID}" ]; then + log "FDB Server start failed because no process" + let status="${status} + 1" + elif ! kill -0 "${FDBSERVERID}" ; then + log "FDB Server start failed because process terminated unexpectedly" let status="${status} + 1" fi return ${status} } -function getStatus { +function getStatus +{ + local status=0 + if [ "${status}" -ne 0 ]; then : elif ! date &>> "${LOGDIR}/fdbclient.log" @@ -209,35 +274,41 @@ function getStatus { } # Verify that the cluster is available. -function verifyAvailable { +function verifyAvailable +{ + local status=0 + + if [ -z "${FDBSERVERID}" ]; then + log "FDB Server process is not defined." + let status="${status} + 1" # Verify that the server is running. - if ! kill -0 "${FDBSERVERID}" + elif ! kill -0 "${FDBSERVERID}" then log "FDB server process (${FDBSERVERID}) is not running" let status="${status} + 1" - return 1 - # Display user message. elif ! displayMessage "Checking cluster availability" then log 'Failed to display user message' let status="${status} + 1" - return 1 - # Determine if status json says the database is available. else - avail=`"${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout 10 2> /dev/null | grep -E '"database_available"|"available"' | grep 'true'` + avail=`"${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'status json' --timeout "${SERVERCHECKS}" 2> /dev/null | grep -E '"database_available"|"available"' | grep 'true'` log "Avail value: ${avail}" "${DEBUGLEVEL}" if [[ -n "${avail}" ]] ; then - return 0 + : else - return 1 + let status="${status} + 1" fi fi + return "${status}" } # Configure the database on the server. -function createDatabase { +function createDatabase +{ + local status=0 + if [ "${status}" -ne 0 ]; then : # Ensure that the server is running @@ -262,7 +333,7 @@ function createDatabase { # Configure the database. else - "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'configure new single memory; status' --timeout 240 --log --log-dir "${LOGDIR}" &>> "${LOGDIR}/fdbclient.log" + "${BINDIR}/fdbcli" -C "${FDBCONF}" --exec 'configure new single memory; status' --timeout "${CONFIGUREWAIT}" --log --log-dir "${LOGDIR}" &>> "${LOGDIR}/fdbclient.log" if ! displayMessage "Checking if config succeeded" then @@ -270,7 +341,7 @@ function createDatabase { fi iteration=0 - while [[ "${iteration}" -lt 10 ]] && ! verifyAvailable + while [[ "${iteration}" -lt "${SERVERCHECKS}" ]] && ! verifyAvailable do log "Database not created (iteration ${iteration})." let iteration="${iteration} + 1" @@ -290,7 +361,10 @@ function createDatabase { } # Begin the local cluster from scratch. -function startCluster { +function startCluster +{ + local status=0 + if [ "${status}" -ne 0 ]; then : elif ! createDirectories