From 62eeefcc8a59b4fee1f43d9a3c8d90028d1c6c15 Mon Sep 17 00:00:00 2001 From: "A.J. Beamon" Date: Wed, 20 Jun 2018 12:09:28 -0700 Subject: [PATCH] Update status schema with new field --- documentation/StatusSchema.json | 1 + tests/fast/SidebandWithStatus.txt | 2 +- tests/rare/LargeApiCorrectnessStatus.txt | 2 +- tests/slow/DDBalanceAndRemoveStatus.txt | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/documentation/StatusSchema.json b/documentation/StatusSchema.json index f309fefd7e..490ebc043e 100644 --- a/documentation/StatusSchema.json +++ b/documentation/StatusSchema.json @@ -75,6 +75,7 @@ "memory":{ "available_bytes":0, "limit_bytes":0, + "unused_allocated_memory":0, "used_bytes":0 }, "messages":[ diff --git a/tests/fast/SidebandWithStatus.txt b/tests/fast/SidebandWithStatus.txt index e1c5a7c796..94f4616d9f 100644 --- a/tests/fast/SidebandWithStatus.txt +++ b/tests/fast/SidebandWithStatus.txt @@ -5,7 +5,7 @@ testTitle=CloggedCausalConsistencyTest testName=Status testDuration=30.0 - schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"datacenter_version_difference":0,"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_version_lag":12341234,"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","data_version":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"queue_disk_available_bytes":12341234,"kvstore_available_bytes":12341234,"queue_disk_total_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"used_bytes":0,"limit_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"incompatible_connections":[],"full_replication":true,"database_locked":false,"generation":2,"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"database_available":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0,"counter":0,"roughness":0.0}},"keys":{"read":{"hz":0.0,"counter":0,"roughness":0.0}},"bytes":{"read":{"hz":0.0,"counter":0,"roughness":0.0},"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","configuration":{"resolvers":1,"regions":[{"satellite_redundancy_mode":"one_satellite_single","satellite_anti_quorum":0,"satellite_usable_dcs":1,"datacenters":[{"priority":1,"satellite":1,"id":"mr"}],"satellite_log_policy":"(zoneid^3x1)","satellite_log_replicas":1,"satellite_logs":2}],"remote_logs":5,"auto_logs":3,"logs":2,"log_anti_quorum":0,"storage_replicas":1,"log_replicas":2,"remote_redundancy_mode":"remote_single","storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"log_replication_policy":"(zoneid^3x1)","storage_replication_policy":"(zoneid^3x1)","remote_log_replicas":3,"excluded_servers":[{"address":"10.0.4.1"}],"auto_proxies":3,"proxies":5,"usable_regions":1,"redundancy_mode":"single","auto_resolvers":1},"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02},"machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"old_logs":[{"satellite_log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"satellite_log_write_anti_quorum":0,"remote_log_fault_tolerance":2,"log_fault_tolerance":2,"log_write_anti_quorum":0,"satellite_log_replication_factor":3,"remote_log_replication_factor":3,"log_replication_factor":3}]},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}} + schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"processes":{"$map":{"version":"3.0.0","machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","locality":{"$map":"value"},"class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"input_bytes":{"hz":0,"counter":0,"roughness":0},"stored_bytes":12341234,"kvstore_used_bytes":12341234,"kvstore_available_bytes":12341234,"kvstore_free_bytes":12341234,"kvstore_total_bytes":12341234,"durable_bytes":{"hz":0,"counter":0,"roughness":0},"queue_disk_used_bytes":12341234,"queue_disk_available_bytes":12341234,"queue_disk_free_bytes":12341234,"queue_disk_total_bytes":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"data_version":12341234,"data_version_lag":12341234,"id":"eb84471d68c12d1d26f692a50000003f","finished_queries":{"hz":0,"counter":0,"roughness":0}}],"command_line":"-r simulation","memory":{"available_bytes":0,"limit_bytes":0,"unused_allocated_memory":0,"used_bytes":0},"messages":[{"time":12345.12312,"type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"","description":"abc"}],"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","excluded":false,"address":"1.2.3.4:1234","disk":{"free_bytes":3451233456234,"reads":{"hz":0,"counter":0,"sectors":0},"busy":0,"writes":{"hz":0,"counter":0,"sectors":0},"total_bytes":123412341234},"uptime_seconds":1234.2345,"cpu":{"usage_cores":0},"network":{"current_connections":0,"connections_established":{"hz":0},"connections_closed":{"hz":0},"connection_errors":{"hz":0},"megabits_sent":{"hz":0},"megabits_received":{"hz":0}}}},"old_logs":[{"logs":[{"id":"7f8d623d0cb9966e","healthy":true,"address":"1.2.3.4:1234"}],"log_replication_factor":3,"log_write_anti_quorum":0,"log_fault_tolerance":2,"remote_log_replication_factor":3,"remote_log_fault_tolerance":2,"satellite_log_replication_factor":3,"satellite_log_write_anti_quorum":0,"satellite_log_fault_tolerance":2}],"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"qos":{"worst_queue_bytes_log_server":460,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]},"description":"The database is not being saturated by the workload."},"transactions_per_second_limit":0,"released_transactions_per_second":0,"limiting_queue_bytes_storage_server":0,"worst_queue_bytes_storage_server":0,"limiting_version_lag_storage_server":0,"worst_version_lag_storage_server":0},"incompatible_connections":[],"datacenter_version_difference":0,"database_available":true,"database_locked":false,"generation":2,"latency_probe":{"read_seconds":7,"immediate_priority_transaction_start_seconds":0,"batch_priority_transaction_start_seconds":0,"transaction_start_seconds":0,"commit_seconds":0.02},"clients":{"count":1,"supported_versions":[{"client_version":"3.0.0","connected_clients":[{"address":"127.0.0.1:9898","log_group":"default"}],"count":1,"protocol_version":"fdb00a400050001","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d"}]},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"recovery_state":{"required_resolvers":1,"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"required_logs":3,"missing_logs":"7f8d623d0cb9966e","description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0,"counter":0,"roughness":0},"reads":{"hz":0,"counter":0,"roughness":0}},"bytes":{"written":{"hz":0,"counter":0,"roughness":0},"read":{"hz":0,"counter":0,"roughness":0}},"keys":{"read":{"hz":0,"counter":0,"roughness":0}},"transactions":{"started":{"hz":0,"counter":0,"roughness":0},"conflicted":{"hz":0,"counter":0,"roughness":0},"committed":{"hz":0,"counter":0,"roughness":0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","full_replication":true,"configuration":{"log_anti_quorum":0,"log_replicas":2,"log_replication_policy":"(zoneid^3x1)","redundancy_mode":"single","regions":[{"datacenters":[{"id":"mr","priority":1,"satellite":1}],"satellite_redundancy_mode":"one_satellite_single","satellite_log_replicas":1,"satellite_usable_dcs":1,"satellite_anti_quorum":0,"satellite_log_policy":"(zoneid^3x1)","satellite_logs":2}],"remote_redundancy_mode":"remote_single","remote_log_replicas":3,"remote_logs":5,"usable_regions":1,"storage_replicas":1,"resolvers":1,"storage_replication_policy":"(zoneid^3x1)","logs":2,"storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"excluded_servers":[{"address":"10.0.4.1"}],"auto_proxies":3,"auto_resolvers":1,"auto_logs":3,"proxies":5},"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"min_replicas_remaining":0,"name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"description":""},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"partitions_count":2,"moving_data":{"total_written_bytes":0,"in_flight_bytes":0,"in_queue_bytes":0},"least_operating_space_bytes_storage_server":0,"max_machine_failures_without_losing_data":0},"machines":{"$map":{"network":{"megabits_sent":{"hz":0},"megabits_received":{"hz":0},"tcp_segments_retransmitted":{"hz":0}},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","locality":{"$map":"value"},"cpu":{"logical_core_utilization":0.4}}}},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"database_status":{"available":true,"healthy":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true}}} testName=RandomClogging testDuration=30.0 diff --git a/tests/rare/LargeApiCorrectnessStatus.txt b/tests/rare/LargeApiCorrectnessStatus.txt index 252efc5f05..7db5388026 100644 --- a/tests/rare/LargeApiCorrectnessStatus.txt +++ b/tests/rare/LargeApiCorrectnessStatus.txt @@ -24,4 +24,4 @@ testTitle=ApiCorrectnessTest testName=Status testDuration=30.0 - schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"datacenter_version_difference":0,"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_version_lag":12341234,"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","data_version":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"queue_disk_available_bytes":12341234,"kvstore_available_bytes":12341234,"queue_disk_total_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"used_bytes":0,"limit_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"incompatible_connections":[],"full_replication":true,"database_locked":false,"generation":2,"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"database_available":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0,"counter":0,"roughness":0.0}},"keys":{"read":{"hz":0.0,"counter":0,"roughness":0.0}},"bytes":{"read":{"hz":0.0,"counter":0,"roughness":0.0},"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","configuration":{"resolvers":1,"regions":[{"satellite_redundancy_mode":"one_satellite_single","satellite_anti_quorum":0,"satellite_usable_dcs":1,"datacenters":[{"priority":1,"satellite":1,"id":"mr"}],"satellite_log_policy":"(zoneid^3x1)","satellite_log_replicas":1,"satellite_logs":2}],"remote_logs":5,"auto_logs":3,"logs":2,"log_anti_quorum":0,"storage_replicas":1,"log_replicas":2,"remote_redundancy_mode":"remote_single","storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"log_replication_policy":"(zoneid^3x1)","storage_replication_policy":"(zoneid^3x1)","remote_log_replicas":3,"excluded_servers":[{"address":"10.0.4.1"}],"auto_proxies":3,"proxies":5,"usable_regions":1,"redundancy_mode":"single","auto_resolvers":1},"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02},"machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"old_logs":[{"satellite_log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"satellite_log_write_anti_quorum":0,"remote_log_fault_tolerance":2,"log_fault_tolerance":2,"log_write_anti_quorum":0,"satellite_log_replication_factor":3,"remote_log_replication_factor":3,"log_replication_factor":3}]},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}} + schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"processes":{"$map":{"version":"3.0.0","machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","locality":{"$map":"value"},"class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"input_bytes":{"hz":0,"counter":0,"roughness":0},"stored_bytes":12341234,"kvstore_used_bytes":12341234,"kvstore_available_bytes":12341234,"kvstore_free_bytes":12341234,"kvstore_total_bytes":12341234,"durable_bytes":{"hz":0,"counter":0,"roughness":0},"queue_disk_used_bytes":12341234,"queue_disk_available_bytes":12341234,"queue_disk_free_bytes":12341234,"queue_disk_total_bytes":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"data_version":12341234,"data_version_lag":12341234,"id":"eb84471d68c12d1d26f692a50000003f","finished_queries":{"hz":0,"counter":0,"roughness":0}}],"command_line":"-r simulation","memory":{"available_bytes":0,"limit_bytes":0,"unused_allocated_memory":0,"used_bytes":0},"messages":[{"time":12345.12312,"type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"","description":"abc"}],"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","excluded":false,"address":"1.2.3.4:1234","disk":{"free_bytes":3451233456234,"reads":{"hz":0,"counter":0,"sectors":0},"busy":0,"writes":{"hz":0,"counter":0,"sectors":0},"total_bytes":123412341234},"uptime_seconds":1234.2345,"cpu":{"usage_cores":0},"network":{"current_connections":0,"connections_established":{"hz":0},"connections_closed":{"hz":0},"connection_errors":{"hz":0},"megabits_sent":{"hz":0},"megabits_received":{"hz":0}}}},"old_logs":[{"logs":[{"id":"7f8d623d0cb9966e","healthy":true,"address":"1.2.3.4:1234"}],"log_replication_factor":3,"log_write_anti_quorum":0,"log_fault_tolerance":2,"remote_log_replication_factor":3,"remote_log_fault_tolerance":2,"satellite_log_replication_factor":3,"satellite_log_write_anti_quorum":0,"satellite_log_fault_tolerance":2}],"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"qos":{"worst_queue_bytes_log_server":460,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]},"description":"The database is not being saturated by the workload."},"transactions_per_second_limit":0,"released_transactions_per_second":0,"limiting_queue_bytes_storage_server":0,"worst_queue_bytes_storage_server":0,"limiting_version_lag_storage_server":0,"worst_version_lag_storage_server":0},"incompatible_connections":[],"datacenter_version_difference":0,"database_available":true,"database_locked":false,"generation":2,"latency_probe":{"read_seconds":7,"immediate_priority_transaction_start_seconds":0,"batch_priority_transaction_start_seconds":0,"transaction_start_seconds":0,"commit_seconds":0.02},"clients":{"count":1,"supported_versions":[{"client_version":"3.0.0","connected_clients":[{"address":"127.0.0.1:9898","log_group":"default"}],"count":1,"protocol_version":"fdb00a400050001","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d"}]},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"recovery_state":{"required_resolvers":1,"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"required_logs":3,"missing_logs":"7f8d623d0cb9966e","description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0,"counter":0,"roughness":0},"reads":{"hz":0,"counter":0,"roughness":0}},"bytes":{"written":{"hz":0,"counter":0,"roughness":0},"read":{"hz":0,"counter":0,"roughness":0}},"keys":{"read":{"hz":0,"counter":0,"roughness":0}},"transactions":{"started":{"hz":0,"counter":0,"roughness":0},"conflicted":{"hz":0,"counter":0,"roughness":0},"committed":{"hz":0,"counter":0,"roughness":0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","full_replication":true,"configuration":{"log_anti_quorum":0,"log_replicas":2,"log_replication_policy":"(zoneid^3x1)","redundancy_mode":"single","regions":[{"datacenters":[{"id":"mr","priority":1,"satellite":1}],"satellite_redundancy_mode":"one_satellite_single","satellite_log_replicas":1,"satellite_usable_dcs":1,"satellite_anti_quorum":0,"satellite_log_policy":"(zoneid^3x1)","satellite_logs":2}],"remote_redundancy_mode":"remote_single","remote_log_replicas":3,"remote_logs":5,"usable_regions":1,"storage_replicas":1,"resolvers":1,"storage_replication_policy":"(zoneid^3x1)","logs":2,"storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"excluded_servers":[{"address":"10.0.4.1"}],"auto_proxies":3,"auto_resolvers":1,"auto_logs":3,"proxies":5},"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"min_replicas_remaining":0,"name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"description":""},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"partitions_count":2,"moving_data":{"total_written_bytes":0,"in_flight_bytes":0,"in_queue_bytes":0},"least_operating_space_bytes_storage_server":0,"max_machine_failures_without_losing_data":0},"machines":{"$map":{"network":{"megabits_sent":{"hz":0},"megabits_received":{"hz":0},"tcp_segments_retransmitted":{"hz":0}},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","locality":{"$map":"value"},"cpu":{"logical_core_utilization":0.4}}}},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"database_status":{"available":true,"healthy":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true}}} diff --git a/tests/slow/DDBalanceAndRemoveStatus.txt b/tests/slow/DDBalanceAndRemoveStatus.txt index fb48f4b154..7e07285c3c 100644 --- a/tests/slow/DDBalanceAndRemoveStatus.txt +++ b/tests/slow/DDBalanceAndRemoveStatus.txt @@ -43,4 +43,4 @@ testTitle=DDBalance_test testName=Status testDuration=30.0 - schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"datacenter_version_difference":0,"processes":{"$map":{"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"data_version_lag":12341234,"input_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"kvstore_used_bytes":12341234,"stored_bytes":12341234,"kvstore_free_bytes":12341234,"durable_bytes":{"hz":0.0,"counter":0,"roughness":0.0},"id":"eb84471d68c12d1d26f692a50000003f","data_version":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"queue_disk_available_bytes":12341234,"kvstore_available_bytes":12341234,"queue_disk_total_bytes":12341234,"queue_disk_used_bytes":12341234,"queue_disk_free_bytes":12341234,"kvstore_total_bytes":12341234,"finished_queries":{"hz":0.0,"counter":0,"roughness":0.0}}],"locality":{"$map":"value"},"messages":[{"description":"abc","type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"","time":12345.12312}],"address":"1.2.3.4:1234","command_line":"-r simulation","disk":{"free_bytes":3451233456234,"reads":{"hz":0.0,"counter":0,"sectors":0},"busy":0.0,"writes":{"hz":0.0,"counter":0,"sectors":0},"total_bytes":123412341234},"version":"3.0.0","excluded":false,"memory":{"available_bytes":0,"used_bytes":0,"limit_bytes":0},"machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","uptime_seconds":1234.2345,"cpu":{"usage_cores":0.0},"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"connections_closed":{"hz":0.0},"connection_errors":{"hz":0.0},"current_connections":0,"connections_established":{"hz":0.0}}}},"clients":{"count":1,"supported_versions":[{"count":1,"protocol_version":"fdb00a400050001","client_version":"3.0.0","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d","connected_clients":[{"log_group":"default","address":"127.0.0.1:9898"}]}]},"qos":{"limiting_version_lag_storage_server":0,"released_transactions_per_second":0,"transactions_per_second_limit":0,"limiting_queue_bytes_storage_server":0,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","description":"The database is not being saturated by the workload.","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]}},"worst_version_lag_storage_server":0,"worst_queue_bytes_log_server":460,"worst_queue_bytes_storage_server":0},"incompatible_connections":[],"full_replication":true,"database_locked":false,"generation":2,"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"description":"","name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"min_replicas_remaining":0},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"max_machine_failures_without_losing_data":0,"moving_data":{"in_queue_bytes":0,"total_written_bytes":0,"in_flight_bytes":0},"least_operating_space_bytes_storage_server":0,"partitions_count":2},"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"database_available":true,"recovery_state":{"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"missing_logs":"7f8d623d0cb9966e","required_resolvers":1,"required_logs":3,"description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0.0,"counter":0,"roughness":0.0},"reads":{"hz":0.0,"counter":0,"roughness":0.0}},"keys":{"read":{"hz":0.0,"counter":0,"roughness":0.0}},"bytes":{"read":{"hz":0.0,"counter":0,"roughness":0.0},"written":{"hz":0.0,"counter":0,"roughness":0.0}},"transactions":{"started":{"hz":0.0,"counter":0,"roughness":0.0},"conflicted":{"hz":0.0,"counter":0,"roughness":0.0},"committed":{"hz":0.0,"counter":0,"roughness":0.0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","configuration":{"resolvers":1,"regions":[{"satellite_redundancy_mode":"one_satellite_single","satellite_anti_quorum":0,"satellite_usable_dcs":1,"datacenters":[{"priority":1,"satellite":1,"id":"mr"}],"satellite_log_policy":"(zoneid^3x1)","satellite_log_replicas":1,"satellite_logs":2}],"remote_logs":5,"auto_logs":3,"logs":2,"log_anti_quorum":0,"storage_replicas":1,"log_replicas":2,"remote_redundancy_mode":"remote_single","storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"log_replication_policy":"(zoneid^3x1)","storage_replication_policy":"(zoneid^3x1)","remote_log_replicas":3,"excluded_servers":[{"address":"10.0.4.1"}],"auto_proxies":3,"proxies":5,"usable_regions":1,"redundancy_mode":"single","auto_resolvers":1},"latency_probe":{"immediate_priority_transaction_start_seconds":0.0,"transaction_start_seconds":0.0,"batch_priority_transaction_start_seconds":0.0,"read_seconds":7,"commit_seconds":0.02},"machines":{"$map":{"network":{"megabits_sent":{"hz":0.0},"megabits_received":{"hz":0.0},"tcp_segments_retransmitted":{"hz":0.0}},"locality":{"$map":"value"},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","cpu":{"logical_core_utilization":0.4}}},"old_logs":[{"satellite_log_fault_tolerance":2,"logs":[{"healthy":true,"id":"7f8d623d0cb9966e","address":"1.2.3.4:1234"}],"satellite_log_write_anti_quorum":0,"remote_log_fault_tolerance":2,"log_fault_tolerance":2,"log_write_anti_quorum":0,"satellite_log_replication_factor":3,"remote_log_replication_factor":3,"log_replication_factor":3}]},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"database_status":{"available":true,"healthy":true}}} + schema={"cluster":{"layers":{"_valid":true,"_error":"some error description"},"processes":{"$map":{"version":"3.0.0","machine_id":"0ccb4e0feddb5583010f6b77d9d10ece","locality":{"$map":"value"},"class_source":{"$enum":["command_line","configure_auto","set_class"]},"class_type":{"$enum":["unset","storage","transaction","resolution","proxy","master","test"]},"roles":[{"query_queue_max":0,"input_bytes":{"hz":0,"counter":0,"roughness":0},"stored_bytes":12341234,"kvstore_used_bytes":12341234,"kvstore_available_bytes":12341234,"kvstore_free_bytes":12341234,"kvstore_total_bytes":12341234,"durable_bytes":{"hz":0,"counter":0,"roughness":0},"queue_disk_used_bytes":12341234,"queue_disk_available_bytes":12341234,"queue_disk_free_bytes":12341234,"queue_disk_total_bytes":12341234,"role":{"$enum":["master","proxy","log","storage","resolver","cluster_controller"]},"data_version":12341234,"data_version_lag":12341234,"id":"eb84471d68c12d1d26f692a50000003f","finished_queries":{"hz":0,"counter":0,"roughness":0}}],"command_line":"-r simulation","memory":{"available_bytes":0,"limit_bytes":0,"unused_allocated_memory":0,"used_bytes":0},"messages":[{"time":12345.12312,"type":"x","name":{"$enum":["file_open_error","incorrect_cluster_file_contents","process_error","io_error","io_timeout","platform_error","storage_server_lagging","(other FDB error messages)"]},"raw_log_message":"","description":"abc"}],"fault_domain":"0ccb4e0fdbdb5583010f6b77d9d10ece","excluded":false,"address":"1.2.3.4:1234","disk":{"free_bytes":3451233456234,"reads":{"hz":0,"counter":0,"sectors":0},"busy":0,"writes":{"hz":0,"counter":0,"sectors":0},"total_bytes":123412341234},"uptime_seconds":1234.2345,"cpu":{"usage_cores":0},"network":{"current_connections":0,"connections_established":{"hz":0},"connections_closed":{"hz":0},"connection_errors":{"hz":0},"megabits_sent":{"hz":0},"megabits_received":{"hz":0}}}},"old_logs":[{"logs":[{"id":"7f8d623d0cb9966e","healthy":true,"address":"1.2.3.4:1234"}],"log_replication_factor":3,"log_write_anti_quorum":0,"log_fault_tolerance":2,"remote_log_replication_factor":3,"remote_log_fault_tolerance":2,"satellite_log_replication_factor":3,"satellite_log_write_anti_quorum":0,"satellite_log_fault_tolerance":2}],"fault_tolerance":{"max_machine_failures_without_losing_availability":0,"max_machine_failures_without_losing_data":0},"qos":{"worst_queue_bytes_log_server":460,"performance_limited_by":{"reason_server_id":"7f8d623d0cb9966e","reason_id":0,"name":{"$enum":["workload","storage_server_write_queue_size","storage_server_write_bandwidth_mvcc","storage_server_readable_behind","log_server_mvcc_write_bandwidth","log_server_write_queue","storage_server_min_free_space","storage_server_min_free_space_ratio","log_server_min_free_space","log_server_min_free_space_ratio"]},"description":"The database is not being saturated by the workload."},"transactions_per_second_limit":0,"released_transactions_per_second":0,"limiting_queue_bytes_storage_server":0,"worst_queue_bytes_storage_server":0,"limiting_version_lag_storage_server":0,"worst_version_lag_storage_server":0},"incompatible_connections":[],"datacenter_version_difference":0,"database_available":true,"database_locked":false,"generation":2,"latency_probe":{"read_seconds":7,"immediate_priority_transaction_start_seconds":0,"batch_priority_transaction_start_seconds":0,"transaction_start_seconds":0,"commit_seconds":0.02},"clients":{"count":1,"supported_versions":[{"client_version":"3.0.0","connected_clients":[{"address":"127.0.0.1:9898","log_group":"default"}],"count":1,"protocol_version":"fdb00a400050001","source_version":"9430e1127b4991cbc5ab2b17f41cfffa5de07e9d"}]},"messages":[{"reasons":[{"description":"Blah."}],"unreachable_processes":[{"address":"1.2.3.4:1234"}],"name":{"$enum":["unreachable_master_worker","unreadable_configuration","full_replication_timeout","client_issues","unreachable_processes","immediate_priority_transaction_start_probe_timeout","batch_priority_transaction_start_probe_timeout","transaction_start_probe_timeout","read_probe_timeout","commit_probe_timeout","storage_servers_error","status_incomplete","layer_status_incomplete","database_availability_timeout"]},"issues":[{"name":{"$enum":["incorrect_cluster_file_contents"]},"description":"Cluster file contents do not match current cluster connection string. Verify cluster file is writable and has not been overwritten externally."}],"description":"abc"}],"recovery_state":{"required_resolvers":1,"required_proxies":1,"name":{"$enum":["reading_coordinated_state","locking_coordinated_state","locking_old_transaction_servers","reading_transaction_system_state","configuration_missing","configuration_never_created","configuration_invalid","recruiting_transaction_servers","initializing_transaction_servers","recovery_transaction","writing_coordinated_state","fully_recovered"]},"required_logs":3,"missing_logs":"7f8d623d0cb9966e","description":"Recovery complete."},"workload":{"operations":{"writes":{"hz":0,"counter":0,"roughness":0},"reads":{"hz":0,"counter":0,"roughness":0}},"bytes":{"written":{"hz":0,"counter":0,"roughness":0},"read":{"hz":0,"counter":0,"roughness":0}},"keys":{"read":{"hz":0,"counter":0,"roughness":0}},"transactions":{"started":{"hz":0,"counter":0,"roughness":0},"conflicted":{"hz":0,"counter":0,"roughness":0},"committed":{"hz":0,"counter":0,"roughness":0}}},"cluster_controller_timestamp":1415650089,"protocol_version":"fdb00a400050001","full_replication":true,"configuration":{"log_anti_quorum":0,"log_replicas":2,"log_replication_policy":"(zoneid^3x1)","redundancy_mode":"single","regions":[{"datacenters":[{"id":"mr","priority":1,"satellite":1}],"satellite_redundancy_mode":"one_satellite_single","satellite_log_replicas":1,"satellite_usable_dcs":1,"satellite_anti_quorum":0,"satellite_log_policy":"(zoneid^3x1)","satellite_logs":2}],"remote_redundancy_mode":"remote_single","remote_log_replicas":3,"remote_logs":5,"usable_regions":1,"storage_replicas":1,"resolvers":1,"storage_replication_policy":"(zoneid^3x1)","logs":2,"storage_engine":{"$enum":["ssd","ssd-1","ssd-2","memory","custom"]},"coordinators_count":1,"excluded_servers":[{"address":"10.0.4.1"}],"auto_proxies":3,"auto_resolvers":1,"auto_logs":3,"proxies":5},"data":{"least_operating_space_bytes_log_server":0,"average_partition_size_bytes":0,"state":{"healthy":true,"min_replicas_remaining":0,"name":{"$enum":["initializing","missing_data","healing","healthy_repartitioning","healthy_removing_server","healthy_rebalancing","healthy"]},"description":""},"least_operating_space_ratio_storage_server":0.1,"max_machine_failures_without_losing_availability":0,"total_disk_used_bytes":0,"total_kv_size_bytes":0,"partitions_count":2,"moving_data":{"total_written_bytes":0,"in_flight_bytes":0,"in_queue_bytes":0},"least_operating_space_bytes_storage_server":0,"max_machine_failures_without_losing_data":0},"machines":{"$map":{"network":{"megabits_sent":{"hz":0},"megabits_received":{"hz":0},"tcp_segments_retransmitted":{"hz":0}},"memory":{"free_bytes":0,"committed_bytes":0,"total_bytes":0},"contributing_workers":4,"datacenter_id":"6344abf1813eb05b","excluded":false,"address":"1.2.3.4","machine_id":"6344abf1813eb05b","locality":{"$map":"value"},"cpu":{"logical_core_utilization":0.4}}}},"client":{"coordinators":{"coordinators":[{"reachable":true,"address":"127.0.0.1:4701"}],"quorum_reachable":true},"database_status":{"available":true,"healthy":true},"messages":[{"name":{"$enum":["inconsistent_cluster_file","unreachable_cluster_controller","no_cluster_controller","status_incomplete_client","status_incomplete_coordinators","status_incomplete_error","status_incomplete_timeout","status_incomplete_cluster","quorum_not_reachable"]},"description":"The cluster file is not up to date."}],"timestamp":1415650089,"cluster_file":{"path":"/etc/foundationdb/fdb.cluster","up_to_date":true}}}