diff --git a/Makefile b/Makefile index caa4a97915..a2311cc52d 100644 --- a/Makefile +++ b/Makefile @@ -144,7 +144,7 @@ OBJDIR := .objs include $(MK_INCLUDE) -clean: $(CLEAN_TARGETS) +clean: $(CLEAN_TARGETS) docpreview_clean @echo "Cleaning toplevel" @rm -rf $(OBJDIR) @rm -rf $(DEPSDIR) @@ -174,12 +174,21 @@ lib/libstdc++.a: $(shell $(CC) -print-file-name=libstdc++_pic.a) @ar rcs $@ .libstdc++/*.o @rm -r .libstdc++ +docpreview: javadoc godoc + TARGETS= $(MAKE) -C documentation docpreview -ifeq ($(PLATFORM),osx) - MD5SUM=md5 -else - MD5SUM=md5sum -endif +docpreview_clean: + CLEAN_TARGETS= $(MAKE) -C documentation docpreview_clean + +packages/foundationdb-docs-$(VERSION).tar.gz: FORCE javadoc godoc + TARGETS= $(MAKE) -C documentation docpackage + @mkdir -p packages + @rm -f packages/foundationdb-docs-$(VERSION).tar.gz + @cp documentation/sphinx/.dist/foundationdb-docs-$(VERSION).tar.gz packages/foundationdb-docs-$(VERSION).tar.gz + +docpackage: packages/foundationdb-docs-$(VERSION).tar.gz + +FORCE: .SECONDEXPANSION: diff --git a/bindings/__init__.py b/bindings/__init__.py index c7f6e4d441..79e21a9a7b 100644 --- a/bindings/__init__.py +++ b/bindings/__init__.py @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/bindingtester/__init__.py b/bindings/bindingtester/__init__.py index b780f71ad4..cfc472b176 100644 --- a/bindings/bindingtester/__init__.py +++ b/bindings/bindingtester/__init__.py @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,36 +21,37 @@ import sys import os -sys.path[:0]=[os.path.join(os.path.dirname(__file__), '..', '..', 'bindings', 'python')] +sys.path[:0] = [os.path.join(os.path.dirname(__file__), '..', '..', 'bindings', 'python')] import util -FDB_API_VERSION = 510 +FDB_API_VERSION = 520 LOGGING = { - 'version' : 1, - 'disable_existing_loggers' : False, - 'formatters' : { - 'simple' : { - 'format' : '%(message)s' + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'simple': { + 'format': '%(message)s' } }, - 'handlers' : { - 'console' : { - 'level' : 'NOTSET', - 'class' : 'logging.StreamHandler', - 'stream' : sys.stdout, - 'formatter' : 'simple' + 'handlers': { + 'console': { + 'level': 'NOTSET', + 'class': 'logging.StreamHandler', + 'stream': sys.stdout, + 'formatter': 'simple' } }, - 'loggers' : { - 'foundationdb.bindingtester' : { - 'level' : 'INFO', - 'handlers' : ['console'] + 'loggers': { + 'foundationdb.bindingtester': { + 'level': 'INFO', + 'handlers': ['console'] } } } + class Result: def __init__(self, subspace, key, values): self.subspace_tuple = util.subspace_to_tuple(subspace) @@ -63,7 +64,7 @@ class Result: left_key = self.key_tuple[specification.key_start_index:] right_key = self.key_tuple[specification.key_start_index:] - + if len(left_key) != len(right_key) or left_key != right_key: return False @@ -81,7 +82,7 @@ class Result: def sequence_num(self, specification): if specification.ordering_index is not None: return self.key_tuple[specification.ordering_index] - + return None def __str__(self): @@ -91,4 +92,3 @@ class Result: value_str = repr(self.values) return '%s = %s' % (repr(self.subspace_tuple + self.key_tuple), value_str) - diff --git a/bindings/bindingtester/bindingtester.py b/bindings/bindingtester/bindingtester.py index 56f62d49a4..983ff9c388 100755 --- a/bindings/bindingtester/bindingtester.py +++ b/bindings/bindingtester/bindingtester.py @@ -5,13 +5,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,8 +34,9 @@ from threading import Timer, Event import logging.config from collections import OrderedDict +from functools import reduce -sys.path[:0]=[os.path.join(os.path.dirname(__file__), '..')] +sys.path[:0] = [os.path.join(os.path.dirname(__file__), '..')] import bindingtester @@ -52,6 +53,7 @@ import fdb.tuple fdb.api_version(FDB_API_VERSION) + class ResultSet(object): def __init__(self, specification): self.specification = specification @@ -80,20 +82,20 @@ class ResultSet(object): has_filtered_error = False while True: - results = { i : r[indices[i]] for i, r in enumerate(self.tester_results.values()) if len(r) > indices[i] } + results = {i: r[indices[i]] for i, r in enumerate(self.tester_results.values()) if len(r) > indices[i]} if len(results) == 0: break - sequence_nums = [ r.sequence_num(self.specification) for r in results.values() ] + sequence_nums = [r.sequence_num(self.specification) for r in results.values()] if any([s is not None for s in sequence_nums]): - results = { i : r for i, r in results.items() if r.sequence_num(self.specification) == min(sequence_nums) } + results = {i: r for i, r in results.items() if r.sequence_num(self.specification) == min(sequence_nums)} else: - results = { i : r for i, r in results.items() if r.matches(min(results.values()), self.specification) } + results = {i: r for i, r in results.items() if r.matches(min(results.values()), self.specification)} for i in results.keys(): indices[i] += 1 - all_results = { i : results[i] if i in results else None for i in range(len(self.tester_results)) } + all_results = {i: results[i] if i in results else None for i in range(len(self.tester_results))} result_str = '\n'.join([' %-*s - %s' % (name_length, self.tester_results.keys()[i], r) for i, r in all_results.items()]) result_list = results.values() @@ -113,12 +115,15 @@ class ResultSet(object): return (num_errors, has_filtered_error) + def choose_api_version(selected_api_version, tester_min_version, tester_max_version, test_min_version, test_max_version): if selected_api_version is not None: if selected_api_version < tester_min_version or selected_api_version > tester_max_version: - raise Exception('Not all testers support the API version %d (min=%d, max=%d)' % (selected_api_version, tester_min_version, tester_max_version)) + raise Exception('Not all testers support the API version %d (min=%d, max=%d)' % + (selected_api_version, tester_min_version, tester_max_version)) elif selected_api_version < test_min_version or selected_api_version > test_max_version: - raise Exception('API version %d is not supported by the specified test (min=%d, max=%d)' % (selected_api_version, test_min_version, test_max_version)) + raise Exception('API version %d is not supported by the specified test (min=%d, max=%d)' % + (selected_api_version, test_min_version, test_max_version)) api_version = selected_api_version else: @@ -126,19 +131,23 @@ def choose_api_version(selected_api_version, tester_min_version, tester_max_vers max_version = min(tester_max_version, test_max_version) if min_version > max_version: - raise Exception('Not all testers support the API versions required by the specified test (tester: min=%d, max=%d; test: min=%d, max=%d)' % (tester_min_version, tester_max_version, test_min_version, test_max_version)) + raise Exception( + 'Not all testers support the API versions required by the specified test' + '(tester: min=%d, max=%d; test: min=%d, max=%d)' % (tester_min_version, tester_max_version, test_min_version, test_max_version)) if random.random() < 0.7: api_version = max_version elif random.random() < 0.7: api_version = min_version elif random.random() < 0.9: - api_version = random.choice([v for v in [13, 14, 16, 21, 22, 23, 100, 200, 300, 400, 410, 420, 430, 440, 450, 460, 500, 510] if v >= min_version and v <= max_version]) + api_version = random.choice([v for v in [13, 14, 16, 21, 22, 23, 100, 200, 300, 400, 410, 420, 430, + 440, 450, 460, 500, 510, 520] if v >= min_version and v <= max_version]) else: api_version = random.randint(min_version, max_version) return api_version + class TestRunner(object): def __init__(self, args): self.args = copy.copy(args) @@ -157,7 +166,8 @@ class TestRunner(object): min_api_version = max([tester.min_api_version for tester in self.testers]) max_api_version = min([tester.max_api_version for tester in self.testers]) - self.args.api_version = choose_api_version(self.args.api_version, min_api_version, max_api_version, self.test.min_api_version, self.test.max_api_version) + self.args.api_version = choose_api_version(self.args.api_version, min_api_version, max_api_version, + self.test.min_api_version, self.test.max_api_version) util.get_logger().info('\nCreating test at API version %d' % self.args.api_version) @@ -165,7 +175,8 @@ class TestRunner(object): if self.args.max_int_bits is None: self.args.max_int_bits = max_int_bits elif self.args.max_int_bits > max_int_bits: - raise Exception('The specified testers support at most %d-bit ints, but --max-int-bits was set to %d' % (max_int_bits, self.args.max_int_bits)) + raise Exception('The specified testers support at most %d-bit ints, but --max-int-bits was set to %d' % + (max_int_bits, self.args.max_int_bits)) self.args.no_threads = self.args.no_threads or any([not tester.threads_enabled for tester in self.testers]) if self.args.no_threads and self.args.concurrency > 1: @@ -189,15 +200,15 @@ class TestRunner(object): for i, instruction in enumerate(instructions): if self.args.print_all or (instruction.operation != 'SWAP' and instruction.operation != 'PUSH'): - util.get_logger().error(' %d. %r' % (i+offset, instruction)) + util.get_logger().error(' %d. %r' % (i + offset, instruction)) - util.get_logger().error(''); + util.get_logger().error('') def run_test(self): test_instructions = self._generate_test() expected_results = self.test.get_expected_results() - tester_results = { s.subspace : ResultSet(s) for s in self.test.get_result_specifications() } + tester_results = {s.subspace: ResultSet(s) for s in self.test.get_result_specifications()} for subspace, results in expected_results.items(): tester_results[subspace].add('expected', results) @@ -208,7 +219,8 @@ class TestRunner(object): self.test.pre_run(self.db, self.args) return_code = self._run_tester(tester) if return_code != 0: - util.get_logger().error('Test of type %s failed to complete successfully with random seed %d and %d operations\n' % (self.args.test_name, self.args.seed, self.args.num_ops)) + util.get_logger().error('Test of type %s failed to complete successfully with random seed %d and %d operations\n' % + (self.args.test_name, self.args.seed, self.args.num_ops)) return 2 tester_errors[tester] = self.test.validate(self.db, self.args) @@ -226,18 +238,19 @@ class TestRunner(object): self._insert_instructions(test_instructions) def _generate_test(self): - util.get_logger().info('Generating %s test at seed %d with %d op(s) and %d concurrent tester(s)...' % (self.args.test_name, self.args.seed, self.args.num_ops, self.args.concurrency)) + util.get_logger().info('Generating %s test at seed %d with %d op(s) and %d concurrent tester(s)...' % + (self.args.test_name, self.args.seed, self.args.num_ops, self.args.concurrency)) random.seed(self.test_seed) if self.args.concurrency == 1: self.test.setup(self.args) - test_instructions = { fdb.Subspace((self.args.instruction_prefix,)) : self.test.generate(self.args, 0) } + test_instructions = {fdb.Subspace((self.args.instruction_prefix,)): self.test.generate(self.args, 0)} else: test_instructions = {} main_thread = InstructionSet() for i in range(self.args.concurrency): - #thread_spec = fdb.Subspace(('thread_spec', i)) + # thread_spec = fdb.Subspace(('thread_spec', i)) thread_spec = 'thread_spec%d' % i main_thread.push_args(thread_spec) main_thread.append('START_THREAD') @@ -260,7 +273,7 @@ class TestRunner(object): params += [self.args.cluster_file] util.get_logger().info('\nRunning tester \'%s\'...' % ' '.join(params)) - sys.stdout.flush(); + sys.stdout.flush() proc = subprocess.Popen(params) timed_out = Event() @@ -321,9 +334,10 @@ class TestRunner(object): if len(errors) > 0: util.get_logger().error('The %s tester reported errors:\n' % tester.name) for i, error in enumerate(errors): - util.get_logger().error(' %d. %s' % (i+1, error)) + util.get_logger().error(' %d. %s' % (i + 1, error)) - log_message = '\nTest with seed %d and concurrency %d had %d incorrect result(s) and %d error(s) at API version %d' % (self.args.seed, self.args.concurrency, num_incorrect, num_errors, self.args.api_version) + log_message = '\nTest with seed %d and concurrency %d had %d incorrect result(s) and %d error(s) at API version %d' %\ + (self.args.seed, self.args.concurrency, num_incorrect, num_errors, self.args.api_version) if num_errors == 0 and (num_incorrect == 0 or has_filtered_error): util.get_logger().info(log_message) if has_filtered_error: @@ -333,6 +347,7 @@ class TestRunner(object): util.get_logger().error(log_message) return 1 + def bisect(test_runner, args): util.get_logger().info('') @@ -354,7 +369,8 @@ def bisect(test_runner, args): util.get_logger().error('Error finding minimal failing test for seed %d. The failure may not be deterministic' % args.seed) return 1 else: - util.get_logger().error('No failing test found for seed %d with %d ops. Try specifying a larger --num-ops parameter.' % (args.seed, args.num_ops)) + util.get_logger().error('No failing test found for seed %d with %d ops. Try specifying a larger --num-ops parameter.' + % (args.seed, args.num_ops)) return 0 elif result == 0: @@ -365,30 +381,45 @@ def bisect(test_runner, args): util.get_logger().info('Test with %d operations failed with error code %d\n' % (test_runner.args.num_ops, result)) upper_bound = test_runner.args.num_ops + def parse_args(argv): parser = argparse.ArgumentParser(description='FoundationDB Binding API Tester') - parser.add_argument('--test-name', default='scripted', help='The name of the test to run. Must be the name of a test specified in the tests folder. (default=\'scripted\')') + parser.add_argument('--test-name', default='scripted', + help='The name of the test to run. Must be the name of a test specified in the tests folder. (default=\'scripted\')') parser.add_argument(metavar='tester1', dest='test1', help='Name of the first tester to invoke') - parser.add_argument('--compare', metavar='tester2', nargs='?', type=str, default=None, const='python', dest='test2', help='When specified, a second tester will be run and compared against the first. This flag takes an optional argument for the second tester to invoke (default = \'python\').') - - parser.add_argument('--print-test', action='store_true', help='Instead of running a test, prints the set of instructions generated for that test. Unless --all is specified, all setup, finalization, PUSH, and SWAP instructions will be excluded.') + parser.add_argument('--compare', metavar='tester2', nargs='?', type=str, default=None, const='python', dest='test2', + help='When specified, a second tester will be run and compared against the first. This flag takes an optional argument ' + 'for the second tester to invoke (default = \'python\').') + parser.add_argument('--print-test', action='store_true', + help='Instead of running a test, prints the set of instructions generated for that test. Unless --all is specified, all ' + 'setup, finalization, PUSH, and SWAP instructions will be excluded.') parser.add_argument('--all', dest='print_all', action='store_true', help='Causes --print-test to print all instructions.') - parser.add_argument('--bisect', action='store_true', help='Run the specified test varying the number of operations until a minimal failing test is found. Does not work for concurrent tests.') + parser.add_argument('--bisect', action='store_true', + help='Run the specified test varying the number of operations until a minimal failing test is found. Does not work for ' + 'concurrent tests.') parser.add_argument('--insert-only', action='store_true', help='Insert the test instructions into the database, but do not run it.') - parser.add_argument('--concurrency', type=int, default=1, help='Number of concurrent test threads to run. (default = 1).') parser.add_argument('--num-ops', type=int, default=100, help='The number of operations to generate per thread (default = 100)') parser.add_argument('--seed', type=int, help='The random seed to use for generating the test') - parser.add_argument('--max-int-bits', type=int, default=None, help='Maximum number of bits to use for int types in testers. By default, the largest value supported by the testers being run will be chosen.') - parser.add_argument('--api-version', default=None, type=int, help='The API version that the testers should use. Not supported in scripted mode. (default = random version supported by all testers)') + parser.add_argument('--max-int-bits', type=int, default=None, + help='Maximum number of bits to use for int types in testers. By default, the largest value supported by the testers being ' + 'run will be chosen.') + parser.add_argument('--api-version', default=None, type=int, + help='The API version that the testers should use. Not supported in scripted mode. (default = random version supported by ' + 'all testers)') parser.add_argument('--cluster-file', type=str, default=None, help='The cluster file for the cluster being connected to. (default None)') parser.add_argument('--timeout', type=int, default=600, help='The timeout in seconds for running each individual tester. (default 600)') - parser.add_argument('--enable-client-trace-logging', nargs='?', type=str, default=None, const='.', help='Enables trace file output. This flag takes an optional argument specifying the output directory (default = \'.\').') - parser.add_argument('--instruction-prefix', type=str, default='test_spec', help='The prefix under which the main thread of test instructions are inserted (default=\'test_spec\').') - parser.add_argument('--output-subspace', type=str, default='tester_output', help='The string used to create the output subspace for the testers. The subspace will be of the form (,). (default=\'tester_output\')') + parser.add_argument('--enable-client-trace-logging', nargs='?', type=str, default=None, const='.', + help='Enables trace file output. This flag takes an optional argument specifying the output directory (default = \'.\').') + parser.add_argument('--instruction-prefix', type=str, default='test_spec', + help='The prefix under which the main thread of test instructions are inserted (default=\'test_spec\').') + parser.add_argument('--output-subspace', type=str, default='tester_output', + help='The string used to create the output subspace for the testers. The subspace will be of the form (,). ' + '(default=\'tester_output\')') - parser.add_argument('--logging-level', type=str, default='INFO', choices=['ERROR', 'WARNING', 'INFO', 'DEBUG'], help='Specifies the level of detail in the tester output (default=\'INFO\').') + parser.add_argument('--logging-level', type=str, default='INFO', + choices=['ERROR', 'WARNING', 'INFO', 'DEBUG'], help='Specifies the level of detail in the tester output (default=\'INFO\').') # SOMEDAY: this applies only to the scripted test. Should we invoke test files specifically (as in circus), # or invoke them here and allow tests to add arguments? @@ -396,6 +427,7 @@ def parse_args(argv): return parser.parse_args(argv) + def validate_args(args): if args.insert_only and args.bisect: raise Exception('--bisect cannot be used with --insert-only') @@ -408,6 +440,7 @@ def validate_args(args): if args.concurrency > 1 and args.test2: raise Exception('--compare cannot be used with concurrent tests') + def main(argv): args = parse_args(argv) try: @@ -444,9 +477,11 @@ def main(argv): util.get_logger().debug(traceback.format_exc()) exit(3) - except: + except BaseException: util.get_logger().error('\nERROR: %s' % sys.exc_info()[0]) util.get_logger().info(traceback.format_exc()) exit(3) -if __name__ == '__main__': sys.exit(main(sys.argv[1:])) + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/bindings/bindingtester/known_testers.py b/bindings/bindingtester/known_testers.py index 23a07ad61a..d2a5475a62 100644 --- a/bindings/bindingtester/known_testers.py +++ b/bindings/bindingtester/known_testers.py @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -20,9 +20,10 @@ import os -MAX_API_VERSION = 510 -COMMON_TYPES = [ 'null', 'bytes', 'string', 'int', 'uuid', 'bool', 'float', 'double', 'tuple' ] -ALL_TYPES = COMMON_TYPES + [ 'versionstamp' ] +MAX_API_VERSION = 520 +COMMON_TYPES = ['null', 'bytes', 'string', 'int', 'uuid', 'bool', 'float', 'double', 'tuple'] +ALL_TYPES = COMMON_TYPES + ['versionstamp'] + class Tester: def __init__(self, name, cmd, max_int_bits=64, min_api_version=0, max_api_version=MAX_API_VERSION, threads_enabled=True, types=COMMON_TYPES): @@ -44,22 +45,24 @@ class Tester: else: return Tester(test_name_or_args.split(' ')[0], test_name_or_args) + def _absolute_path(path): return os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', path) + _java_cmd = 'java -ea -cp %s:%s com.apple.foundationdb.test.' % ( - _absolute_path('java/foundationdb-client.jar'), - _absolute_path('java/foundationdb-tests.jar')) + _absolute_path('java/foundationdb-client.jar'), + _absolute_path('java/foundationdb-tests.jar')) # We could set min_api_version lower on some of these if the testers were updated to support them testers = { - 'python' : Tester('python', 'python ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES), - 'python3' : Tester('python3', 'python3 ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES), - 'node' : Tester('node', _absolute_path('nodejs/tests/tester.js'), 53, 500, MAX_API_VERSION), - 'streamline' : Tester('streamline', _absolute_path('nodejs/tests/streamline_tester._js'), 53, 500, MAX_API_VERSION), - 'ruby' : Tester('ruby', _absolute_path('ruby/tests/tester.rb'), 64, 23, MAX_API_VERSION), - 'java' : Tester('java', _java_cmd + 'StackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES), - 'java_async' : Tester('java', _java_cmd + 'AsyncStackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES), - 'go' : Tester('go', _absolute_path('go/build/bin/_stacktester'), 63, 200, MAX_API_VERSION), - 'flow' : Tester('flow', _absolute_path('flow/bin/fdb_flow_tester'), 63, 500, MAX_API_VERSION), + 'python': Tester('python', 'python ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES), + 'python3': Tester('python3', 'python3 ' + _absolute_path('python/tests/tester.py'), 2040, 23, MAX_API_VERSION, types=ALL_TYPES), + 'node': Tester('node', _absolute_path('nodejs/tests/tester.js'), 53, 500, MAX_API_VERSION), + 'streamline': Tester('streamline', _absolute_path('nodejs/tests/streamline_tester._js'), 53, 500, MAX_API_VERSION), + 'ruby': Tester('ruby', _absolute_path('ruby/tests/tester.rb'), 64, 23, MAX_API_VERSION), + 'java': Tester('java', _java_cmd + 'StackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES), + 'java_async': Tester('java', _java_cmd + 'AsyncStackTester', 2040, 510, MAX_API_VERSION, types=ALL_TYPES), + 'go': Tester('go', _absolute_path('go/build/bin/_stacktester'), 63, 200, MAX_API_VERSION), + 'flow': Tester('flow', _absolute_path('flow/bin/fdb_flow_tester'), 63, 500, MAX_API_VERSION), } diff --git a/bindings/bindingtester/tests/__init__.py b/bindings/bindingtester/tests/__init__.py index c65073bf18..20f45835e6 100644 --- a/bindings/bindingtester/tests/__init__.py +++ b/bindings/bindingtester/tests/__init__.py @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,11 +28,12 @@ from bindingtester import util fdb.api_version(FDB_API_VERSION) + class ResultSpecification(object): def __init__(self, subspace, key_start_index=0, ordering_index=None, global_error_filter=None): self.subspace = subspace self.key_start_index = key_start_index - self.ordering_index = ordering_index + self.ordering_index = ordering_index if global_error_filter is not None: error_str = '|'.join(['%d' % e for e in global_error_filter]) @@ -45,7 +46,7 @@ class ResultSpecification(object): return False return self.error_regex.search(str) is not None - + class Test(object): def __init__(self, subspace, min_api_version=0, max_api_version=int(1e9)): @@ -54,7 +55,7 @@ class Test(object): self.max_api_version = max_api_version # Returns nothing - def setup(self, args): + def setup(self, args): pass # Returns an instance of TestInstructions @@ -75,7 +76,7 @@ class Test(object): def get_expected_results(self): return {} - # Returns a list of error strings + # Returns a list of error strings def validate(self, db, args): return [] @@ -88,6 +89,7 @@ class Test(object): return test_class[0](subspace) + class Instruction(object): def __init__(self, operation): self.operation = operation @@ -103,6 +105,7 @@ class Instruction(object): def __repr__(self): return repr(self.operation) + class PushInstruction(Instruction): def __init__(self, argument): self.operation = 'PUSH' @@ -115,6 +118,7 @@ class PushInstruction(Instruction): def __repr__(self): return '%r %r' % (self.operation, self.argument) + class TestInstructions(object): def __init__(self): pass @@ -126,13 +130,14 @@ class TestInstructions(object): def insert_operations(self, db, subspace): pass + class InstructionSet(TestInstructions, list): def __init__(self): TestInstructions.__init__(self) list.__init__(self) self.core_test_begin = 0 - self.core_test_end = None + self.core_test_end = None def push_args(self, *args): self.extend([PushInstruction(arg) for arg in reversed(args)]) @@ -144,7 +149,7 @@ class InstructionSet(TestInstructions, list): list.append(self, Instruction(instruction)) def get_threads(self, subspace): - return { subspace : self } + return {subspace: self} def setup_complete(self): self.core_test_begin = len(self) @@ -153,16 +158,17 @@ class InstructionSet(TestInstructions, list): self.core_test_end = len(self) def core_instructions(self): - return self[self.core_test_begin : self.core_test_end] - + return self[self.core_test_begin: self.core_test_end] + @fdb.transactional def _insert_operations_transactional(self, tr, subspace, start, count): - for i, instruction in enumerate(self[start : start+count]): + for i, instruction in enumerate(self[start: start + count]): tr[subspace.pack((start + i,))] = instruction.to_value() def insert_operations(self, db, subspace): for i in range(0, int(math.ceil(len(self) / 1000.0))): - self._insert_operations_transactional(db, subspace, i*1000, 1000) + self._insert_operations_transactional(db, subspace, i * 1000, 1000) + class ThreadedInstructionSet(TestInstructions): def __init__(self): @@ -194,4 +200,5 @@ class ThreadedInstructionSet(TestInstructions): self.threads[subspace] = thread_instructions return thread_instructions + util.import_subclasses(__file__, 'bindingtester.tests') diff --git a/bindings/bindingtester/tests/api.py b/bindings/bindingtester/tests/api.py index 2e539efd47..e31b8636fb 100644 --- a/bindings/bindingtester/tests/api.py +++ b/bindings/bindingtester/tests/api.py @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,7 +18,6 @@ # limitations under the License. # -import ctypes import random import struct @@ -32,11 +31,12 @@ from bindingtester.tests import test_util fdb.api_version(FDB_API_VERSION) + class ApiTest(Test): def __init__(self, subspace): super(ApiTest, self).__init__(subspace) - self.workspace = self.subspace['workspace'] # The keys and values here must match between subsequent runs of the same test - self.scratch = self.subspace['scratch'] # The keys and values here can differ between runs + self.workspace = self.subspace['workspace'] # The keys and values here must match between subsequent runs of the same test + self.scratch = self.subspace['scratch'] # The keys and values here can differ between runs self.stack_subspace = self.subspace['stack'] self.versionstamped_values = self.scratch['versionstamped_values'] @@ -78,7 +78,7 @@ class ApiTest(Test): self.key_depth = max(0, self.key_depth - num) self.outstanding_ops = [i for i in self.outstanding_ops if i[0] <= self.stack_size] - + def ensure_string(self, instructions, num): while self.string_depth < num: instructions.push_args(self.random.random_string(random.randint(0, 100))) @@ -97,7 +97,7 @@ class ApiTest(Test): tup = self.random.random_tuple(5) self.generated_keys.append(tup) - return self.workspace.pack(tup) + return self.workspace.pack(tup) def ensure_key(self, instructions, num): while self.key_depth < num: @@ -131,7 +131,7 @@ class ApiTest(Test): def wait_for_reads(self, instructions): while len(self.outstanding_ops) > 0 and self.outstanding_ops[-1][0] <= self.stack_size: read = self.outstanding_ops.pop() - #print '%d. waiting for read at instruction %r' % (len(instructions), read) + # print '%d. waiting for read at instruction %r' % (len(instructions), read) test_util.to_front(instructions, self.stack_size - read[0]) instructions.append('WAIT_FUTURE') @@ -168,7 +168,7 @@ class ApiTest(Test): op_choices += resets idempotent_atomic_ops = [u'BIT_AND', u'BIT_OR', u'MAX', u'MIN', u'BYTE_MIN', u'BYTE_MAX'] - atomic_ops = idempotent_atomic_ops + [u'ADD', u'BIT_XOR'] + atomic_ops = idempotent_atomic_ops + [u'ADD', u'BIT_XOR', u'APPEND_IF_FITS'] if args.concurrency > 1: self.max_keys = random.randint(100, 1000) @@ -187,7 +187,7 @@ class ApiTest(Test): index = len(instructions) read_performed = False - #print 'Adding instruction %s at %d' % (op, index) + # print 'Adding instruction %s at %d' % (op, index) if args.concurrency == 1 and (op in database_mutations): self.wait_for_reads(instructions) @@ -211,7 +211,7 @@ class ApiTest(Test): instructions.push_args(random.randint(0, 5000)) instructions.append(op) - self.outstanding_ops.append((self.stack_size, len(instructions)-1)) + self.outstanding_ops.append((self.stack_size, len(instructions) - 1)) if args.concurrency == 1: self.wait_for_reads(instructions) @@ -236,7 +236,7 @@ class ApiTest(Test): test_util.to_front(instructions, 3) instructions.append(op) - #Don't add key here because we may be outside of our prefix + # Don't add key here because we may be outside of our prefix self.add_strings(1) self.can_set_version = False read_performed = True @@ -249,7 +249,7 @@ class ApiTest(Test): test_util.to_front(instructions, 4) instructions.append(op) - if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large + if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large self.add_strings(1) else: self.add_stack_items(1) @@ -258,14 +258,14 @@ class ApiTest(Test): read_performed = True elif op == 'GET_RANGE_STARTS_WITH' or op == 'GET_RANGE_STARTS_WITH_SNAPSHOT' or op == 'GET_RANGE_STARTS_WITH_DATABASE': - #TODO: not tested well + # TODO: not tested well self.ensure_key(instructions, 1) range_params = self.random.random_range_params() instructions.push_args(*range_params) test_util.to_front(instructions, 3) instructions.append(op) - if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large + if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large self.add_strings(1) else: self.add_stack_items(1) @@ -285,7 +285,7 @@ class ApiTest(Test): test_util.to_front(instructions, 9) instructions.append(op) - if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large + if range_params[0] >= 1 and range_params[0] <= 1000: # avoid adding a string if the limit is large self.add_strings(1) else: self.add_stack_items(1) @@ -302,8 +302,8 @@ class ApiTest(Test): self.ensure_key_value(instructions) instructions.append(op) if op == 'SET_DATABASE': - self.add_stack_items(1) - + self.add_stack_items(1) + elif op == 'SET_READ_VERSION': if self.has_version and self.can_set_version: instructions.append(op) @@ -316,7 +316,7 @@ class ApiTest(Test): self.add_stack_items(1) elif op == 'CLEAR_RANGE' or op == 'CLEAR_RANGE_DATABASE': - #Protect against inverted range + # Protect against inverted range key1 = self.workspace.pack(self.random.random_tuple(5)) key2 = self.workspace.pack(self.random.random_tuple(5)) @@ -334,7 +334,7 @@ class ApiTest(Test): instructions.append(op) if op == 'CLEAR_RANGE_STARTS_WITH_DATABASE': self.add_stack_items(1) - + elif op == 'ATOMIC_OP' or op == 'ATOMIC_OP_DATABASE': self.ensure_key_value(instructions) if op == 'ATOMIC_OP' or args.concurrency > 1: @@ -351,10 +351,10 @@ class ApiTest(Test): key1 = self.versionstamped_values.pack((rand_str1,)) split = random.randint(0, 70) - rand_str2 = self.random.random_string(20+split) + fdb.tuple.Versionstamp._UNSET_TR_VERSION + self.random.random_string(70-split) + rand_str2 = self.random.random_string(20 + split) + fdb.tuple.Versionstamp._UNSET_TR_VERSION + self.random.random_string(70 - split) key2 = self.versionstamped_keys.pack() + rand_str2 index = key2.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION) - key2 += chr(index%256)+chr(index/256) + key2 += chr(index % 256) + chr(index / 256) instructions.push_args(u'SET_VERSIONSTAMPED_VALUE', key1, fdb.tuple.Versionstamp._UNSET_TR_VERSION + rand_str2) instructions.append('ATOMIC_OP') @@ -436,8 +436,8 @@ class ApiTest(Test): version_key = self.versionstamped_keys.pack(tup) first_incomplete = version_key.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION) - second_incomplete = -1 if first_incomplete < 0 else version_key.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION, - first_incomplete + len(fdb.tuple.Versionstamp._UNSET_TR_VERSION) + 1) + second_incomplete = -1 if first_incomplete < 0 else \ + version_key.find(fdb.tuple.Versionstamp._UNSET_TR_VERSION, first_incomplete + len(fdb.tuple.Versionstamp._UNSET_TR_VERSION) + 1) # If there is exactly one incomplete versionstamp, perform the versionstamped key operation. if first_incomplete >= 0 and second_incomplete < 0: @@ -449,7 +449,8 @@ class ApiTest(Test): instructions.append('ATOMIC_OP') version_value_key = self.versionstamped_values.pack((rand_str,)) - instructions.push_args(u'SET_VERSIONSTAMPED_VALUE', version_value_key, fdb.tuple.Versionstamp._UNSET_TR_VERSION + fdb.tuple.pack(tup)) + instructions.push_args(u'SET_VERSIONSTAMPED_VALUE', version_value_key, + fdb.tuple.Versionstamp._UNSET_TR_VERSION + fdb.tuple.pack(tup)) instructions.append('ATOMIC_OP') self.can_use_key_selectors = False @@ -469,7 +470,7 @@ class ApiTest(Test): instructions.append(op) self.add_strings(len(tups)) - #Use SUB to test if integers are correctly unpacked + # Use SUB to test if integers are correctly unpacked elif op == 'SUB': a = self.random.random_int() / 2 b = self.random.random_int() / 2 @@ -512,7 +513,7 @@ class ApiTest(Test): assert False if read_performed and op not in database_reads: - self.outstanding_ops.append((self.stack_size, len(instructions)-1)) + self.outstanding_ops.append((self.stack_size, len(instructions) - 1)) if args.concurrency == 1 and (op in database_reads or op in database_mutations): instructions.append('WAIT_FUTURE') @@ -536,7 +537,7 @@ class ApiTest(Test): def check_versionstamps(self, tr, begin_key, limit): next_begin = None incorrect_versionstamps = 0 - for k,v in tr.get_range(begin_key, self.versionstamped_values.range().stop, limit=limit): + for k, v in tr.get_range(begin_key, self.versionstamped_values.range().stop, limit=limit): next_begin = k + '\x00' tup = fdb.tuple.unpack(k) key = self.versionstamped_keys.pack() + v[10:].replace(fdb.tuple.Versionstamp._UNSET_TR_VERSION, v[:10], 1) @@ -545,7 +546,6 @@ class ApiTest(Test): util.get_logger().error(' %s != %s', repr(tr[key]), repr(tup[-1])) incorrect_versionstamps += 1 - return (next_begin, incorrect_versionstamps) def validate(self, db, args): @@ -564,8 +564,7 @@ class ApiTest(Test): return errors def get_result_specifications(self): - return [ - ResultSpecification(self.workspace, global_error_filter=[1007, 1021]), - ResultSpecification(self.stack_subspace, key_start_index=1, ordering_index=1, global_error_filter=[1007, 1021]) + return [ + ResultSpecification(self.workspace, global_error_filter=[1007, 1021]), + ResultSpecification(self.stack_subspace, key_start_index=1, ordering_index=1, global_error_filter=[1007, 1021]) ] - diff --git a/bindings/bindingtester/tests/directory.py b/bindings/bindingtester/tests/directory.py index 6886081b62..861ca93972 100644 --- a/bindings/bindingtester/tests/directory.py +++ b/bindings/bindingtester/tests/directory.py @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -32,6 +32,7 @@ from bindingtester.tests.directory_util import DirListEntry fdb.api_version(FDB_API_VERSION) + class DirectoryTest(Test): def __init__(self, subspace): @@ -71,12 +72,12 @@ class DirectoryTest(Test): instructions = InstructionSet() op_choices = ['NEW_TRANSACTION', 'COMMIT'] - + general = ['DIRECTORY_CREATE_SUBSPACE', 'DIRECTORY_CREATE_LAYER'] op_choices += general - directory_mutations = ['DIRECTORY_CREATE_OR_OPEN', 'DIRECTORY_CREATE', 'DIRECTORY_MOVE', 'DIRECTORY_MOVE_TO', + directory_mutations = ['DIRECTORY_CREATE_OR_OPEN', 'DIRECTORY_CREATE', 'DIRECTORY_MOVE', 'DIRECTORY_MOVE_TO', 'DIRECTORY_REMOVE', 'DIRECTORY_REMOVE_IF_EXISTS'] directory_reads = ['DIRECTORY_EXISTS', 'DIRECTORY_OPEN', 'DIRECTORY_LIST'] @@ -105,17 +106,18 @@ class DirectoryTest(Test): # Generate some directories that we are going to create in advance. This tests that other bindings # are compatible with the Python implementation - self.prepopulated_dirs = [ (generate_path(min_length=1), self.generate_layer()) for i in range(5) ] + self.prepopulated_dirs = [(generate_path(min_length=1), self.generate_layer()) for i in range(5)] for path, layer in self.prepopulated_dirs: instructions.push_args(layer) instructions.push_args(*test_util.with_length(path)) instructions.append('DIRECTORY_OPEN') - #print '%d. Selected %s, dir=%s, has_known_prefix=%s, dir_list_len=%d' % (len(instructions), 'DIRECTORY_OPEN', repr(self.dir_index), False, len(self.dir_list)) + # print '%d. Selected %s, dir=%s, has_known_prefix=%s, dir_list_len=%d' \ + # % (len(instructions), 'DIRECTORY_OPEN', repr(self.dir_index), False, len(self.dir_list)) self.dir_list.append(self.dir_list[0].add_child(path, default_path, self.root, DirListEntry(True, True, has_known_prefix=False))) instructions.setup_complete() - + for i in range(args.num_ops): if random.random() < 0.5: self.dir_index = random.randrange(0, len(self.dir_list)) @@ -131,7 +133,8 @@ class DirectoryTest(Test): op = random.choice(choices) dir_entry = self.dir_list[self.dir_index] - #print '%d. Selected %s, dir=%s, has_known_prefix=%s, dir_list_len=%d' % (len(instructions), op, repr(self.dir_index), repr(dir_entry.has_known_prefix), len(self.dir_list)) + # print '%d. Selected %s, dir=%s, has_known_prefix=%s, dir_list_len=%d' \ + # % (len(instructions), op, repr(self.dir_index), repr(dir_entry.has_known_prefix), len(self.dir_list)) if op.endswith('_DATABASE') or op.endswith('_SNAPSHOT'): root_op = op[0:-9] @@ -160,7 +163,7 @@ class DirectoryTest(Test): indices.append(len(self.dir_list)) self.dir_list.append(DirListEntry(False, True)) - instructions.push_args(random.choice([0,1])) + instructions.push_args(random.choice([0, 1])) instructions.push_args(*indices) instructions.append(op) self.dir_list.append(DirListEntry(True, False, False)) @@ -172,7 +175,7 @@ class DirectoryTest(Test): test_util.blocking_commit(instructions) path = generate_path() - op_args = test_util.with_length(path) + (self.generate_layer(),) + op_args = test_util.with_length(path) + (self.generate_layer(),) directory_util.push_instruction_and_record_prefix(instructions, op, op_args, path, len(self.dir_list), self.random, self.prefix_log) if not op.endswith('_DATABASE') and args.concurrency == 1: @@ -189,18 +192,19 @@ class DirectoryTest(Test): # Because allocated prefixes are non-deterministic, we cannot have overlapping # transactions that allocate/remove these prefixes in a comparison test - if op.endswith('_DATABASE') and args.concurrency == 1: # and allow_empty_prefix: + if op.endswith('_DATABASE') and args.concurrency == 1: # and allow_empty_prefix: test_util.blocking_commit(instructions) path = generate_path() - op_args = test_util.with_length(path) + (layer, prefix) + op_args = test_util.with_length(path) + (layer, prefix) if prefix is None: - directory_util.push_instruction_and_record_prefix(instructions, op, op_args, path, len(self.dir_list), self.random, self.prefix_log) + directory_util.push_instruction_and_record_prefix( + instructions, op, op_args, path, len(self.dir_list), self.random, self.prefix_log) else: instructions.push_args(*op_args) instructions.append(op) - if not op.endswith('_DATABASE') and args.concurrency == 1: # and allow_empty_prefix: + if not op.endswith('_DATABASE') and args.concurrency == 1: # and allow_empty_prefix: test_util.blocking_commit(instructions) self.dir_list.append(dir_entry.add_child(path, default_path, self.root, DirListEntry(True, True, bool(prefix)))) @@ -228,13 +232,14 @@ class DirectoryTest(Test): new_path = generate_path() instructions.push_args(*test_util.with_length(new_path)) instructions.append(op) - self.dir_list.append(dir_entry.root.add_child(new_path, default_path, self.root, DirListEntry(True, True, dir_entry.has_known_prefix))) + self.dir_list.append(dir_entry.root.add_child(new_path, default_path, self.root, + DirListEntry(True, True, dir_entry.has_known_prefix))) # Make sure that the default directory subspace still exists after moving the current directory self.ensure_default_directory_subspace(instructions, default_path) # FIXME: There is currently a problem with removing partitions. In these generated tests, it's possible - # for a removed partition to resurrect itself and insert keys into the database using its allocated + # for a removed partition to resurrect itself and insert keys into the database using its allocated # prefix. The result is non-deterministic HCA errors. elif root_op == 'DIRECTORY_REMOVE' or root_op == 'DIRECTORY_REMOVE_IF_EXISTS': # Because allocated prefixes are non-deterministic, we cannot have overlapping @@ -242,7 +247,7 @@ class DirectoryTest(Test): if op.endswith('_DATABASE') and args.concurrency == 1: test_util.blocking_commit(instructions) - path = () + path = () count = random.randint(0, 1) if count == 1: path = generate_path() @@ -256,14 +261,14 @@ class DirectoryTest(Test): self.ensure_default_directory_subspace(instructions, default_path) elif root_op == 'DIRECTORY_LIST' or root_op == 'DIRECTORY_EXISTS': - path = () + path = () count = random.randint(0, 1) if count == 1: path = generate_path() instructions.push_args(*test_util.with_length(path)) instructions.push_args(count) instructions.append(op) - + elif root_op == 'DIRECTORY_PACK_KEY': t = self.random.random_tuple(5) instructions.push_args(*test_util.with_length(t)) @@ -305,10 +310,10 @@ class DirectoryTest(Test): instructions.push_args(self.directory_log.key()) instructions.append('DIRECTORY_LOG_DIRECTORY') if dir_entry.has_known_prefix and dir_entry.is_subspace: - #print '%d. Logging subspace: %d' % (i, dir_entry.dir_id) + # print '%d. Logging subspace: %d' % (i, dir_entry.dir_id) instructions.push_args(self.subspace_log.key()) instructions.append('DIRECTORY_LOG_SUBSPACE') - if (i+1) % 100 == 0: + if (i + 1) % 100 == 0: test_util.blocking_commit(instructions) instructions.push_args(self.stack_subspace.key()) @@ -332,18 +337,21 @@ class DirectoryTest(Test): # If a partition is created, allocates a prefix, and then is removed, subsequent prefix # allocations could collide with prior ones. We can get around this by not allowing # a removed directory (or partition) to be used, but that weakens the test in another way. - #errors += directory_util.check_for_duplicate_prefixes(db, self.prefix_log) + # errors += directory_util.check_for_duplicate_prefixes(db, self.prefix_log) return errors - def get_result_specfications(self): - return [ - ResultSpecification(self.stack, key_start_index=1, ordering_index=1), - ResultSpecification(self.directory_log, ordering_index=0), - ResultSpecification(self.subspace_log, ordering_index=0) + def get_result_specifications(self): + return [ + ResultSpecification(self.stack_subspace, key_start_index=1, ordering_index=1), + ResultSpecification(self.directory_log, ordering_index=0), + ResultSpecification(self.subspace_log, ordering_index=0) ] + # Utility functions -def generate_path(min_length = 0): + + +def generate_path(min_length=0): length = int(random.random() * random.random() * (4 - min_length)) + min_length path = () for i in range(length): @@ -351,9 +359,10 @@ def generate_path(min_length = 0): path = path + (u'',) else: path = path + (random.choice([u'1', u'2', u'3']),) - + return path + def generate_prefix(allow_empty=True, is_partition=False): if allow_empty and random.random() < 0.8: return None @@ -364,7 +373,7 @@ def generate_prefix(allow_empty=True, is_partition=False): if not is_partition: first = chr(random.randint(ord('\x1d'), 255) % 255) - return first + ''.join(chr(random.randrange(0, 256)) for i in range(0, length-1)) + return first + ''.join(chr(random.randrange(0, 256)) for i in range(0, length - 1)) else: return ''.join(chr(random.randrange(ord('\x02'), ord('\x14'))) for i in range(0, length)) else: diff --git a/bindings/bindingtester/tests/directory_hca.py b/bindings/bindingtester/tests/directory_hca.py index f0c05d871c..94fc19debc 100644 --- a/bindings/bindingtester/tests/directory_hca.py +++ b/bindings/bindingtester/tests/directory_hca.py @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -30,6 +30,7 @@ from bindingtester.tests import test_util, directory_util fdb.api_version(FDB_API_VERSION) + class DirectoryHcaTest(Test): def __init__(self, subspace): super(DirectoryHcaTest, self).__init__(subspace) @@ -39,7 +40,7 @@ class DirectoryHcaTest(Test): def setup(self, args): self.random = test_util.RandomGenerator(args.max_int_bits, args.api_version, args.types) - self.transactions = ['tr%d' % i for i in range(3)] # SOMEDAY: parameterize this number? + self.transactions = ['tr%d' % i for i in range(3)] # SOMEDAY: parameterize this number? self.barrier_num = 0 self.max_directories_per_transaction = 30 @@ -58,7 +59,7 @@ class DirectoryHcaTest(Test): def barrier(self, instructions, thread_number, thread_ending=False): if not thread_ending: - instructions.push_args(self.coordination[(self.barrier_num+1)][thread_number].key(), '') + instructions.push_args(self.coordination[(self.barrier_num + 1)][thread_number].key(), '') instructions.append('SET_DATABASE') instructions.append('WAIT_FUTURE') @@ -101,8 +102,9 @@ class DirectoryHcaTest(Test): for i in range(num_directories): path = (self.random.random_unicode_str(16),) - op_args = test_util.with_length(path) + ('', None) - directory_util.push_instruction_and_record_prefix(instructions, 'DIRECTORY_CREATE', op_args, path, num_dirs, self.random, self.prefix_log) + op_args = test_util.with_length(path) + ('', None) + directory_util.push_instruction_and_record_prefix(instructions, 'DIRECTORY_CREATE', + op_args, path, num_dirs, self.random, self.prefix_log) num_dirs += 1 current_op += num_directories @@ -127,4 +129,3 @@ class DirectoryHcaTest(Test): errors += directory_util.validate_hca_state(db) return errors - diff --git a/bindings/bindingtester/tests/directory_util.py b/bindings/bindingtester/tests/directory_util.py index de49ac9957..3a95ed848e 100644 --- a/bindings/bindingtester/tests/directory_util.py +++ b/bindings/bindingtester/tests/directory_util.py @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -34,8 +34,9 @@ DEFAULT_DIRECTORY_INDEX = 4 DEFAULT_DIRECTORY_PREFIX = 'default' DIRECTORY_ERROR_STRING = 'DIRECTORY_ERROR' + class DirListEntry: - dir_id = 0 # Used for debugging + dir_id = 0 # Used for debugging def __init__(self, is_directory, is_subspace, has_known_prefix=True, path=(), root=None): self.root = root or self @@ -53,45 +54,46 @@ class DirListEntry: def add_child(self, subpath, default_path, root, child): if default_path in root.children: - #print 'Adding child %r to default directory %r at %r' % (child, root.children[DirectoryTest.DEFAULT_DIRECTORY_PATH].path, subpath) + # print 'Adding child %r to default directory %r at %r' % (child, root.children[DirectoryTest.DEFAULT_DIRECTORY_PATH].path, subpath) c = root.children[default_path]._add_child_impl(subpath, child) child.has_known_prefix = c.has_known_prefix and child.has_known_prefix - #print 'Added %r' % c + # print 'Added %r' % c - #print 'Adding child %r to directory %r at %r' % (child, self.path, subpath) + # print 'Adding child %r to directory %r at %r' % (child, self.path, subpath) c = self._add_child_impl(subpath, child) - #print 'Added %r' % c + # print 'Added %r' % c return c def _add_child_impl(self, subpath, child): - #print '%d, %d. Adding child (recursive): %s %s' % (self.dir_id, child.dir_id, repr(self.path), repr(subpath)) + # print '%d, %d. Adding child (recursive): %s %s' % (self.dir_id, child.dir_id, repr(self.path), repr(subpath)) if len(subpath) == 0: self.has_known_prefix = self.has_known_prefix and child.has_known_prefix - #print '%d, %d. Setting child: %d' % (self.dir_id, child.dir_id, self.has_known_prefix) + # print '%d, %d. Setting child: %d' % (self.dir_id, child.dir_id, self.has_known_prefix) self._merge_children(child) return self else: if not subpath[0] in self.children: - #print '%d, %d. Path %s was absent (%s)' % (self.dir_id, child.dir_id, repr(self.path + subpath[0:1]), repr(self.children)) - subdir = DirListEntry(True, True, path = self.path+subpath[0:1], root = self.root) + # print '%d, %d. Path %s was absent (%s)' % (self.dir_id, child.dir_id, repr(self.path + subpath[0:1]), repr(self.children)) + subdir = DirListEntry(True, True, path=self.path + subpath[0:1], root=self.root) subdir.has_known_prefix = len(subpath) == 1 self.children[subpath[0]] = subdir else: subdir = self.children[subpath[0]] subdir.has_known_prefix = False - #print '%d, %d. Path was present' % (self.dir_id, child.dir_id) + # print '%d, %d. Path was present' % (self.dir_id, child.dir_id) return subdir._add_child_impl(subpath[1:], child) def _merge_children(self, other): for c in other.children: - if not c in self.children: + if c not in self.children: self.children[c] = other.children[c] else: self.children[c].has_known_prefix = self.children[c].has_known_prefix and other.children[c].has_known_prefix self.children[c]._merge_children(other.children[c]) + def setup_directories(instructions, default_path, random): dir_list = [DirListEntry(True, False, True)] instructions.push_args(0, '\xfe') @@ -114,6 +116,7 @@ def setup_directories(instructions, default_path, random): return dir_list + def create_default_directory_subspace(instructions, path, random): test_util.blocking_commit(instructions) instructions.push_args(3) @@ -125,6 +128,7 @@ def create_default_directory_subspace(instructions, path, random): instructions.push_args(DEFAULT_DIRECTORY_INDEX) instructions.append('DIRECTORY_CHANGE') + def push_instruction_and_record_prefix(instructions, op, op_args, path, dir_index, random, subspace): if not op.endswith('_DATABASE'): instructions.push_args(1, *test_util.with_length(path)) @@ -141,17 +145,18 @@ def push_instruction_and_record_prefix(instructions, op, op_args, path, dir_inde instructions.push_args(1, '', random.random_string(16), '') instructions.append('DIRECTORY_PACK_KEY') - test_util.to_front(instructions, 3) # move the existence result up to the front of the stack + test_util.to_front(instructions, 3) # move the existence result up to the front of the stack t = util.subspace_to_tuple(subspace) instructions.push_args(len(t) + 3, *t) - instructions.append('TUPLE_PACK') # subspace[][][random.random_string(16)] = '' + instructions.append('TUPLE_PACK') # subspace[][][random.random_string(16)] = '' instructions.append('SET') instructions.push_args(DEFAULT_DIRECTORY_INDEX) instructions.append('DIRECTORY_CHANGE') + def check_for_duplicate_prefixes(db, subspace): last_prefix = None start_key = subspace[0].range().start @@ -164,18 +169,19 @@ def check_for_duplicate_prefixes(db, subspace): break start_key = fdb.KeySelector.first_greater_than(prefixes[-1].key) - + prefixes = [subspace[0].unpack(kv.key)[0] for kv in prefixes] prefixes = [p for p in prefixes if not (p.startswith(DEFAULT_DIRECTORY_PREFIX) or p == DIRECTORY_ERROR_STRING)] count += len(prefixes) prefixes = [last_prefix] + prefixes - duplicates.update([p for i,p in enumerate(prefixes[1:]) if p == prefixes[i]]) + duplicates.update([p for i, p in enumerate(prefixes[1:]) if p == prefixes[i]]) last_prefix = prefixes[-1] util.get_logger().info('Checked %d directory prefixes for duplicates' % count) return ['The prefix %r was allocated multiple times' % d[:-2] for d in set(duplicates)] + def validate_hca_state(db): hca = fdb.Subspace(('\xfe', 'hca'), '\xfe') counters = hca[0] @@ -184,7 +190,7 @@ def validate_hca_state(db): last_counter = db.get_range(counters.range().start, counters.range().stop, limit=1, reverse=True) [(start, reported_count)] = [(counters.unpack(kv.key)[0], struct.unpack(' reported_count: return ['The HCA reports %d prefixes allocated in current window, but it actually allocated %d' % (reported_count, actual_count)] diff --git a/bindings/bindingtester/tests/scripted.py b/bindings/bindingtester/tests/scripted.py index 6b9c755bdf..dd768957b9 100644 --- a/bindings/bindingtester/tests/scripted.py +++ b/bindings/bindingtester/tests/scripted.py @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,22 +31,24 @@ from bindingtester.tests import test_util fdb.api_version(FDB_API_VERSION) # SOMEDAY: This should probably be broken up into smaller tests + + class ScriptedTest(Test): - TEST_API_VERSION = 510 + TEST_API_VERSION = 520 def __init__(self, subspace): super(ScriptedTest, self).__init__(subspace, ScriptedTest.TEST_API_VERSION, ScriptedTest.TEST_API_VERSION) self.workspace = self.subspace['workspace'] self.results_subspace = self.subspace['results'] - #self.thread_subspace = self.subspace['threads'] # TODO: update START_THREAD so that we can create threads in subspaces + # self.thread_subspace = self.subspace['threads'] # TODO: update START_THREAD so that we can create threads in subspaces def setup(self, args): if args.concurrency > 1: raise Exception('Scripted tests cannot be run with a concurrency greater than 1') - + # SOMEDAY: this is only a limitation because we don't know how many operations the bisection should start with # it should be fixable. - # + # # We also need to enable the commented out support for num_ops in this file and make it so the default value runs # the entire test if args.bisect: @@ -58,7 +60,7 @@ class ScriptedTest(Test): test_instructions = ThreadedInstructionSet() main_thread = test_instructions.create_thread() - foo = [self.workspace.pack(('foo%d' % i,)) for i in range(0,6)] + foo = [self.workspace.pack(('foo%d' % i,)) for i in range(0, 6)] main_thread.append('NEW_TRANSACTION') main_thread.push_args(1020) @@ -270,8 +272,8 @@ class ScriptedTest(Test): stampKey = 'stampedXXXXXXXXXXsuffix' stampKeyIndex = stampKey.find('XXXXXXXXXX') - stampKeyStr = chr(stampKeyIndex%256) + chr(stampKeyIndex/256) - main_thread.push_args(u'SET_VERSIONSTAMPED_KEY', stampKey + stampKeyStr, 'stampedBar') + stampKeyStr = chr(stampKeyIndex % 256) + chr(stampKeyIndex / 256) + main_thread.push_args(u'SET_VERSIONSTAMPED_KEY', stampKey + stampKeyStr, 'stampedBar') main_thread.append('ATOMIC_OP') main_thread.push_args(u'SET_VERSIONSTAMPED_VALUE', 'stampedValue', 'XXXXXXXXXX') main_thread.append('ATOMIC_OP') @@ -305,7 +307,7 @@ class ScriptedTest(Test): if not args.no_threads: wait_key = 'waitKey' - #threads = [self.thread_subspace[i] for i in range(0, 2)] + # threads = [self.thread_subspace[i] for i in range(0, 2)] threads = ['thread_spec%d' % i for i in range(0, 2)] for thread_spec in threads: main_thread.push_args(self.workspace.pack((wait_key, thread_spec)), '') @@ -314,11 +316,12 @@ class ScriptedTest(Test): for thread_spec in threads: main_thread.push_args(thread_spec) - #if len(main_thread) < args.num_ops: + # if len(main_thread) < args.num_ops: main_thread.append('START_THREAD') thread = test_instructions.create_thread(fdb.Subspace((thread_spec,))) thread.append('NEW_TRANSACTION') - thread.push_args(foo[1], foo[1], 'bar%s' % thread_spec, self.workspace.pack((wait_key, thread_spec)), self.workspace.pack((wait_key, thread_spec))) + thread.push_args(foo[1], foo[1], 'bar%s' % thread_spec, self.workspace.pack( + (wait_key, thread_spec)), self.workspace.pack((wait_key, thread_spec))) thread.append('GET') thread.append('POP') thread.append('SET') @@ -333,20 +336,20 @@ class ScriptedTest(Test): thread.push_args(foo[1]) thread.append('GET') self.add_result(thread, args, 'barthread_spec0', 'barthread_spec1') - + main_thread.append('EMPTY_STACK') - #if len(main_thread) > args.num_ops: - #main_thread[args.num_ops:] = [] + # if len(main_thread) > args.num_ops: + # main_thread[args.num_ops:] = [] return test_instructions def get_result_specifications(self): - return [ - ResultSpecification(self.results_subspace, ordering_index=0, global_error_filter=[1007, 1021]) + return [ + ResultSpecification(self.results_subspace, ordering_index=0, global_error_filter=[1007, 1021]) ] def get_expected_results(self): - return { self.results_subspace : self.results } + return {self.results_subspace: self.results} def append_range_test(self, instructions, args, num_pairs, kv_length): instructions.append('NEW_TRANSACTION') @@ -355,7 +358,7 @@ class ScriptedTest(Test): instructions.append('CLEAR_RANGE_STARTS_WITH') kvpairs = [] - for i in range(0, num_pairs*2): + for i in range(0, num_pairs * 2): kvpairs.append(self.workspace.pack(('foo', ''.join(chr(random.randint(0, 254)) for i in range(0, kv_length))))) kvpairs = list(set(kvpairs)) @@ -364,7 +367,7 @@ class ScriptedTest(Test): kvpairs.sort() instructions.push_args(*kvpairs) - for i in range(0, len(kvpairs)/2): + for i in range(0, len(kvpairs) / 2): instructions.append('SET') if i % 100 == 99: test_util.blocking_commit(instructions) @@ -388,8 +391,7 @@ class ScriptedTest(Test): instructions.push_args(key) instructions.append('SET_DATABASE') - #if len(instructions) <= args.num_ops: + # if len(instructions) <= args.num_ops: self.results.append(Result(self.results_subspace, key, values)) instructions.append('POP') - diff --git a/bindings/bindingtester/tests/test_util.py b/bindings/bindingtester/tests/test_util.py index 854d867afa..5e3c406aa1 100644 --- a/bindings/bindingtester/tests/test_util.py +++ b/bindings/bindingtester/tests/test_util.py @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,6 +31,7 @@ from bindingtester import util from bindingtester import FDB_API_VERSION from bindingtester.known_testers import COMMON_TYPES + class RandomGenerator(object): def __init__(self, max_int_bits=64, api_version=FDB_API_VERSION, types=COMMON_TYPES): self.max_int_bits = max_int_bits @@ -41,13 +42,13 @@ class RandomGenerator(object): return u''.join(self.random_unicode_char() for i in range(0, length)) def random_int(self): - num_bits = random.randint(0, self.max_int_bits) # This way, we test small numbers with higher probability + num_bits = random.randint(0, self.max_int_bits) # This way, we test small numbers with higher probability max_value = (1 << num_bits) - 1 min_value = -max_value - 1 num = random.randint(min_value, max_value) - #util.get_logger().debug('generating int (%d): %d - %s' % (num_bits, num, repr(fdb.tuple.pack((num,))))) + # util.get_logger().debug('generating int (%d): %d - %s' % (num_bits, num, repr(fdb.tuple.pack((num,))))) return num def random_float(self, exp_bits): @@ -57,7 +58,7 @@ class RandomGenerator(object): else: # Choose a value from all over the range of acceptable floats for this precision. sign = -1 if random.random() < 0.5 else 1 - exponent = random.randint(-(1 << (exp_bits-1))-10, (1 << (exp_bits-1) - 1)) + exponent = random.randint(-(1 << (exp_bits - 1)) - 10, (1 << (exp_bits - 1) - 1)) mantissa = random.random() return sign * math.pow(2, exponent) * mantissa @@ -117,12 +118,12 @@ class RandomGenerator(object): smaller_size = random.randint(1, len(to_add)) tuples.append(to_add[:smaller_size]) else: - non_empty = filter(lambda (i,x): (isinstance(x, list) or isinstance(x, tuple)) and len(x) > 0, enumerate(to_add)) + non_empty = filter(lambda (_, x): (isinstance(x, list) or isinstance(x, tuple)) and len(x) > 0, enumerate(to_add)) if len(non_empty) > 0 and random.random() < 0.25: # Add a smaller list to test prefixes of nested structures. idx, choice = random.choice(non_empty) smaller_size = random.randint(0, len(to_add[idx])) - tuples.append(to_add[:idx] + (choice[:smaller_size],) + to_add[idx+1:]) + tuples.append(to_add[:idx] + (choice[:smaller_size],) + to_add[idx + 1:]) random.shuffle(tuples) return tuples @@ -133,7 +134,7 @@ class RandomGenerator(object): elif random.random() < 0.75: limit = 0 else: - limit = random.randint(1e8, (1<<31)-1) + limit = random.randint(1e8, (1 << 31) - 1) return (limit, random.randint(0, 1), random.randint(-2, 4)) @@ -149,13 +150,13 @@ class RandomGenerator(object): if length == 0: return '' - return chr(random.randint(0, 254)) + ''.join(chr(random.randint(0, 255)) for i in range(0, length-1)) + return chr(random.randint(0, 254)) + ''.join(chr(random.randint(0, 255)) for i in range(0, length - 1)) def random_unicode_char(self): while True: if random.random() < 0.05: # Choose one of these special character sequences. - specials = [u'\U0001f4a9', u'\U0001f63c', u'\U0001f3f3\ufe0f\u200d\U0001f308', u'\U0001f1f5\U0001f1f2', u'\uf8ff', + specials = [u'\U0001f4a9', u'\U0001f63c', u'\U0001f3f3\ufe0f\u200d\U0001f308', u'\U0001f1f5\U0001f1f2', u'\uf8ff', u'\U0002a2b2', u'\u05e9\u05dc\u05d5\u05dd'] return random.choice(specials) c = random.randint(0, 0xffff) @@ -166,11 +167,13 @@ class RandomGenerator(object): def error_string(error_code): return fdb.tuple.pack(('ERROR', str(error_code))) + def blocking_commit(instructions): instructions.append('COMMIT') instructions.append('WAIT_FUTURE') instructions.append('RESET') + def to_front(instructions, index): if index == 0: pass @@ -178,19 +181,19 @@ def to_front(instructions, index): instructions.push_args(1) instructions.append('SWAP') elif index == 2: - instructions.push_args(index-1) + instructions.push_args(index - 1) instructions.append('SWAP') instructions.push_args(index) instructions.append('SWAP') else: - instructions.push_args(index-1) + instructions.push_args(index - 1) instructions.append('SWAP') instructions.push_args(index) instructions.append('SWAP') - instructions.push_args(index-1) + instructions.push_args(index - 1) instructions.append('SWAP') - to_front(instructions, index-1) + to_front(instructions, index - 1) + def with_length(tup): return (len(tup),) + tup - diff --git a/bindings/bindingtester/util.py b/bindings/bindingtester/util.py index 486fa5d0cd..1acdecc739 100644 --- a/bindings/bindingtester/util.py +++ b/bindings/bindingtester/util.py @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,6 +25,7 @@ import glob import fdb + def initialize_logger_level(logging_level): logger = get_logger() @@ -39,9 +40,11 @@ def initialize_logger_level(logging_level): elif logging_level == "ERROR": logger.setLevel(logging.ERROR) + def get_logger(): return logging.getLogger('foundationdb.bindingtester') + # Attempts to get the name associated with a process termination signal def signal_number_to_name(signal_num): name = [] @@ -53,6 +56,7 @@ def signal_number_to_name(signal_num): else: return str(signal_num) + def import_subclasses(filename, module_path): for f in glob.glob(os.path.join(os.path.dirname(filename), '*.py')): fn = os.path.basename(f) @@ -60,6 +64,7 @@ def import_subclasses(filename, module_path): continue __import__('%s.%s' % (module_path, os.path.splitext(fn)[0])) + # Attempts to unpack a subspace # This throws an exception if the subspace cannot be unpacked as a tuple # As a result, the binding tester cannot use subspaces that have non-tuple raw prefixes @@ -69,4 +74,3 @@ def subspace_to_tuple(subspace): except Exception as e: get_logger().debug(e) raise Exception('The binding tester does not support subspaces with non-tuple raw prefixes') - diff --git a/bindings/c/ThreadCleanup.cpp b/bindings/c/ThreadCleanup.cpp index 1bee0e6b4f..20b49cf8e5 100644 --- a/bindings/c/ThreadCleanup.cpp +++ b/bindings/c/ThreadCleanup.cpp @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/c/fdb_c.cpp b/bindings/c/fdb_c.cpp index 47833ad21f..a51d51a7e2 100644 --- a/bindings/c/fdb_c.cpp +++ b/bindings/c/fdb_c.cpp @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,7 +18,7 @@ * limitations under the License. */ -#define FDB_API_VERSION 510 +#define FDB_API_VERSION 520 #include "fdbclient/MultiVersionTransaction.h" #include "foundationdb/fdb_c.h" diff --git a/bindings/c/foundationdb/fdb_c.h b/bindings/c/foundationdb/fdb_c.h index e8324318d0..059c88e0a4 100644 --- a/bindings/c/foundationdb/fdb_c.h +++ b/bindings/c/foundationdb/fdb_c.h @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -28,10 +28,10 @@ #endif #if !defined(FDB_API_VERSION) -#error You must #define FDB_API_VERSION prior to including fdb_c.h (current version is 510) +#error You must #define FDB_API_VERSION prior to including fdb_c.h (current version is 520) #elif FDB_API_VERSION < 13 #error API version no longer supported (upgrade to 13) -#elif FDB_API_VERSION > 510 +#elif FDB_API_VERSION > 520 #error Requested API version requires a newer version of this header #endif diff --git a/bindings/c/generate_asm.py b/bindings/c/generate_asm.py index 1a2349b598..a9af7a542b 100755 --- a/bindings/c/generate_asm.py +++ b/bindings/c/generate_asm.py @@ -5,13 +5,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,19 +27,21 @@ import sys functions = {} -func_re = re.compile("^\s*FDB_API_(?:CHANGED|REMOVED)\s*\(\s*([^,]*),\s*([^)]*)\).*") +func_re = re.compile( + "^\s*FDB_API_(?:CHANGED|REMOVED)\s*\(\s*([^,]*),\s*([^)]*)\).*") with open(source, 'r') as srcfile: for l in srcfile: m = func_re.match(l) if m: func, ver = m.groups() - if not func in functions: + if func not in functions: functions[func] = [] ver = int(ver) - if not ver in functions[func]: + if ver not in functions[func]: functions[func].append(ver) + def write_windows_asm(asmfile, functions): asmfile.write(".data\n") for f in functions: @@ -55,6 +57,7 @@ def write_windows_asm(asmfile, functions): asmfile.write("\nEND\n") + def write_unix_asm(asmfile, functions, prefix): asmfile.write(".intel_syntax noprefix\n") @@ -70,13 +73,17 @@ def write_unix_asm(asmfile, functions, prefix): for f in functions: asmfile.write("\n.globl %s%s\n" % (prefix, f)) asmfile.write("%s%s:\n" % (prefix, f)) - asmfile.write("\tmov r11, qword ptr [%sfdb_api_ptr_%s@GOTPCREL+rip]\n" % (prefix, f)) + asmfile.write( + "\tmov r11, qword ptr [%sfdb_api_ptr_%s@GOTPCREL+rip]\n" % (prefix, f)) asmfile.write("\tmov r11, qword ptr [r11]\n") asmfile.write("\tjmp r11\n") + with open(asm, 'w') as asmfile, open(h, 'w') as hfile: - hfile.write("void fdb_api_ptr_unimpl() { fprintf(stderr, \"UNIMPLEMENTED FDB API FUNCTION\\n\"); abort(); }\n\n") - hfile.write("void fdb_api_ptr_removed() { fprintf(stderr, \"REMOVED FDB API FUNCTION\\n\"); abort(); }\n\n") + hfile.write( + "void fdb_api_ptr_unimpl() { fprintf(stderr, \"UNIMPLEMENTED FDB API FUNCTION\\n\"); abort(); }\n\n") + hfile.write( + "void fdb_api_ptr_removed() { fprintf(stderr, \"REMOVED FDB API FUNCTION\\n\"); abort(); }\n\n") if platform == "linux": write_unix_asm(asmfile, functions, '') @@ -90,4 +97,4 @@ with open(asm, 'w') as asmfile, open(h, 'w') as hfile: hfile.write("extern \"C\" ") hfile.write("void* fdb_api_ptr_%s = (void*)&fdb_api_ptr_unimpl;\n" % f) for v in functions[f]: - hfile.write("#define %s_v%d_PREV %s_v%d\n" % (f, v, f, v-1)) + hfile.write("#define %s_v%d_PREV %s_v%d\n" % (f, v, f, v - 1)) diff --git a/bindings/c/local.mk b/bindings/c/local.mk index baa757c16a..7bab3d9919 100644 --- a/bindings/c/local.mk +++ b/bindings/c/local.mk @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -26,6 +26,8 @@ fdb_c_LIBS := lib/libfdbclient.a lib/libfdbrpc.a lib/libflow.a fdb_c_tests_LIBS := -Llib -lfdb_c fdb_c_tests_HEADERS := -Ibindings/c +CLEAN_TARGETS += fdb_c_tests_clean + ifeq ($(PLATFORM),linux) fdb_c_LIBS += lib/libstdc++.a -lm -lpthread -lrt -ldl fdb_c_LDFLAGS += -Wl,--version-script=bindings/c/fdb_c.map -static-libgcc -Wl,-z,nodelete @@ -98,4 +100,7 @@ packages/fdb-c-tests-$(VERSION)-$(PLATFORM).tar.gz: bin/fdb_c_performance_test b fdb_c_tests: packages/fdb-c-tests-$(VERSION)-$(PLATFORM).tar.gz +fdb_c_tests_clean: + @rm -f packages/fdb-c-tests-$(VERSION)-$(PLATFORM).tar.gz + packages: fdb_c_tests diff --git a/bindings/c/test/performance_test.c b/bindings/c/test/performance_test.c index 71eaf1c162..953953c1f6 100644 --- a/bindings/c/test/performance_test.c +++ b/bindings/c/test/performance_test.c @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -602,7 +602,7 @@ void runTests(struct ResultSet *rs) { int main(int argc, char **argv) { srand(time(NULL)); struct ResultSet *rs = newResultSet(); - checkError(fdb_select_api_version(510), "select API version", rs); + checkError(fdb_select_api_version(520), "select API version", rs); printf("Running performance test at client version: %s\n", fdb_get_client_version()); valueStr = (uint8_t*)malloc((sizeof(uint8_t))*valueSize); diff --git a/bindings/c/test/ryw_benchmark.c b/bindings/c/test/ryw_benchmark.c index fb4e7a9cad..e7c291209d 100644 --- a/bindings/c/test/ryw_benchmark.c +++ b/bindings/c/test/ryw_benchmark.c @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -243,7 +243,7 @@ void runTests(struct ResultSet *rs) { int main(int argc, char **argv) { srand(time(NULL)); struct ResultSet *rs = newResultSet(); - checkError(fdb_select_api_version(510), "select API version", rs); + checkError(fdb_select_api_version(520), "select API version", rs); printf("Running RYW Benchmark test at client version: %s\n", fdb_get_client_version()); keys = generateKeys(numKeys, keySize); diff --git a/bindings/c/test/test.h b/bindings/c/test/test.h index 9e91e81337..adf4403640 100644 --- a/bindings/c/test/test.h +++ b/bindings/c/test/test.h @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -27,7 +27,7 @@ #include #ifndef FDB_API_VERSION -#define FDB_API_VERSION 510 +#define FDB_API_VERSION 520 #endif #include diff --git a/bindings/flow/DirectoryLayer.actor.cpp b/bindings/flow/DirectoryLayer.actor.cpp index 242c46f812..c8d26c21e0 100644 --- a/bindings/flow/DirectoryLayer.actor.cpp +++ b/bindings/flow/DirectoryLayer.actor.cpp @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/DirectoryLayer.h b/bindings/flow/DirectoryLayer.h index 778058075b..36ef61c9fb 100755 --- a/bindings/flow/DirectoryLayer.h +++ b/bindings/flow/DirectoryLayer.h @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/DirectoryPartition.h b/bindings/flow/DirectoryPartition.h index 1c3f27de9c..e154806373 100644 --- a/bindings/flow/DirectoryPartition.h +++ b/bindings/flow/DirectoryPartition.h @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/DirectorySubspace.cpp b/bindings/flow/DirectorySubspace.cpp index 27483b4a6e..237d72285f 100755 --- a/bindings/flow/DirectorySubspace.cpp +++ b/bindings/flow/DirectorySubspace.cpp @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/DirectorySubspace.h b/bindings/flow/DirectorySubspace.h index 9990ac7c5f..9a40cbf0b4 100755 --- a/bindings/flow/DirectorySubspace.h +++ b/bindings/flow/DirectorySubspace.h @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/FDBLoanerTypes.h b/bindings/flow/FDBLoanerTypes.h index fc76079634..4797f863c5 100755 --- a/bindings/flow/FDBLoanerTypes.h +++ b/bindings/flow/FDBLoanerTypes.h @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/HighContentionAllocator.actor.cpp b/bindings/flow/HighContentionAllocator.actor.cpp index db6ba09b3b..8a14437dc6 100644 --- a/bindings/flow/HighContentionAllocator.actor.cpp +++ b/bindings/flow/HighContentionAllocator.actor.cpp @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/HighContentionAllocator.h b/bindings/flow/HighContentionAllocator.h index 602bddaf56..37e50a7897 100644 --- a/bindings/flow/HighContentionAllocator.h +++ b/bindings/flow/HighContentionAllocator.h @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/IDirectory.h b/bindings/flow/IDirectory.h index 73ebe94ffb..39256b1661 100755 --- a/bindings/flow/IDirectory.h +++ b/bindings/flow/IDirectory.h @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/Node.actor.cpp b/bindings/flow/Node.actor.cpp index ca93f832b0..27521b9524 100755 --- a/bindings/flow/Node.actor.cpp +++ b/bindings/flow/Node.actor.cpp @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/Subspace.cpp b/bindings/flow/Subspace.cpp index bbc064cbf1..a86ffb25e3 100755 --- a/bindings/flow/Subspace.cpp +++ b/bindings/flow/Subspace.cpp @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/Subspace.h b/bindings/flow/Subspace.h index 542a41d8e1..ffd22ac02c 100755 --- a/bindings/flow/Subspace.h +++ b/bindings/flow/Subspace.h @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/Tuple.cpp b/bindings/flow/Tuple.cpp index f15a7de356..8ce411dce4 100755 --- a/bindings/flow/Tuple.cpp +++ b/bindings/flow/Tuple.cpp @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/Tuple.h b/bindings/flow/Tuple.h index e9ccda9ca2..25867ef71f 100755 --- a/bindings/flow/Tuple.h +++ b/bindings/flow/Tuple.h @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/fdb_flow.actor.cpp b/bindings/flow/fdb_flow.actor.cpp index 9ed7b7bb16..fa3f41b439 100644 --- a/bindings/flow/fdb_flow.actor.cpp +++ b/bindings/flow/fdb_flow.actor.cpp @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -140,8 +140,13 @@ namespace FDB { API::API(int version) : version(version) {} API* API::selectAPIVersion(int apiVersion) { - if(API::instance && apiVersion != API::instance->version) { - throw api_version_already_set(); + if(API::instance) { + if(apiVersion != API::instance->version) { + throw api_version_already_set(); + } + else { + return API::instance; + } } if(apiVersion < 500 || apiVersion > FDB_API_VERSION) { @@ -150,13 +155,23 @@ namespace FDB { throw_on_error( fdb_select_api_version_impl(apiVersion, FDB_API_VERSION) ); - if(!API::instance) { - API::instance = new API(apiVersion); - } - + API::instance = new API(apiVersion); return API::instance; } + bool API::isAPIVersionSelected() { + return API::instance != NULL; + } + + API* API::getInstance() { + if(API::instance == NULL) { + throw api_version_unset(); + } + else { + return API::instance; + } + } + void API::setupNetwork() { throw_on_error( fdb_setup_network() ); } @@ -183,6 +198,10 @@ namespace FDB { return Reference( new Cluster(c) ); } + int API::getAPIVersion() const { + return version; + } + Reference Cluster::createDatabase() { const char *dbName = "DB"; CFuture f( fdb_cluster_create_database( c, (uint8_t*)dbName, (int)strlen(dbName) ) ); diff --git a/bindings/flow/fdb_flow.h b/bindings/flow/fdb_flow.h index 2cc148c6e1..33c195208d 100644 --- a/bindings/flow/fdb_flow.h +++ b/bindings/flow/fdb_flow.h @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -23,7 +23,7 @@ #include -#define FDB_API_VERSION 510 +#define FDB_API_VERSION 520 #include #undef DLLEXPORT @@ -64,6 +64,8 @@ namespace FDB { class API { public: static API* selectAPIVersion(int apiVersion); + static API* getInstance(); + static bool isAPIVersionSelected(); void setNetworkOption(FDBNetworkOption option, Optional value = Optional()); @@ -74,6 +76,7 @@ namespace FDB { Reference createCluster( std::string const& connFilename ); bool evaluatePredicate(FDBErrorPredicate pred, Error const& e); + int getAPIVersion() const; private: static API* instance; diff --git a/bindings/flow/local.mk b/bindings/flow/local.mk index f7927000b0..7299e6a22a 100644 --- a/bindings/flow/local.mk +++ b/bindings/flow/local.mk @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/tester/DirectoryTester.actor.cpp b/bindings/flow/tester/DirectoryTester.actor.cpp index 713359a2ef..db99f6a635 100644 --- a/bindings/flow/tester/DirectoryTester.actor.cpp +++ b/bindings/flow/tester/DirectoryTester.actor.cpp @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/tester/Tester.actor.cpp b/bindings/flow/tester/Tester.actor.cpp index eb89079ad9..c62dd08a22 100644 --- a/bindings/flow/tester/Tester.actor.cpp +++ b/bindings/flow/tester/Tester.actor.cpp @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -1510,6 +1510,25 @@ struct UnitTestsFunc : InstructionFunc { ASSERT(data->api->evaluatePredicate(FDBErrorPredicate::FDB_ERROR_PREDICATE_RETRYABLE, Error(1020))); ASSERT(!data->api->evaluatePredicate(FDBErrorPredicate::FDB_ERROR_PREDICATE_RETRYABLE, Error(10))); + ASSERT(API::isAPIVersionSelected()); + state API *fdb = API::getInstance(); + ASSERT(fdb->getAPIVersion() <= FDB_API_VERSION); + try { + API::selectAPIVersion(fdb->getAPIVersion() + 1); + ASSERT(false); + } + catch(Error &e) { + ASSERT(e.code() == error_code_api_version_already_set); + } + try { + API::selectAPIVersion(fdb->getAPIVersion() - 1); + ASSERT(false); + } + catch(Error &e) { + ASSERT(e.code() == error_code_api_version_already_set); + } + API::selectAPIVersion(fdb->getAPIVersion()); + state Reference tr(new Transaction(data->db)); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_PRIORITY_SYSTEM_IMMEDIATE); tr->setOption(FDBTransactionOption::FDB_TR_OPTION_PRIORITY_SYSTEM_IMMEDIATE); @@ -1574,7 +1593,7 @@ ACTOR static Future doInstructions(Reference data) { try { if(LOG_INSTRUCTIONS) { if(op != LiteralStringRef("SWAP") && op != LiteralStringRef("PUSH")) { - printf("%lu. %s\n", idx, tupleToString(opTuple).c_str()); + printf("%zu. %s\n", idx, tupleToString(opTuple).c_str()); fflush(stdout); } } @@ -1654,6 +1673,7 @@ void populateAtomicOpMap() { optionInfo["BIT_OR"] = FDBMutationType::FDB_MUTATION_TYPE_BIT_OR; optionInfo["XOR"] = FDBMutationType::FDB_MUTATION_TYPE_XOR; optionInfo["BIT_XOR"] = FDBMutationType::FDB_MUTATION_TYPE_BIT_XOR; + optionInfo["APPEND_IF_FITS"] = FDBMutationType::FDB_MUTATION_TYPE_APPEND_IF_FITS; optionInfo["MAX"] = FDBMutationType::FDB_MUTATION_TYPE_MAX; optionInfo["MIN"] = FDBMutationType::FDB_MUTATION_TYPE_MIN; optionInfo["SET_VERSIONSTAMPED_KEY"] = FDBMutationType::FDB_MUTATION_TYPE_SET_VERSIONSTAMPED_KEY; @@ -1681,7 +1701,18 @@ ACTOR void startTest(std::string clusterFilename, StringRef prefix, int apiVersi // This is "our" network g_network = newNet2(NetworkAddress(), false); + ASSERT(!API::isAPIVersionSelected()); + try { + API::getInstance(); + ASSERT(false); + } + catch(Error& e) { + ASSERT(e.code() == error_code_api_version_unset); + } + API *fdb = API::selectAPIVersion(apiVersion); + ASSERT(API::isAPIVersionSelected()); + ASSERT(fdb->getAPIVersion() == apiVersion); //fdb->setNetworkOption(FDBNetworkOption::FDB_NET_OPTION_TRACE_ENABLE); // We have to start the fdb_flow network and thread separately! diff --git a/bindings/flow/tester/Tester.actor.h b/bindings/flow/tester/Tester.actor.h index 1c320b6670..13fd8b9285 100644 --- a/bindings/flow/tester/Tester.actor.h +++ b/bindings/flow/tester/Tester.actor.h @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/flow/tester/local.mk b/bindings/flow/tester/local.mk index 695e98bea9..2ef4fcb753 100644 --- a/bindings/flow/tester/local.mk +++ b/bindings/flow/tester/local.mk @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. diff --git a/bindings/go/README.md b/bindings/go/README.md index d06a5dc656..a7c75ef0be 100644 --- a/bindings/go/README.md +++ b/bindings/go/README.md @@ -1,14 +1,14 @@ fdb-go ====== -[Go language](http://golang.org) bindings for [FoundationDB](http://foundationdb.org/documentation/), a distributed key-value store with ACID transactions. +[Go language](http://golang.org) bindings for [FoundationDB](https://www.foundationdb.org/documentation/), a distributed key-value store with ACID transactions. This package requires: - Go 1.1+ with CGO enabled -- FoundationDB C API 2.0.x, 3.0.x, or 4.x.y (part of the [FoundationDB clients package](https://files.foundationdb.org/fdb-c/)) +- FoundationDB C API 2.0.x, 3.0.x, or 4.x.y (part of the [FoundationDB clients package](https://www.foundationdb.org/downloads/fdb-c/)) -Use of this package requires the selection of a FoundationDB API version at runtime. This package currently supports FoundationDB API versions 200-510. +Use of this package requires the selection of a FoundationDB API version at runtime. This package currently supports FoundationDB API versions 200-520. To build this package, in the top level of this repository run: @@ -27,5 +27,5 @@ of downloading from the remote repository. Documentation ------------- -* [API documentation](https://foundationdb.org/documentation/godoc/fdb.html) -* [Tutorial](https://foundationdb.org/documentation/class-scheduling-go.html) +* [API documentation](https://www.foundationdb.org/documentation/godoc/fdb.html) +* [Tutorial](https://www.foundationdb.org/documentation/class-scheduling-go.html) diff --git a/bindings/go/include.mk b/bindings/go/include.mk index bfaa2a0334..d56669b084 100644 --- a/bindings/go/include.mk +++ b/bindings/go/include.mk @@ -4,13 +4,13 @@ # This source file is part of the FoundationDB open source project # # Copyright 2013-2018 Apple Inc. and the FoundationDB project authors -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,7 +25,7 @@ GOPATH := $(CURDIR)/bindings/go/build GO_IMPORT_PATH := github.com/apple/foundationdb/bindings/go/src GO_DEST := $(GOPATH)/src/$(GO_IMPORT_PATH) -.PHONY: fdb_go fdb_go_path fdb_go_tester fdb_go_tester_clean godoc godoc_clean +.PHONY: fdb_go fdb_go_path fdb_go_fmt fdb_go_fmt_check fdb_go_tester fdb_go_tester_clean godoc godoc_clean # We only override if the environment didn't set it (this is used by # the fdbwebsite documentation build process) @@ -49,12 +49,23 @@ GO_PACKAGE_OBJECTS := $(addprefix $(GO_PACKAGE_OUTDIR)/,$(GO_PACKAGES:=.a)) GO_SRC := $(shell find $(CURDIR)/bindings/go/src -name '*.go') -fdb_go: $(GO_PACKAGE_OBJECTS) $(GO_SRC) +fdb_go: $(GO_PACKAGE_OBJECTS) $(GO_SRC) fdb_go_fmt_check -fdb_go_path: $(GO_SRC) +fdb_go_fmt: $(GO_SRC) + @echo "Formatting fdb_go" + @gofmt -w $(GO_SRC) + +fdb_go_fmt_check: $(GO_SRC) + @echo "Checking fdb_go" + @bash -c 'fmtoutstr=$$(gofmt -l $(GO_SRC)) ; if [[ -n "$${fmtoutstr}" ]] ; then echo "Detected go formatting violations for the following files:" ; echo "$${fmtoutstr}" ; echo "Try running: make fdb_go_fmt"; exit 1 ; fi' + +$(GO_DEST)/.stamp: $(GO_SRC) @echo "Creating fdb_go_path" @mkdir -p $(GO_DEST) @cp -r bindings/go/src/* $(GO_DEST) + @touch $(GO_DEST)/.stamp + +fdb_go_path: $(GO_DEST)/.stamp fdb_go_clean: @echo "Cleaning fdb_go" @@ -66,31 +77,31 @@ fdb_go_tester_clean: @echo "Cleaning fdb_go_tester" @rm -rf $(GOPATH)/bin -$(GOPATH)/bin/_stacktester: fdb_go_path $(GO_SRC) $(GO_PACKAGE_OBJECTS) $(GO_DEST)/fdb/generated.go +$(GOPATH)/bin/_stacktester: $(GO_DEST)/.stamp $(GO_SRC) $(GO_PACKAGE_OBJECTS) $(GO_DEST)/fdb/generated.go @echo "Compiling $(basename $(notdir $@))" @go install $(GO_IMPORT_PATH)/_stacktester -$(GO_PACKAGE_OUTDIR)/fdb/tuple.a: fdb_go_path $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_DEST)/fdb/generated.go +$(GO_PACKAGE_OUTDIR)/fdb/tuple.a: $(GO_DEST)/.stamp $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_DEST)/fdb/generated.go @echo "Compiling fdb/tuple" @go install $(GO_IMPORT_PATH)/fdb/tuple -$(GO_PACKAGE_OUTDIR)/fdb/subspace.a: fdb_go_path $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_PACKAGE_OUTDIR)/fdb/tuple.a $(GO_DEST)/fdb/generated.go +$(GO_PACKAGE_OUTDIR)/fdb/subspace.a: $(GO_DEST)/.stamp $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_PACKAGE_OUTDIR)/fdb/tuple.a $(GO_DEST)/fdb/generated.go @echo "Compiling fdb/subspace" @go install $(GO_IMPORT_PATH)/fdb/subspace -$(GO_PACKAGE_OUTDIR)/fdb/directory.a: fdb_go_path $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_PACKAGE_OUTDIR)/fdb/tuple.a $(GO_PACKAGE_OUTDIR)/fdb/subspace.a $(GO_DEST)/fdb/generated.go +$(GO_PACKAGE_OUTDIR)/fdb/directory.a: $(GO_DEST)/.stamp $(GO_SRC) $(GO_PACKAGE_OUTDIR)/fdb.a $(GO_PACKAGE_OUTDIR)/fdb/tuple.a $(GO_PACKAGE_OUTDIR)/fdb/subspace.a $(GO_DEST)/fdb/generated.go @echo "Compiling fdb/directory" @go install $(GO_IMPORT_PATH)/fdb/directory -$(GO_PACKAGE_OUTDIR)/fdb.a: fdb_go_path $(GO_SRC) $(GO_DEST)/fdb/generated.go +$(GO_PACKAGE_OUTDIR)/fdb.a: $(GO_DEST)/.stamp lib/libfdb_c.$(DLEXT) $(GO_SRC) $(GO_DEST)/fdb/generated.go @echo "Compiling fdb" @go install $(GO_IMPORT_PATH)/fdb -$(GO_DEST)/fdb/generated.go: fdb_go_path lib/libfdb_c.$(DLEXT) bindings/go/src/_util/translate_fdb_options.go fdbclient/vexillographer/fdb.options +$(GO_DEST)/fdb/generated.go: $(GO_DEST)/.stamp bindings/go/src/_util/translate_fdb_options.go fdbclient/vexillographer/fdb.options @echo "Building $@" @go run bindings/go/src/_util/translate_fdb_options.go < fdbclient/vexillographer/fdb.options > $@ -godoc: fdb_go_path $(GO_SRC) +godoc: fdb_go_path $(GO_SRC) $(GO_DEST)/fdb/generated.go @echo "Generating Go Documentation" @rm -rf $(GODOC_DIR)/godoc @mkdir -p $(GODOC_DIR)/godoc @@ -105,6 +116,12 @@ godoc: fdb_go_path $(GO_SRC) @(sed -i -e 's_a href="tuple/"_a href="fdb.tuple.html"_' $(GODOC_DIR)/godoc/fdb.html) @(sed -i -e 's_a href="subspace/"_a href="fdb.subspace.html"_' $(GODOC_DIR)/godoc/fdb.html) @(sed -i -e 's_a href="directory/"_a href="fdb.directory.html"_' $(GODOC_DIR)/godoc/fdb.html) + + @(sed -i -e 's_a href="/pkg/builtin_a href="https://godoc.org/pkg/builtin_g;s_a href="/src/github.com/apple/foundationdb_a href="https://github.com/apple/foundationdb/tree/master_g;s_a href="/pkg/github.com/apple/foundationdb/bindings/go/src/fdb/_a href="./fdb.html_g' $(GODOC_DIR)/godoc/fdb.html) + @(sed -i -e 's_a href="/pkg/builtin_a href="https://godoc.org/pkg/builtin_g;s_a href="/src/github.com/apple/foundationdb_a href="https://github.com/apple/foundationdb/tree/master_g;s_a href="/pkg/github.com/apple/foundationdb/bindings/go/src/fdb/_a href="./fdb.html_g' $(GODOC_DIR)/godoc/fdb.directory.html) + @(sed -i -e 's_a href="/pkg/builtin_a href="https://godoc.org/pkg/builtin_g;s_a href="/src/github.com/apple/foundationdb_a href="https://github.com/apple/foundationdb/tree/master_g;s_a href="/pkg/github.com/apple/foundationdb/bindings/go/src/fdb/_a href="./fdb.html_g' $(GODOC_DIR)/godoc/fdb.subspace.html) + @(sed -i -e 's_a href="/pkg/builtin_a href="https://godoc.org/pkg/builtin_g;s_a href="/src/github.com/apple/foundationdb_a href="https://github.com/apple/foundationdb/tree/master_g;s_a href="/pkg/github.com/apple/foundationdb/bindings/go/src/fdb/_a href="./fdb.html_g' $(GODOC_DIR)/godoc/fdb.tuple.html) + godoc_clean: @echo "Cleaning Go Documentation" diff --git a/bindings/go/src/_stacktester/directory.go b/bindings/go/src/_stacktester/directory.go index fa66205fde..3b71173792 100644 --- a/bindings/go/src/_stacktester/directory.go +++ b/bindings/go/src/_stacktester/directory.go @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -21,12 +21,12 @@ package main import ( - "github.com/apple/foundationdb/bindings/go/src/fdb" - "github.com/apple/foundationdb/bindings/go/src/fdb/tuple" - "github.com/apple/foundationdb/bindings/go/src/fdb/subspace" - "github.com/apple/foundationdb/bindings/go/src/fdb/directory" - "strings" "bytes" + "github.com/apple/foundationdb/bindings/go/src/fdb" + "github.com/apple/foundationdb/bindings/go/src/fdb/directory" + "github.com/apple/foundationdb/bindings/go/src/fdb/subspace" + "github.com/apple/foundationdb/bindings/go/src/fdb/tuple" + "strings" ) func (sm *StackMachine) popTuples(count int) []tuple.Tuple { @@ -60,8 +60,8 @@ func tuplePackStrings(s []string) []byte { } type DirectoryExtension struct { - list []interface{} - index int64 + list []interface{} + index int64 errorIndex int64 } @@ -93,15 +93,15 @@ func (sm *StackMachine) maybePath() []string { return path } -var createOps = map[string]bool { +var createOps = map[string]bool{ "CREATE_SUBSPACE": true, - "CREATE_LAYER": true, - "CREATE_OR_OPEN": true, - "CREATE": true, - "OPEN": true, - "MOVE": true, - "MOVE_TO": true, - "OPEN_SUBSPACE": true, + "CREATE_LAYER": true, + "CREATE_OR_OPEN": true, + "CREATE": true, + "OPEN": true, + "MOVE": true, + "MOVE_TO": true, + "OPEN_SUBSPACE": true, } func (de *DirectoryExtension) processOp(sm *StackMachine, op string, isDB bool, idx int, t fdb.Transactor, rt fdb.ReadTransactor) { @@ -142,7 +142,9 @@ func (de *DirectoryExtension) processOp(sm *StackMachine, op string, isDB bool, layer = l.([]byte) } d, e := de.cwd().CreateOrOpen(t, tupleToPath(tuples[0]), layer) - if e != nil { panic(e) } + if e != nil { + panic(e) + } de.store(d) case op == "CREATE": tuples := sm.popTuples(1) @@ -159,7 +161,9 @@ func (de *DirectoryExtension) processOp(sm *StackMachine, op string, isDB bool, // p.([]byte) itself may be nil, but CreatePrefix handles that appropriately d, e = de.cwd().CreatePrefix(t, tupleToPath(tuples[0]), layer, p.([]byte)) } - if e != nil { panic(e) } + if e != nil { + panic(e) + } de.store(d) case op == "OPEN": tuples := sm.popTuples(1) @@ -169,7 +173,9 @@ func (de *DirectoryExtension) processOp(sm *StackMachine, op string, isDB bool, layer = l.([]byte) } d, e := de.cwd().Open(rt, tupleToPath(tuples[0]), layer) - if e != nil { panic(e) } + if e != nil { + panic(e) + } de.store(d) case op == "CHANGE": i := sm.waitAndPop().item.(int64) @@ -182,12 +188,16 @@ func (de *DirectoryExtension) processOp(sm *StackMachine, op string, isDB bool, case op == "MOVE": tuples := sm.popTuples(2) d, e := de.cwd().Move(t, tupleToPath(tuples[0]), tupleToPath(tuples[1])) - if e != nil { panic(e) } + if e != nil { + panic(e) + } de.store(d) case op == "MOVE_TO": tuples := sm.popTuples(1) d, e := de.cwd().MoveTo(t, tupleToPath(tuples[0])) - if e != nil { panic(e) } + if e != nil { + panic(e) + } de.store(d) case strings.HasPrefix(op, "REMOVE"): path := sm.maybePath() @@ -197,9 +207,11 @@ func (de *DirectoryExtension) processOp(sm *StackMachine, op string, isDB bool, // doesn't end up committing the version key. (Other languages have // separate remove() and remove_if_exists() so don't have this tricky // issue). - _, e := t.Transact(func (tr fdb.Transaction) (interface{}, error) { + _, e := t.Transact(func(tr fdb.Transaction) (interface{}, error) { ok, e := de.cwd().Remove(tr, path) - if e != nil { panic(e) } + if e != nil { + panic(e) + } switch op[6:] { case "": if !ok { @@ -209,16 +221,24 @@ func (de *DirectoryExtension) processOp(sm *StackMachine, op string, isDB bool, } return nil, nil }) - if e != nil { panic(e) } + if e != nil { + panic(e) + } case op == "LIST": subs, e := de.cwd().List(rt, sm.maybePath()) - if e != nil { panic(e) } + if e != nil { + panic(e) + } t := make(tuple.Tuple, len(subs)) - for i, s := range subs { t[i] = s } + for i, s := range subs { + t[i] = s + } sm.store(idx, t.Pack()) case op == "EXISTS": b, e := de.cwd().Exists(rt, sm.maybePath()) - if e != nil { panic(e) } + if e != nil { + panic(e) + } if b { sm.store(idx, int64(1)) } else { @@ -229,8 +249,10 @@ func (de *DirectoryExtension) processOp(sm *StackMachine, op string, isDB bool, sm.store(idx, de.css().Pack(tuples[0])) case op == "UNPACK_KEY": t, e := de.css().Unpack(fdb.Key(sm.waitAndPop().item.([]byte))) - if e != nil { panic(e) } - for _, el := range(t) { + if e != nil { + panic(e) + } + for _, el := range t { sm.store(idx, el) } case op == "RANGE": @@ -252,7 +274,7 @@ func (de *DirectoryExtension) processOp(sm *StackMachine, op string, isDB bool, k := sm.waitAndPop().item.([]byte) k = append(k, tuple.Tuple{de.index}.Pack()...) v := de.css().Bytes() - t.Transact(func (tr fdb.Transaction) (interface{}, error) { + t.Transact(func(tr fdb.Transaction) (interface{}, error) { tr.Set(fdb.Key(k), v) return nil, nil }) @@ -266,7 +288,9 @@ func (de *DirectoryExtension) processOp(sm *StackMachine, op string, isDB bool, k3 := ss.Pack(tuple.Tuple{"exists"}) var v3 []byte exists, e := de.cwd().Exists(rt, nil) - if e != nil { panic(e) } + if e != nil { + panic(e) + } if exists { v3 = tuple.Tuple{1}.Pack() } else { @@ -276,10 +300,12 @@ func (de *DirectoryExtension) processOp(sm *StackMachine, op string, isDB bool, var subs []string if exists { subs, e = de.cwd().List(rt, nil) - if e != nil { panic(e) } + if e != nil { + panic(e) + } } v4 := tuplePackStrings(subs) - t.Transact(func (tr fdb.Transaction) (interface{}, error) { + t.Transact(func(tr fdb.Transaction) (interface{}, error) { tr.Set(k1, v1) tr.Set(k2, v2) tr.Set(k3, v3) diff --git a/bindings/go/src/_stacktester/stacktester.go b/bindings/go/src/_stacktester/stacktester.go index 999269656e..a7dc4fd88d 100644 --- a/bindings/go/src/_stacktester/stacktester.go +++ b/bindings/go/src/_stacktester/stacktester.go @@ -4,13 +4,13 @@ * This source file is part of the FoundationDB open source project * * Copyright 2013-2018 Apple Inc. and the FoundationDB project authors - * + * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at - * + * * http://www.apache.org/licenses/LICENSE-2.0 - * + * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -24,23 +24,23 @@ import ( "bytes" "encoding/binary" "encoding/hex" + "fmt" "github.com/apple/foundationdb/bindings/go/src/fdb" "github.com/apple/foundationdb/bindings/go/src/fdb/tuple" "log" - "fmt" "os" + "reflect" + "runtime" "sort" + "strconv" "strings" "sync" - "runtime" - "reflect" "time" - "strconv" ) const verbose bool = false -var trMap = map[string]fdb.Transaction {} +var trMap = map[string]fdb.Transaction{} var trMapLock = sync.RWMutex{} // Make tuples sortable by byte-order @@ -69,17 +69,17 @@ func int64ToBool(i int64) bool { type stackEntry struct { item interface{} - idx int + idx int } type StackMachine struct { - prefix []byte - trName string - stack []stackEntry + prefix []byte + trName string + stack []stackEntry lastVersion int64 - threads sync.WaitGroup - verbose bool - de *DirectoryExtension + threads sync.WaitGroup + verbose bool + de *DirectoryExtension } func newStackMachine(prefix []byte, verbose bool) *StackMachine { @@ -99,7 +99,7 @@ func (sm *StackMachine) waitAndPop() (ret stackEntry) { } }() - ret, sm.stack = sm.stack[len(sm.stack) - 1], sm.stack[:len(sm.stack) - 1] + ret, sm.stack = sm.stack[len(sm.stack)-1], sm.stack[:len(sm.stack)-1] switch el := ret.item.(type) { case []byte: ret.item = el @@ -150,9 +150,9 @@ func (sm *StackMachine) popPrefixRange() fdb.ExactRange { } func (sm *StackMachine) pushRange(idx int, sl []fdb.KeyValue, prefixFilter []byte) { - var t tuple.Tuple = make(tuple.Tuple, 0, len(sl) * 2) + var t tuple.Tuple = make(tuple.Tuple, 0, len(sl)*2) - for _, kv := range(sl) { + for _, kv := range sl { if prefixFilter == nil || bytes.HasPrefix(kv.Key, prefixFilter) { t = append(t, kv.Key) t = append(t, kv.Value) @@ -240,7 +240,7 @@ func (sm *StackMachine) dumpStack() { } } -func (sm *StackMachine) executeMutation(t fdb.Transactor, f func (fdb.Transaction) (interface{}, error), isDB bool, idx int) { +func (sm *StackMachine) executeMutation(t fdb.Transactor, f func(fdb.Transaction) (interface{}, error), isDB bool, idx int) { _, e := t.Transact(f) if e != nil { panic(e) @@ -250,8 +250,8 @@ func (sm *StackMachine) executeMutation(t fdb.Transactor, f func (fdb.Transactio } } -func (sm *StackMachine) checkWatches(watches [4]fdb.FutureNil, expected bool) (bool) { - for _, watch := range(watches) { +func (sm *StackMachine) checkWatches(watches [4]fdb.FutureNil, expected bool) bool { + for _, watch := range watches { if watch.IsReady() || expected { e := watch.Get() if e != nil { @@ -283,7 +283,9 @@ func (sm *StackMachine) testWatches() { tr.Set(fdb.Key("w3"), []byte("3")) return nil, nil }) - if e != nil { panic(e) } + if e != nil { + panic(e) + } var watches [4]fdb.FutureNil @@ -297,7 +299,9 @@ func (sm *StackMachine) testWatches() { tr.Clear(fdb.Key("w1")) return nil, nil }) - if e != nil { panic(e) } + if e != nil { + panic(e) + } time.Sleep(5 * time.Second) @@ -312,7 +316,9 @@ func (sm *StackMachine) testWatches() { tr.BitXor(fdb.Key("w3"), []byte("\xff\xff")) return nil, nil }) - if e != nil { panic(e) } + if e != nil { + panic(e) + } if sm.checkWatches(watches, true) { return @@ -322,21 +328,23 @@ func (sm *StackMachine) testWatches() { func (sm *StackMachine) testLocality() { _, e := db.Transact(func(tr fdb.Transaction) (interface{}, error) { - tr.Options().SetTimeout(60*1000) + tr.Options().SetTimeout(60 * 1000) tr.Options().SetReadSystemKeys() boundaryKeys, e := db.LocalityGetBoundaryKeys(fdb.KeyRange{fdb.Key(""), fdb.Key("\xff\xff")}, 0, 0) - if e != nil { panic(e) } + if e != nil { + panic(e) + } - for i:=0; i