foundationdb/contrib/apiversioner.py

#!/usr/bin/env python3
#
# alloc_instrumentation.py
#
# This source file is part of the FoundationDB open source project
#
# Copyright 2013-2021 Apple Inc. and the FoundationDB project authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

import argparse
import logging
import os
import re
import sys
import traceback


LOG_FORMAT = '%(created)f [%(levelname)s] %(message)s'

EXCLUDED_FILES = list(map(re.compile, [
    # Output directories
    r'\.git/.*', r'bin/.*', r'packages/.*', r'\.objs/.*', r'\.deps/.*', r'bindings/go/build/.*', r'documentation/sphinx/\.out/.*',

    # Generated files
    r'.*\.g\.cpp$', r'.*\.g\.h$', r'(^|.*/)generated.mk$', r'.*\.g\.S$',
    r'.*/MutationType\.java', r'.*/generated\.go',

    # Binary files
    r'.*\.class$', r'.*\.o$', r'.*\.a$', r'.*[\.-]debug', r'.*\.so$', r'.*\.dylib$', r'.*\.dll$', r'.*\.tar[^/]*$', r'.*\.jar$', r'.*pyc$', r'bindings/flow/bin/.*',
    r'.*\.pdf$', r'.*\.jp[e]*g', r'.*\.png', r'.*\.ico',
    r'packaging/msi/art/.*',

    # Project configuration files
    r'.*foundationdb\.VC\.db$', r'.*foundationdb\.VC\.VC\.opendb$', r'.*iml$',

    # Source files from someone else
    r'(^|.*/)Hash3\..*', r'(^|.*/)sqlite.*',
    r'bindings/go/godoc-resources/.*',
    r'bindings/go/src/fdb/tuple/testdata/tuples.golden',
    r'fdbcli/linenoise/.*',
    r'fdbrpc/rapidjson/.*', r'fdbrpc/rapidxml/.*', r'fdbrpc/zlib/.*', r'fdbrpc/sha1/.*',
    r'fdbrpc/xml2json.hpp$', r'fdbrpc/libcoroutine/.*', r'fdbrpc/libeio/.*', r'fdbrpc/lib64/.*',
    r'fdbrpc/generated-constants.cpp$',

    # Miscellaneous
    r'bindings/nodejs/node_modules/.*', r'bindings/go/godoc/.*', r'.*trace.*xml$', r'.*log$', r'.*\.DS_Store$', r'simfdb/\.*', r'.*~$', r'.*.swp$'
]))

SUSPECT_PHRASES = map(re.compile, [
    r'#define\s+FDB_API_VERSION\s+(\d+)',
    r'\.\s*selectApiVersion\s*\(\s*(\d+)\s*\)',
    r'\.\s*APIVersion\s*\(\s*(\d+)\s*\)',
    r'\.\s*MustAPIVersion\s*\(\s*(\d+)\s*\)',
    r'header_version\s+=\s+(\d+)',
    r'\.\s*apiVersion\s*\(\s*(\d+)\s*\)',
    r'API_VERSION\s*=\s*(\d+)',
    r'fdb_select_api_version\s*\((\d+)\)'
])

DIM_CODE = '\033[2m'
BOLD_CODE = '\033[1m'
RED_COLOR = '\033[91m'
GREEN_COLOR = '\033[92m'
END_COLOR = '\033[0m'


def positive_response(val):
    return val.lower() in {'y', 'yes'}


# Returns: new line list + a dirty flag
def rewrite_lines(lines, version_re, new_version, suspect_only=True, print_diffs=False, ask_confirm=False, grayscale=False):
    new_lines = []
    dirty = False
    new_str = str(new_version)
    regexes = SUSPECT_PHRASES if suspect_only else [version_re]
    group_index = 1 if suspect_only else 2
    for line_no, line in enumerate(lines):
        new_line = line
        offset = 0

        for regex in regexes:
            for m in regex.finditer(line):
                # Replace suspect code with new version.
                start = m.start(group_index)
                end = m.end(group_index)
                new_line = new_line[:start + offset] + new_str + new_line[end + offset:]
                offset += len(new_str) - (end - start)

        if (print_diffs or ask_confirm) and line != new_line:
            print('Rewrite:')
            print('\n'.join(map(lambda pair: ' {:4d}: {}'.format(line_no - 1 + pair[0], pair[1]), enumerate(lines[line_no - 2:line_no]))))
            print((DIM_CODE if grayscale else RED_COLOR) + '-{:4d}: {}'.format(line_no + 1, line) + END_COLOR)
            print((BOLD_CODE if grayscale else GREEN_COLOR) + '+{:4d}: {}'.format(line_no + 1, new_line) + END_COLOR)
            print('\n'.join(map(lambda pair: ' {:4d}: {}'.format(line_no + 2 + pair[0], pair[1]), enumerate(lines[line_no + 1:line_no + 3]))))

            if ask_confirm:
                text = input('Looks good (y/n)? ')
                if not positive_response(text):
                    print('Okay, skipping.')
                    new_line = line

        dirty = dirty or (new_line != line)
        new_lines.append(new_line)

    return new_lines, dirty


def address_file(base_path, file_path, version, new_version=None, suspect_only=False, show_diffs=False,
                 rewrite=False, ask_confirm=True, grayscale=False, paths_only=False):
    if any(map(lambda x: x.match(file_path), EXCLUDED_FILES)):
        logging.debug('skipping file %s as matches excluded list', file_path)
        return True

    # Look for all instances of the version number where it is not part of a larger number
    version_re = re.compile('(^|[^\\d])(' + str(version) + ')([^\\d]|$)')
    try:
        contents = open(os.path.join(base_path, file_path), 'r').read()
        lines = contents.split('\n')
        new_lines = lines
        dirty = False

        if suspect_only:
            # Look for suspect lines (lines that attempt to set a version)
            found = False
            for line_no, line in enumerate(lines):
                for suspect_phrase in SUSPECT_PHRASES:
                    for match in suspect_phrase.finditer(line):
                        curr_version = int(match.groups()[0])
                        if (new_version is None and curr_version < version) or (new_version is not None and curr_version < new_version):
                            found = True
                            logging.info('Old version: %s:%d:%s', file_path, line_no + 1, line)

            if found and new_version is not None and (show_diffs or rewrite):
                new_lines, dirty = rewrite_lines(lines, version_re, new_version, True, print_diffs=True,
                                                 ask_confirm=(rewrite and ask_confirm), grayscale=grayscale)

        else:
            matching_lines = filter(lambda pair: version_re.search(pair[1]), enumerate(lines))

            # Look for lines with the version
            if matching_lines:
                if paths_only:
                    logging.info('File %s matches', file_path)
                else:
                    for line_no, line in matching_lines:
                        logging.info('Match: %s:%d:%s', file_path, line_no + 1, line)
                    if new_version is not None and (show_diffs or rewrite):
                        new_lines, dirty = rewrite_lines(lines, version_re, new_version, False, print_diffs=True,
                                                         ask_confirm=(rewrite and ask_confirm), grayscale=grayscale)
            else:
                logging.debug('File %s does not match', file_path)

        if dirty and rewrite:
            logging.info('Rewriting %s', os.path.join(base_path, file_path))
            with open(os.path.join(base_path, file_path), 'w') as fout:
                fout.write('\n'.join(new_lines))

        return True
    except (OSError, UnicodeDecodeError) as e:
        logging.exception('Unable to read file %s due to OSError', os.path.join(base_path, file_path))
        return False


def address_path(path, version, new_version=None, suspect_only=False, show_diffs=False, rewrite=False, ask_confirm=True, grayscale=False, paths_only=False):
    try:
        if os.path.exists(path):
            if os.path.isdir(path):
                status = True
                for dir_path, dir_names, file_names in os.walk(path):
                    for file_name in file_names:
                        file_path = os.path.relpath(os.path.join(dir_path, file_name), path)
                        status = address_file(path, file_path, version, new_version, suspect_only, show_diffs,
                                              rewrite, ask_confirm, grayscale, paths_only) and status
                return status
            else:
                base_name, file_name = os.path.split(path)
                return address_file(base_name, file_name, version, new_version, suspect_only, show_diffs, rewrite, ask_confirm, grayscale)
        else:
            logging.error('Path %s does not exist', path)
            return False
    except OSError as e:
        logging.exception('Unable to find all API versions due to OSError')
        return False


def run(arg_list):
    parser = argparse.ArgumentParser(description='finds and rewrites the API version in FDB source files')
    parser.add_argument('path', help='path to search for FDB source files')
    parser.add_argument('version', type=int, help='current/old version to search for')
    parser.add_argument('--new-version', type=int, default=None, help='new version to update to')
    parser.add_argument('--suspect-only', action='store_true', default=False, help='only look for phrases trying to set the API version')
    parser.add_argument('--show-diffs', action='store_true', default=False, help='show suggested diffs for fixing version')
    parser.add_argument('--rewrite', action='store_true', default=False, help='rewrite offending files')
    parser.add_argument('-y', '--skip-confirm', action='store_true', default=False, help='do not ask for confirmation before rewriting')
    parser.add_argument('--grayscale', action='store_true', default=False,
                        help='print diffs using grayscale output instead of red and green')
    parser.add_argument('--paths-only', action='store_true', default=False, help='display only the path instead of the offending lines')
    args = parser.parse_args(arg_list)
    return address_path(args.path, args.version, args.new_version, args.suspect_only, args.show_diffs,
                        args.rewrite, not args.skip_confirm, args.grayscale, args.paths_only)


if __name__ == '__main__':
    logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
    if not run(sys.argv[1:]):
        exit(1)