llvm-project/lldb/utils/sync-source/syncsource.py

#!/usr/bin/env python
"""
                     The LLVM Compiler Infrastructure

This file is distributed under the University of Illinois Open Source
License. See LICENSE.TXT for details.

Sync lldb and related source from a local machine to a remote machine.

This facilitates working on the lldb sourcecode on multiple machines
and multiple OS types, verifying changes across all.
"""

import argparse
import cStringIO
import importlib
import json
import os.path
import re
import sys

# Add the local lib directory to the python path.
LOCAL_LIB_PATH = os.path.join(
    os.path.dirname(os.path.realpath(__file__)),
    "lib")
sys.path.append(LOCAL_LIB_PATH)

import transfer.transfer_spec


DOTRC_BASE_FILENAME = ".syncsourcerc"


class Configuration(object):
    """Provides chaining configuration lookup."""

    def __init__(self, rcdata_configs):
        self.__rcdata_configs = rcdata_configs

    def get_value(self, key):
        """
        Return the first value in the parent chain that has the key.

        The traversal starts from the most derived configuration (i.e.
        child) and works all the way up the parent chain.

        @return the value of the first key in the parent chain that
        contains a value for the given key.
        """
        for config in self.__rcdata_configs:
            if key in config:
                return config[key]
        return None

    def __getitem__(self, key):
        value = self.get_value(key)
        if value:
            return value
        else:
            raise KeyError(key)


def parse_args():
    """@return options parsed from the command line."""
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--config-name", "-c", action="store", default="default",
        help="specify configuration name to use")
    parser.add_argument(
        "--default-excludes", action="store", default="*.git,*.svn,*.pyc",
        help=("comma-separated list of default file patterns to exclude "
              "from each source directory and to protect from deletion "
              "on each destination directory; if starting with forward "
              "slash, it only matches at the top of the base directory"))
    parser.add_argument(
        "--dry-run", "-n", action="store_true",
        help="do a dry run of the transfer operation, don't really transfer")
    parser.add_argument(
        "--rc-file", "-r", action="store",
        help="specify the sync-source rc file to use for configurations")
    parser.add_argument(
        "--verbose", "-v", action="store_true", help="turn on verbose output")
    return parser.parse_args()


def read_rcfile(filename):
    """Returns the json-parsed contents of the input file."""

    # First parse file contents, removing all comments but
    # preserving the line count.
    regex = re.compile(r"#.*$")

    comment_stripped_file = cStringIO.StringIO()
    with open(filename, "r") as json_file:
        for line in json_file:
            comment_stripped_file.write(regex.sub("", line))
    return json.load(cStringIO.StringIO(comment_stripped_file.getvalue()))


def find_appropriate_rcfile(options):
    # Use an options-specified rcfile if specified.
    if options.rc_file and len(options.rc_file) > 0:
        if not os.path.isfile(options.rc_file):
            # If it doesn't exist, error out here.
            raise "rcfile '{}' specified but doesn't exist".format(
                options.rc_file)
        return options.rc_file

    # Check if current directory .sync-sourcerc exists.  If so, use it.
    local_rc_filename = os.path.abspath(DOTRC_BASE_FILENAME)
    if os.path.isfile(local_rc_filename):
        return local_rc_filename

    # Check if home directory .sync-sourcerc exists.  If so, use it.
    homedir_rc_filename = os.path.abspath(
        os.path.join(os.path.expanduser("~"), DOTRC_BASE_FILENAME))
    if os.path.isfile(homedir_rc_filename):
        return homedir_rc_filename

    # Nothing matched.  We don't have an rc filename candidate.
    return None


def get_configuration(options, rcdata, config_name):
    rcdata_configs = []
    next_config_name = config_name
    while next_config_name:
        # Find the next rcdata configuration for the given name.
        rcdata_config = next(
            config for config in rcdata["configurations"]
            if config["name"] == next_config_name)

        # See if we found it.
        if rcdata_config:
            # This is our next configuration to use in the chain.
            rcdata_configs.append(rcdata_config)

            # If we have a parent, check that next.
            if "parent" in rcdata_config:
                next_config_name = rcdata_config["parent"]
            else:
                next_config_name = None
        else:
            raise "failed to find specified parent config '{}'".format(
                next_config_name)
    return Configuration(rcdata_configs)


def create_transfer_agent(options, configuration):
    transfer_class_spec = configuration.get_value("transfer_class")
    if options.verbose:
        print "specified transfer class: '{}'".format(transfer_class_spec)

    # Load the module (possibly package-qualified).
    components = transfer_class_spec.split(".")
    module = importlib.import_module(".".join(components[:-1]))

    # Create the class name we need to load.
    clazz = getattr(module, components[-1])
    return clazz(options, configuration)


def sync_configured_sources(options, configuration, default_excludes):
    # Look up the transfer method.
    transfer_agent = create_transfer_agent(options, configuration)

    # For each configured dir_names source, do the following transfer:
    #   1. Start with base_dir + {source-dir-name}_dir
    #   2. Copy all files recursively, but exclude
    #      all dirs specified by source_excludes:
    #      skip all base_dir + {source-dir-name}_dir +
    #      {source-dir-name}_dir excludes.
    source_dirs = configuration.get_value("source")
    source_excludes = configuration.get_value("source_excludes")
    dest_dirs = configuration.get_value("dest")

    source_base_dir = source_dirs["base_dir"]
    dest_base_dir = dest_dirs["base_dir"]
    dir_ids = configuration.get_value("dir_names")
    transfer_specs = []

    for dir_id in dir_ids:
        dir_key = "{}_dir".format(dir_id)

        # Build the source dir (absolute) that we're copying from.
        # Defaults the base-relative source dir to the source id (e.g. lldb)
        rel_source_dir = source_dirs.get(dir_key, dir_id)
        transfer_source_dir = os.path.expanduser(
            os.path.join(source_base_dir, rel_source_dir))

        # Exclude dirs do two things:
        # 1) stop items from being copied on the source side, and
        # 2) protect things from being deleted on the dest side.
        #
        # In both cases, they are specified relative to the base
        # directory on either the source or dest side.
        #
        # Specifying a leading '/' in the directory will limit it to
        # be rooted in the base directory.  i.e. "/.git" will only
        # match {base-dir}/.git, not {base-dir}/subdir/.git, but
        # ".svn" will match {base-dir}/.svn and
        # {base-dir}/subdir/.svn.
        #
        # If excludes are specified for this dir_id, then pass along
        # the excludes.  These are relative to the dir_id directory
        # source, and get passed along that way as well.
        transfer_source_excludes = []

        # Add the source excludes for this dir.
        skip_defaults = False
        if source_excludes and dir_key in source_excludes:
            transfer_source_excludes.extend(source_excludes[dir_key])
            if "<no-defaults>" in source_excludes[dir_key]:
                skip_defaults = True
                transfer_source_excludes.remove("<no-defaults>")

        if not skip_defaults and default_excludes is not None:
            transfer_source_excludes.extend(list(default_excludes))

        # Build the destination-base-relative dest dir into which
        # we'll be syncing.  Relative directory defaults to the
        # dir id
        rel_dest_dir = dest_dirs.get(dir_key, dir_id)
        transfer_dest_dir = os.path.join(dest_base_dir, rel_dest_dir)

        # Add the exploded paths to the list that we'll ask the
        # transfer agent to transfer for us.
        transfer_specs.append(
            transfer.transfer_spec.TransferSpec(
                transfer_source_dir,
                transfer_source_excludes,
                transfer_dest_dir))

    # Do the transfer.
    if len(transfer_specs) > 0:
        transfer_agent.transfer(transfer_specs, options.dry_run)
    else:
        raise Exception("nothing to transfer, bad configuration?")


def main():
    """Drives the main program."""
    options = parse_args()

    if options.default_excludes and len(options.default_excludes) > 0:
        default_excludes = options.default_excludes.split(",")
    else:
        default_excludes = []

    # Locate the rc filename to load, then load it.
    rc_filename = find_appropriate_rcfile(options)
    if rc_filename:
        if options.verbose:
            print "reading rc data from file '{}'".format(rc_filename)
        rcdata = read_rcfile(rc_filename)
    else:
        sys.stderr.write("no rcfile specified, cannot guess configuration")
        exit(1)

    # Find configuration.
    configuration = get_configuration(options, rcdata, options.config_name)
    if not configuration:
        sys.stderr.write("failed to find configuration for {}".format(
            options.config_data))
        exit(2)

    # Kick off the transfer.
    sync_configured_sources(options, configuration, default_excludes)

if __name__ == "__main__":
    main()