llvm-project/lldb/scripts/Python/remote-build.py

#!/usr/bin/python

from __future__ import print_function

import argparse
import getpass
import os
import os.path
import re
import select
import sys
import subprocess

_COMMON_SYNC_OPTS = "-avzh --delete"
_COMMON_EXCLUDE_OPTS = "--exclude=DerivedData --exclude=.svn --exclude=.git --exclude=llvm-build/Release+Asserts"


def normalize_configuration(config_text):
    if not config_text:
        return "debug"

    config_lower = config_text.lower()
    if config_lower in ["debug", "release"]:
        return config_lower
    else:
        raise Exception("unknown configuration specified: %s" % config_text)


def parse_args():
    DEFAULT_REMOTE_ROOT_DIR = "/mnt/ssd/work/macosx.sync"
    DEFAULT_REMOTE_HOSTNAME = "tfiala2.mtv.corp.google.com"
    OPTIONS_FILENAME = ".remote-build.conf"
    DEFAULT_SSH_PORT = "22"

    parser = argparse.ArgumentParser(fromfile_prefix_chars='@')

    parser.add_argument(
        "--configuration",
        "-c",
        help="specify configuration (Debug, Release)",
        default=normalize_configuration(
            os.environ.get(
                'CONFIGURATION',
                'Debug')))
    parser.add_argument(
        "--debug", "-d",
        action="store_true",
        help="help debug the remote-build script by adding extra logging")
    parser.add_argument(
        "--local-lldb-dir", "-l", metavar="DIR",
        help="specify local lldb directory (Xcode layout assumed for llvm/clang)",
        default=os.getcwd())
    parser.add_argument(
        "--port", "-p",
        help="specify the port ssh should use to connect to the remote side",
        default=DEFAULT_SSH_PORT)
    parser.add_argument(
        "--remote-address", "-r", metavar="REMOTE-ADDR",
        help="specify the dns name or ip address of the remote linux system",
        default=DEFAULT_REMOTE_HOSTNAME)
    parser.add_argument(
        "--remote-dir", metavar="DIR",
        help="specify the root of the linux source/build dir",
        default=DEFAULT_REMOTE_ROOT_DIR)
    parser.add_argument(
        "--user", "-u", help="specify the user name for the remote system",
        default=getpass.getuser())
    parser.add_argument(
        "--xcode-action",
        "-x",
        help="$(ACTION) from Xcode",
        nargs='?',
        default=None)

    command_line_args = sys.argv[1:]
    if os.path.exists(OPTIONS_FILENAME):
        # Prepend the file so that command line args override the file
        # contents.
        command_line_args.insert(0, "@%s" % OPTIONS_FILENAME)

    return parser.parse_args(command_line_args)


def maybe_create_remote_root_dir(args):
    commandline = [
        "ssh",
        "-p", args.port,
        "%s@%s" % (args.user, args.remote_address),
        "mkdir",
        "-p",
        args.remote_dir]
    print("create remote root dir command:\n{}".format(commandline))
    return subprocess.call(commandline)


def init_with_args(args):
    # Expand any user directory specs in local-side source dir (on MacOSX).
    args.local_lldb_dir = os.path.expanduser(args.local_lldb_dir)

    # Append the configuration type to the remote build dir.
    args.configuration = normalize_configuration(args.configuration)
    args.remote_build_dir = os.path.join(
        args.remote_dir,
        "build-%s" % args.configuration)

    # We assume the local lldb directory is really named 'lldb'.
    # This is because on the remote end, the local lldb root dir
    # is copied over underneath llvm/tools and will be named there
    # whatever it is named locally.  The remote build will assume
    # is is called lldb.
    if os.path.basename(args.local_lldb_dir) != 'lldb':
        raise Exception(
            "local lldb root needs to be called 'lldb' but was {} instead"
            .format(os.path.basename(args.local_lldb_dir)))

    args.lldb_dir_relative_regex = re.compile(
        "%s/llvm/tools/lldb/" % args.remote_dir)
    args.llvm_dir_relative_regex = re.compile("%s/" % args.remote_dir)

    print("Xcode action:", args.xcode_action)

    # Ensure the remote directory exists.
    result = maybe_create_remote_root_dir(args)
    if result == 0:
        print("using remote root dir: %s" % args.remote_dir)
    else:
        print("remote root dir doesn't exist and could not be created, "
              + "error code:", result)
        return False

    return True


def sync_llvm(args):
    commandline = ["rsync"]
    commandline.extend(_COMMON_SYNC_OPTS.split())
    commandline.extend(_COMMON_EXCLUDE_OPTS.split())
    commandline.append("--exclude=/llvm/tools/lldb")
    commandline.extend(["-e", "ssh -p {}".format(args.port)])
    commandline.extend([
        "%s/llvm" % args.local_lldb_dir,
        "%s@%s:%s" % (args.user, args.remote_address, args.remote_dir)])
    if args.debug:
        print("going to execute llvm sync: {}".format(commandline))
    return subprocess.call(commandline)


def sync_lldb(args):
    commandline = ["rsync"]
    commandline.extend(_COMMON_SYNC_OPTS.split())
    commandline.extend(_COMMON_EXCLUDE_OPTS.split())
    commandline.append("--exclude=/lldb/llvm")
    commandline.extend(["-e", "ssh -p {}".format(args.port)])
    commandline.extend([args.local_lldb_dir, "%s@%s:%s/llvm/tools" %
                        (args.user, args.remote_address, args.remote_dir)])
    if args.debug:
        print("going to execute lldb sync: {}".format(commandline))
    return subprocess.call(commandline)


def build_cmake_command(args):
    # args.remote_build_dir
    # args.configuration in ('release', 'debug')

    if args.configuration == 'debug-optimized':
        build_type_name = "RelWithDebInfo"
    elif args.configuration == 'release':
        build_type_name = "Release"
    else:
        build_type_name = "Debug"

    ld_flags = "\"-lstdc++ -lm\""

    install_dir = os.path.join(
        args.remote_build_dir, "..", "install-{}".format(args.configuration))

    command_line = [
        "cmake",
        "-GNinja",
        "-DCMAKE_CXX_COMPILER=clang",
        "-DCMAKE_C_COMPILER=clang",
        # "-DCMAKE_CXX_FLAGS=%s" % cxx_flags,
        "-DCMAKE_SHARED_LINKER_FLAGS=%s" % ld_flags,
        "-DCMAKE_EXE_LINKER_FLAGS=%s" % ld_flags,
        "-DCMAKE_INSTALL_PREFIX:PATH=%s" % install_dir,
        "-DCMAKE_BUILD_TYPE=%s" % build_type_name,
        "-Wno-dev",
        os.path.join("..", "llvm")
    ]

    return command_line


def maybe_configure(args):
    commandline = [
        "ssh",
        "-p", args.port,
        "%s@%s" % (args.user, args.remote_address),
        "cd", args.remote_dir, "&&",
        "mkdir", "-p", args.remote_build_dir, "&&",
        "cd", args.remote_build_dir, "&&"
    ]
    commandline.extend(build_cmake_command(args))

    if args.debug:
        print("configure command: {}".format(commandline))

    return subprocess.call(commandline)


def filter_build_line(args, line):
    lldb_relative_line = args.lldb_dir_relative_regex.sub('', line)
    if len(lldb_relative_line) != len(line):
        # We substituted - return the modified line
        return lldb_relative_line

    # No match on lldb path (longer on linux than llvm path).  Try
    # the llvm path match.
    return args.llvm_dir_relative_regex.sub('', line)


def run_remote_build_command(args, build_command_list):
    commandline = [
        "ssh",
        "-p", args.port,
        "%s@%s" % (args.user, args.remote_address),
        "cd", args.remote_build_dir, "&&"]
    commandline.extend(build_command_list)

    if args.debug:
        print("running remote build command: {}".format(commandline))

    proc = subprocess.Popen(
        commandline,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE)

    # Filter stdout/stderr output for file path mapping.
    # We do this to enable Xcode to see filenames relative to the
    # MacOSX-side directory structure.
    while True:
        reads = [proc.stdout.fileno(), proc.stderr.fileno()]
        select_result = select.select(reads, [], [])

        for fd in select_result[0]:
            if fd == proc.stdout.fileno():
                line = proc.stdout.readline()
                display_line = filter_build_line(args, line.rstrip())
                if display_line and len(display_line) > 0:
                    print(display_line)
            elif fd == proc.stderr.fileno():
                line = proc.stderr.readline()
                display_line = filter_build_line(args, line.rstrip())
                if display_line and len(display_line) > 0:
                    print(display_line, file=sys.stderr)

        proc_retval = proc.poll()
        if proc_retval is not None:
            # Process stopped.  Drain output before finishing up.

            # Drain stdout.
            while True:
                line = proc.stdout.readline()
                if line:
                    display_line = filter_build_line(args, line.rstrip())
                    if display_line and len(display_line) > 0:
                        print(display_line)
                else:
                    break

            # Drain stderr.
            while True:
                line = proc.stderr.readline()
                if line:
                    display_line = filter_build_line(args, line.rstrip())
                    if display_line and len(display_line) > 0:
                        print(display_line, file=sys.stderr)
                else:
                    break

            return proc_retval


def build(args):
    return run_remote_build_command(args, ["time", "ninja"])


def clean(args):
    return run_remote_build_command(args, ["ninja", "clean"])


if __name__ == "__main__":
    # Handle arg parsing.
    args = parse_args()

    # Initialize the system.
    if not init_with_args(args):
        exit(1)

    # Sync over llvm and clang source.
    sync_llvm(args)

    # Sync over lldb source.
    sync_lldb(args)

    # Configure the remote build if it's not already.
    maybe_configure(args)

    if args.xcode_action == 'clean':
        exit(clean(args))
    else:
        exit(build(args))