llvm-project/lld/utils/benchmark.py

#!/usr/bin/env python
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
# ==------------------------------------------------------------------------==#

import os
import glob
import re
import subprocess
import json
import datetime
import argparse
try:
    from urllib.parse import urlencode
    from urllib.request import urlopen, Request
except ImportError:
    from urllib import urlencode
    from urllib2 import urlopen, Request


parser = argparse.ArgumentParser()
parser.add_argument('benchmark_directory')
parser.add_argument('--runs', type=int, default=10)
parser.add_argument('--wrapper', default='')
parser.add_argument('--machine', required=True)
parser.add_argument('--revision', required=True)
parser.add_argument('--threads', action='store_true')
parser.add_argument('--url', help='The lnt server url to send the results to',
                    default='http://localhost:8000/db_default/v4/link/submitRun')
args = parser.parse_args()

class Bench:
    def __init__(self, directory, variant):
        self.directory = directory
        self.variant = variant
    def __str__(self):
        if not self.variant:
            return self.directory
        return '%s-%s' % (self.directory, self.variant)

def getBenchmarks():
    ret = []
    for i in glob.glob('*/response*.txt'):
        m = re.match('response-(.*)\.txt', os.path.basename(i))
        variant = m.groups()[0] if m else None
        ret.append(Bench(os.path.dirname(i), variant))
    return ret

def parsePerfNum(num):
    num = num.replace(b',',b'')
    try:
        return int(num)
    except ValueError:
        return float(num)

def parsePerfLine(line):
    ret = {}
    line = line.split(b'#')[0].strip()
    if len(line) != 0:
        p = line.split()
        ret[p[1].strip().decode('ascii')] = parsePerfNum(p[0])
    return ret

def parsePerf(output):
    ret = {}
    lines = [x.strip() for x in output.split(b'\n')]

    seconds = [x for x in lines if b'seconds time elapsed' in x][0]
    seconds = seconds.strip().split()[0].strip()
    ret['seconds-elapsed'] = parsePerfNum(seconds)

    measurement_lines = [x for x in lines if b'#' in x]
    for l in measurement_lines:
        ret.update(parsePerfLine(l))
    return ret

def run(cmd):
    try:
        return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
    except subprocess.CalledProcessError as e:
        print(e.output)
        raise e

def combinePerfRun(acc, d):
    for k,v in d.items():
        a = acc.get(k, [])
        a.append(v)
        acc[k] = a

def perf(cmd):
    # Discard the first run to warm up any system cache.
    run(cmd)

    ret = {}
    wrapper_args = [x for x in args.wrapper.split(',') if x]
    for i in range(args.runs):
        os.unlink('t')
        out = run(wrapper_args + ['perf', 'stat'] + cmd)
        r = parsePerf(out)
        combinePerfRun(ret, r)
    os.unlink('t')
    return ret

def runBench(bench):
    thread_arg = [] if args.threads else ['--no-threads']
    os.chdir(bench.directory)
    suffix = '-%s' % bench.variant if bench.variant else ''
    response = 'response' + suffix + '.txt'
    ret = perf(['../ld.lld', '@' + response, '-o', 't'] + thread_arg)
    ret['name'] = str(bench)
    os.chdir('..')
    return ret

def buildLntJson(benchmarks):
    start = datetime.datetime.utcnow().isoformat()
    tests = [runBench(b) for b in benchmarks]
    end = datetime.datetime.utcnow().isoformat()
    ret = {
        'format_version' : 2,
        'machine' : { 'name' : args.machine },
        'run' : {
            'end_time' : start,
            'start_time' : end,
            'llvm_project_revision': args.revision
        },
        'tests' : tests
    }
    return json.dumps(ret, sort_keys=True, indent=4)

def submitToServer(data):
    data2 = urlencode({ 'input_data' : data }).encode('ascii')
    urlopen(Request(args.url, data2))

os.chdir(args.benchmark_directory)
data = buildLntJson(getBenchmarks())
submitToServer(data)