forked from OSchip/llvm-project
287 lines
11 KiB
Python
287 lines
11 KiB
Python
# DExTer : Debugging Experience Tester
|
|
# ~~~~~~ ~ ~~ ~ ~~
|
|
#
|
|
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
# See https://llvm.org/LICENSE.txt for license information.
|
|
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
"""Clang opt-bisect tool."""
|
|
|
|
from collections import defaultdict
|
|
import os
|
|
import csv
|
|
import re
|
|
import pickle
|
|
|
|
from dex.builder import run_external_build_script
|
|
from dex.debugger.Debuggers import empty_debugger_steps, get_debugger_steps
|
|
from dex.heuristic import Heuristic
|
|
from dex.tools import TestToolBase
|
|
from dex.utils.Exceptions import DebuggerException, Error
|
|
from dex.utils.Exceptions import BuildScriptException, HeuristicException
|
|
from dex.utils.PrettyOutputBase import Stream
|
|
from dex.utils.ReturnCode import ReturnCode
|
|
|
|
|
|
class BisectPass(object):
|
|
def __init__(self, no, description, description_no_loc):
|
|
self.no = no
|
|
self.description = description
|
|
self.description_no_loc = description_no_loc
|
|
|
|
self.penalty = 0
|
|
self.differences = []
|
|
|
|
|
|
class Tool(TestToolBase):
|
|
"""Use the LLVM "-opt-bisect-limit=<n>" flag to get information on the
|
|
contribution of each LLVM pass to the overall DExTer score when using
|
|
clang.
|
|
|
|
Clang is run multiple times, with an increasing value of n, measuring the
|
|
debugging experience at each value.
|
|
"""
|
|
|
|
_re_running_pass = re.compile(
|
|
r'^BISECT\: running pass \((\d+)\) (.+?)( \(.+\))?$')
|
|
|
|
def __init__(self, *args, **kwargs):
|
|
super(Tool, self).__init__(*args, **kwargs)
|
|
self._all_bisect_pass_summary = defaultdict(list)
|
|
|
|
@property
|
|
def name(self):
|
|
return 'DExTer clang opt bisect'
|
|
|
|
def _get_bisect_limits(self):
|
|
options = self.context.options
|
|
|
|
max_limit = 999999
|
|
limits = [max_limit for _ in options.source_files]
|
|
all_passes = [
|
|
l for l in self._clang_opt_bisect_build(limits)[1].splitlines()
|
|
if l.startswith('BISECT: running pass (')
|
|
]
|
|
|
|
results = []
|
|
for i, pass_ in enumerate(all_passes[1:]):
|
|
if pass_.startswith('BISECT: running pass (1)'):
|
|
results.append(all_passes[i])
|
|
results.append(all_passes[-1])
|
|
|
|
assert len(results) == len(
|
|
options.source_files), (results, options.source_files)
|
|
|
|
limits = [
|
|
int(Tool._re_running_pass.match(r).group(1)) for r in results
|
|
]
|
|
|
|
return limits
|
|
|
|
def _run_test(self, test_name): # noqa
|
|
options = self.context.options
|
|
|
|
per_pass_score = []
|
|
current_bisect_pass_summary = defaultdict(list)
|
|
|
|
max_limits = self._get_bisect_limits()
|
|
overall_limit = sum(max_limits)
|
|
prev_score = 1.0
|
|
prev_steps_str = None
|
|
|
|
for current_limit in range(overall_limit + 1):
|
|
# Take the overall limit number and split it across buckets for
|
|
# each source file.
|
|
limit_remaining = current_limit
|
|
file_limits = [0] * len(max_limits)
|
|
for i, max_limit in enumerate(max_limits):
|
|
if limit_remaining < max_limit:
|
|
file_limits[i] += limit_remaining
|
|
break
|
|
else:
|
|
file_limits[i] = max_limit
|
|
limit_remaining -= file_limits[i]
|
|
|
|
f = [l for l in file_limits if l]
|
|
current_file_index = len(f) - 1 if f else 0
|
|
|
|
_, err, builderIR = self._clang_opt_bisect_build(file_limits)
|
|
err_lines = err.splitlines()
|
|
# Find the last line that specified a running pass.
|
|
for l in err_lines[::-1]:
|
|
match = Tool._re_running_pass.match(l)
|
|
if match:
|
|
pass_info = match.groups()
|
|
break
|
|
else:
|
|
pass_info = (0, None, None)
|
|
|
|
try:
|
|
steps = get_debugger_steps(self.context)
|
|
except DebuggerException:
|
|
steps = empty_debugger_steps(self.context)
|
|
|
|
steps.builder = builderIR
|
|
|
|
try:
|
|
heuristic = Heuristic(self.context, steps)
|
|
except HeuristicException as e:
|
|
raise Error(e)
|
|
|
|
score_difference = heuristic.score - prev_score
|
|
prev_score = heuristic.score
|
|
|
|
isnan = heuristic.score != heuristic.score
|
|
if isnan or score_difference < 0:
|
|
color1 = 'r'
|
|
color2 = 'r'
|
|
elif score_difference > 0:
|
|
color1 = 'g'
|
|
color2 = 'g'
|
|
else:
|
|
color1 = 'y'
|
|
color2 = 'd'
|
|
|
|
summary = '<{}>running pass {}/{} on "{}"'.format(
|
|
color2, pass_info[0], max_limits[current_file_index],
|
|
test_name)
|
|
if len(options.source_files) > 1:
|
|
summary += ' [{}/{}]'.format(current_limit, overall_limit)
|
|
|
|
pass_text = ''.join(p for p in pass_info[1:] if p)
|
|
summary += ': {} <{}>{:+.4f}</> <{}>{}</></>\n'.format(
|
|
heuristic.summary_string, color1, score_difference, color2,
|
|
pass_text)
|
|
|
|
self.context.o.auto(summary)
|
|
|
|
heuristic_verbose_output = heuristic.verbose_output
|
|
|
|
if options.verbose:
|
|
self.context.o.auto(heuristic_verbose_output)
|
|
|
|
steps_str = str(steps)
|
|
steps_changed = steps_str != prev_steps_str
|
|
prev_steps_str = steps_str
|
|
|
|
# If this is the first pass, or something has changed, write a text
|
|
# file containing verbose information on the current status.
|
|
if current_limit == 0 or score_difference or steps_changed:
|
|
file_name = '-'.join(
|
|
str(s) for s in [
|
|
'status', test_name, '{{:0>{}}}'.format(
|
|
len(str(overall_limit))).format(current_limit),
|
|
'{:.4f}'.format(heuristic.score).replace(
|
|
'.', '_'), pass_info[1]
|
|
] if s is not None)
|
|
|
|
file_name = ''.join(
|
|
c for c in file_name
|
|
if c.isalnum() or c in '()-_./ ').strip().replace(
|
|
' ', '_').replace('/', '_')
|
|
|
|
output_text_path = os.path.join(options.results_directory,
|
|
'{}.txt'.format(file_name))
|
|
with open(output_text_path, 'w') as fp:
|
|
self.context.o.auto(summary + '\n', stream=Stream(fp))
|
|
self.context.o.auto(str(steps) + '\n', stream=Stream(fp))
|
|
self.context.o.auto(
|
|
heuristic_verbose_output + '\n', stream=Stream(fp))
|
|
|
|
output_dextIR_path = os.path.join(options.results_directory,
|
|
'{}.dextIR'.format(file_name))
|
|
with open(output_dextIR_path, 'wb') as fp:
|
|
pickle.dump(steps, fp, protocol=pickle.HIGHEST_PROTOCOL)
|
|
|
|
per_pass_score.append((test_name, pass_text,
|
|
heuristic.score))
|
|
|
|
if pass_info[1]:
|
|
self._all_bisect_pass_summary[pass_info[1]].append(
|
|
score_difference)
|
|
|
|
current_bisect_pass_summary[pass_info[1]].append(
|
|
score_difference)
|
|
|
|
per_pass_score_path = os.path.join(
|
|
options.results_directory,
|
|
'{}-per_pass_score.csv'.format(test_name))
|
|
|
|
with open(per_pass_score_path, mode='w', newline='') as fp:
|
|
writer = csv.writer(fp, delimiter=',')
|
|
writer.writerow(['Source File', 'Pass', 'Score'])
|
|
|
|
for path, pass_, score in per_pass_score:
|
|
writer.writerow([path, pass_, score])
|
|
self.context.o.blue('wrote "{}"\n'.format(per_pass_score_path))
|
|
|
|
pass_summary_path = os.path.join(
|
|
options.results_directory, '{}-pass-summary.csv'.format(test_name))
|
|
|
|
self._write_pass_summary(pass_summary_path,
|
|
current_bisect_pass_summary)
|
|
|
|
def _handle_results(self) -> ReturnCode:
|
|
options = self.context.options
|
|
pass_summary_path = os.path.join(options.results_directory,
|
|
'overall-pass-summary.csv')
|
|
|
|
self._write_pass_summary(pass_summary_path,
|
|
self._all_bisect_pass_summary)
|
|
return ReturnCode.OK
|
|
|
|
|
|
def _clang_opt_bisect_build(self, opt_bisect_limits):
|
|
options = self.context.options
|
|
compiler_options = [
|
|
'{} -mllvm -opt-bisect-limit={}'.format(options.cflags,
|
|
opt_bisect_limit)
|
|
for opt_bisect_limit in opt_bisect_limits
|
|
]
|
|
linker_options = options.ldflags
|
|
|
|
try:
|
|
return run_external_build_script(
|
|
self.context,
|
|
source_files=options.source_files,
|
|
compiler_options=compiler_options,
|
|
linker_options=linker_options,
|
|
script_path=self.build_script,
|
|
executable_file=options.executable)
|
|
except BuildScriptException as e:
|
|
raise Error(e)
|
|
|
|
def _write_pass_summary(self, path, pass_summary):
|
|
# Get a list of tuples.
|
|
pass_summary_list = list(pass_summary.items())
|
|
|
|
for i, item in enumerate(pass_summary_list):
|
|
# Add elems for the sum, min, and max of the values, as well as
|
|
# 'interestingness' which is whether any of these values are
|
|
# non-zero.
|
|
pass_summary_list[i] += (sum(item[1]), min(item[1]), max(item[1]),
|
|
any(item[1]))
|
|
|
|
# Split the pass name into the basic name and kind.
|
|
pass_summary_list[i] += tuple(item[0].rsplit(' on ', 1))
|
|
|
|
# Sort the list by the following columns in order of precedence:
|
|
# - Is interesting (True first)
|
|
# - Sum (smallest first)
|
|
# - Number of times pass ran (largest first)
|
|
# - Kind (alphabetically)
|
|
# - Name (alphabetically)
|
|
pass_summary_list.sort(
|
|
key=lambda tup: (not tup[5], tup[2], -len(tup[1]), tup[7], tup[6]))
|
|
|
|
with open(path, mode='w', newline='') as fp:
|
|
writer = csv.writer(fp, delimiter=',')
|
|
writer.writerow(
|
|
['Pass', 'Kind', 'Sum', 'Min', 'Max', 'Interesting'])
|
|
|
|
for (_, vals, sum_, min_, max_, interesting, name,
|
|
kind) in pass_summary_list:
|
|
writer.writerow([name, kind, sum_, min_, max_, interesting] +
|
|
vals)
|
|
|
|
self.context.o.blue('wrote "{}"\n'.format(path))
|