maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[UpdateTestChecks] Share the code to parse RUN: lines between all scripts 

Summary:
This commit also introduces a common.debug() function to avoid many
`if args.verbose:` statements. Depends on D70428.

Reviewers: xbolva00, MaskRay, jdoerfert

Reviewed By: MaskRay

Subscribers: llvm-commits

Tags: #llvm

Differential Revision: https://reviews.llvm.org/D70432
This commit is contained in:
Alex Richardson 2019-12-02 10:50:23 +00:00
parent 160a5045c6
commit d9542db49e
7 changed files with 62 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
llvm/utils/UpdateTestChecks/common.py
View File

@ -14,12 +14,17 @@ else:
##### Common utilities for update_*test_checks.py
_verbose = False
def parse_commandline_args(parser):
parser.add_argument('-v', '--verbose', action='store_true',
help='Show verbose output')
parser.add_argument('-u', '--update-only', action='store_true',
help='Only update test if it was already autogened')
return parser.parse_args()
args = parser.parse_args()
global _verbose
_verbose = args.verbose
return args
def should_add_line_to_output(input_line, prefix_set):
# Skip any blank comment lines in the IR.
@ -53,7 +58,7 @@ def invoke_tool(exe, cmd_args, ir):
##### LLVM IR parser
RUN_LINE_RE = re.compile(r'^\s*[;#]\s*RUN:\s*(.*)$')
RUN_LINE_RE = re.compile(r'^\s*(?://|[;#])\s*RUN:\s*(.*)$')
CHECK_PREFIX_RE = re.compile(r'--?check-prefix(?:es)?[= ](\S+)')
PREFIX_RE = re.compile('^[a-zA-Z0-9_-]+$')
CHECK_RE = re.compile(r'^\s*[;#]\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL|-SAME)?:')
@ -91,6 +96,28 @@ def warn(msg, test_file=None):
msg = '{}: {}'.format(msg, test_file)
print('WARNING: {}'.format(msg), file=sys.stderr)
def debug(*args, **kwargs):
# Python2 does not allow def debug(*args, file=sys.stderr, **kwargs):
if 'file' not in kwargs:
kwargs['file'] = sys.stderr
if _verbose:
print(*args, **kwargs)
def find_run_lines(test, lines):
debug('Scanning for RUN lines in test file:', test)
raw_lines = [m.group(1)
for m in [RUN_LINE_RE.match(l) for l in lines] if m]
run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
for l in raw_lines[1:]:
if run_lines[-1].endswith('\\'):
run_lines[-1] = run_lines[-1].rstrip('\\' + ' ' + l)
else:
run_lines.append(l)
debug('Found {} RUN lines in {}:'.format(len(run_lines), test))
for l in run_lines:
debug(' RUN: {}'.format(l))
return run_lines
def scrub_body(body):
# Scrub runs of whitespace out of the assembly, but leave the leading
# whitespace in place.

28
llvm/utils/update_analyze_test_checks.py
View File

@ -69,8 +69,6 @@ def main():
test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
for test in test_paths:
if args.verbose:
print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)
with open(test) as f:
input_lines = [l.rstrip() for l in f]
@ -84,20 +82,7 @@ def main():
common.warn("Skipping test which isn't autogenerated: " + test)
continue
raw_lines = [m.group(1)
for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
for l in raw_lines[1:]:
if run_lines[-1].endswith("\\"):
run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
else:
run_lines.append(l)
if args.verbose:
print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
for l in run_lines:
print(' RUN: ' + l, file=sys.stderr)
run_lines = common.find_run_lines(test, input_lines)
prefix_list = []
for l in run_lines:
if '|' not in l:
@ -132,9 +117,8 @@ def main():
for prefix in prefixes:
func_dict.update({prefix: dict()})
for prefixes, opt_args in prefix_list:
if args.verbose:
print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
common.debug('Extracted opt cmd:', opt_basename, opt_args, file=sys.stderr)
common.debug('Extracted FileCheck prefixes:', str(prefixes), file=sys.stderr)
raw_tool_outputs = common.invoke_tool(args.opt_binary, opt_args, test)
@ -147,8 +131,7 @@ def main():
is_in_function = False
is_in_function_start = False
prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
if args.verbose:
print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
common.debug('Rewriting FileCheck prefixes:', str(prefix_set), file=sys.stderr)
output_lines = []
output_lines.append(autogenerated_note)
@ -194,8 +177,7 @@ def main():
continue
is_in_function = is_in_function_start = True
if args.verbose:
print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
common.debug('Writing %d lines to %s...' % (len(output_lines), test))
with open(test, 'wb') as f:
f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])

36
llvm/utils/update_cc_test_checks.py
View File

@ -29,7 +29,6 @@ from UpdateTestChecks import asm, common
ADVERT = '// NOTE: Assertions have been autogenerated by '
CHECK_RE = re.compile(r'^\s*//\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL)?:')
RUN_LINE_RE = re.compile(r'^//\s*RUN:\s*(.*)$')
SUBST = {
'%clang': [],
@ -38,9 +37,6 @@ SUBST = {
}
def get_line2spell_and_mangled(args, clang_args):
def debug_mangled(*print_args, **kwargs):
if args.verbose:
print(*print_args, file=sys.stderr, **kwargs)
ret = {}
# Use clang's JSON AST dump to get the mangled name
json_dump_args = [args.clang, *clang_args, '-fsyntax-only', '-o', '-']
@ -49,7 +45,7 @@ def get_line2spell_and_mangled(args, clang_args):
# -Xclang -ast-dump=json instead:
json_dump_args.append('-Xclang')
json_dump_args.append('-ast-dump=json')
debug_mangled('Running', ' '.join(json_dump_args))
common.debug('Running', ' '.join(json_dump_args))
status = subprocess.run(json_dump_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if status.returncode != 0:
sys.stderr.write('Failed to run ' + ' '.join(json_dump_args) + '\n')
@ -67,20 +63,19 @@ def get_line2spell_and_mangled(args, clang_args):
if node['kind'] != 'FunctionDecl':
continue
if node.get('isImplicit') is True and node.get('storageClass') == 'extern':
debug_mangled('Skipping builtin function:', node['name'], '@', node['loc'])
common.debug('Skipping builtin function:', node['name'], '@', node['loc'])
continue
debug_mangled('Found function:', node['kind'], node['name'], '@', node['loc'])
common.debug('Found function:', node['kind'], node['name'], '@', node['loc'])
line = node['loc'].get('line')
# If there is no line it is probably a builtin function -> skip
if line is None:
debug_mangled('Skipping function without line number:', node['name'], '@', node['loc'])
common.debug('Skipping function without line number:', node['name'], '@', node['loc'])
continue
spell = node['name']
mangled = node.get('mangledName', spell)
ret[int(line)-1] = (spell, mangled)
if args.verbose:
for line, func_name in sorted(ret.items()):
print('line {}: found function {}'.format(line+1, func_name), file=sys.stderr)
for line, func_name in sorted(ret.items()):
common.debug('line {}: found function {}'.format(line+1, func_name), file=sys.stderr)
if not ret:
common.warn('Did not find any functions using', ' '.join(json_dump_args))
return ret
@ -191,19 +186,7 @@ def main():
continue
# Extract RUN lines.
raw_lines = [m.group(1)
for m in [RUN_LINE_RE.match(l) for l in input_lines] if m]
run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
for l in raw_lines[1:]:
if run_lines[-1].endswith("\\"):
run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
else:
run_lines.append(l)
if args.verbose:
print('Found {} RUN lines:'.format(len(run_lines)), file=sys.stderr)
for l in run_lines:
print(' RUN: ' + l, file=sys.stderr)
run_lines = common.find_run_lines(filename, input_lines)
# Build a list of clang command lines and check prefixes from RUN lines.
run_list = []
@ -260,9 +243,8 @@ def main():
for prefix in prefixes:
func_dict.update({prefix: dict()})
for prefixes, clang_args, extra_commands, triple_in_cmd in run_list:
if args.verbose:
print('Extracted clang cmd: clang {}'.format(clang_args), file=sys.stderr)
print('Extracted FileCheck prefixes: {}'.format(prefixes), file=sys.stderr)
common.debug('Extracted clang cmd: clang {}'.format(clang_args))
common.debug('Extracted FileCheck prefixes: {}'.format(prefixes))
get_function_body(args, filename, clang_args, extra_commands, prefixes, triple_in_cmd, func_dict)

40
llvm/utils/update_llc_test_checks.py
View File

@ -43,8 +43,6 @@ def main():
test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
for test in test_paths:
if args.verbose:
print('Scanning for RUN lines in test file: %s' % (test,), file=sys.stderr)
with open(test) as f:
input_lines = [l.rstrip() for l in f]
@ -65,20 +63,7 @@ def main():
triple_in_ir = m.groups()[0]
break
raw_lines = [m.group(1)
for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
for l in raw_lines[1:]:
if run_lines[-1].endswith("\\"):
run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
else:
run_lines.append(l)
if args.verbose:
print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
for l in run_lines:
print(' RUN: ' + l, file=sys.stderr)
run_lines = common.find_run_lines(test, input_lines)
run_list = []
for l in run_lines:
if '|' not in l:
@ -115,12 +100,6 @@ def main():
llc_cmd_args = llc_cmd_args.replace('< %s', '').replace('%s', '').strip()
if test.endswith('.mir'):
llc_cmd_args += ' -x mir'
comment_sym = '#'
check_indent = ' '
else:
comment_sym = ';'
check_indent = ''
check_prefixes = [item for m in common.CHECK_PREFIX_RE.finditer(filecheck_cmd)
for item in m.group(1).split(',')]
if not check_prefixes:
@ -130,6 +109,12 @@ def main():
# now, we just ignore all but the last.
run_list.append((check_prefixes, llc_cmd_args, triple_in_cmd, march_in_cmd))
if test.endswith('.mir'):
comment_sym = '#'
check_indent = ' '
else:
comment_sym = ';'
check_indent = ''
autogenerated_note = (comment_sym + ADVERT + 'utils/' + script_name)
func_dict = {}
@ -138,9 +123,8 @@ def main():
for prefix in prefixes:
func_dict.update({prefix: dict()})
for prefixes, llc_args, triple_in_cmd, march_in_cmd in run_list:
if args.verbose:
print('Extracted LLC cmd: ' + llc_tool + ' ' + llc_args, file=sys.stderr)
print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
common.debug('Extracted LLC cmd:', llc_tool, llc_args)
common.debug('Extracted FileCheck prefixes:', str(prefixes))
raw_tool_output = common.invoke_tool(args.llc_binary, llc_args, test)
triple = triple_in_cmd or triple_in_ir
@ -154,8 +138,7 @@ def main():
is_in_function_start = False
func_name = None
prefix_set = set([prefix for p in run_list for prefix in p[0]])
if args.verbose:
print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
output_lines = []
output_lines.append(autogenerated_note)
@ -199,8 +182,7 @@ def main():
continue
is_in_function = is_in_function_start = True
if args.verbose:
print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
common.debug('Writing %d lines to %s...' % (len(output_lines), test))
with open(test, 'wb') as f:
f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])

30
llvm/utils/update_mca_test_checks.py
View File

@ -83,26 +83,6 @@ def _parse_args():
return args
def _find_run_lines(input_lines, args):
raw_lines = [m.group(1)
for m in [common.RUN_LINE_RE.match(l) for l in input_lines]
if m]
run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
for l in raw_lines[1:]:
if run_lines[-1].endswith(r'\\'):
run_lines[-1] = run_lines[-1].rstrip('\\') + ' ' + l
else:
run_lines.append(l)
if args.verbose:
sys.stderr.write('Found {} RUN line{}:\n'.format(
len(run_lines), '' if len(run_lines) == 1 else 's'))
for line in run_lines:
sys.stderr.write(' RUN: {}\n'.format(line))
return run_lines
def _get_run_infos(run_lines, args):
run_infos = []
for run_line in run_lines:
@ -544,9 +524,7 @@ def _write_output(test_path, input_lines, prefix_list, block_infos, # noqa
return
sys.stderr.write(' [{} lines total]\n'.format(len(output_lines)))
if args.verbose:
sys.stderr.write(
'Writing {} lines to {}...\n\n'.format(len(output_lines), test_path))
common.debug('Writing', len(output_lines), 'lines to', test_path, '..\n\n')
with open(test_path, 'wb') as f:
f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])
@ -562,17 +540,13 @@ def main():
# will be written once per source location per test.
_configure_warnings(args)
if args.verbose:
sys.stderr.write(
'Scanning for RUN lines in test file: {}\n'.format(test_path))
if not os.path.isfile(test_path):
raise Error('could not find test file: {}'.format(test_path))
with open(test_path) as f:
input_lines = [l.rstrip() for l in f]
run_lines = _find_run_lines(input_lines, args)
run_lines = common.find_run_lines(test_path, input_lines)
run_infos = _get_run_infos(run_lines, args)
common_prefix, prefix_pad = _get_useful_prefix_info(run_infos)
block_infos = _get_block_infos(run_infos, test_path, args, common_prefix)

19
llvm/utils/update_mir_test_checks.py
View File

@ -96,22 +96,6 @@ def find_triple_in_ir(lines, verbose=False):
return None
def find_run_lines(test, lines, verbose=False):
raw_lines = [m.group(1)
for m in [common.RUN_LINE_RE.match(l) for l in lines] if m]
run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
for l in raw_lines[1:]:
if run_lines[-1].endswith("\\"):
run_lines[-1] = run_lines[-1].rstrip("\\") + " " + l
else:
run_lines.append(l)
if verbose:
log('Found {} RUN lines:'.format(len(run_lines)))
for l in run_lines:
log(' RUN: {}'.format(l))
return run_lines
def build_run_list(test, run_lines, verbose=False):
run_list = []
all_prefixes = []
@ -296,7 +280,6 @@ def should_add_line_to_output(input_line, prefix_set):
def update_test_file(args, test):
log('Scanning for RUN lines in test file: {}'.format(test), args.verbose)
with open(test) as fd:
input_lines = [l.rstrip() for l in fd]
@ -313,7 +296,7 @@ def update_test_file(args, test):
return
triple_in_ir = find_triple_in_ir(input_lines, args.verbose)
run_lines = find_run_lines(test, input_lines, args.verbose)
run_lines = common.find_run_lines(test, input_lines)
run_list, common_prefixes = build_run_list(test, run_lines, args.verbose)
simple_functions = find_functions_with_one_bb(input_lines, args.verbose)

28
llvm/utils/update_test_checks.py
View File

@ -84,8 +84,6 @@ def main():
# On Windows we must expand the patterns ourselves.
test_paths = [test for pattern in args.tests for test in glob.glob(pattern)]
for test in test_paths:
if args.verbose:
print('Scanning for RUN lines in test file: ' + test, file=sys.stderr)
with open(test) as f:
input_lines = [l.rstrip() for l in f]
@ -99,20 +97,7 @@ def main():
common.warn("Skipping test which isn't autogenerated: " + test)
continue
raw_lines = [m.group(1)
for m in [common.RUN_LINE_RE.match(l) for l in input_lines] if m]
run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
for l in raw_lines[1:]:
if run_lines[-1].endswith('\\'):
run_lines[-1] = run_lines[-1].rstrip('\\') + ' ' + l
else:
run_lines.append(l)
if args.verbose:
print('Found %d RUN lines:' % (len(run_lines),), file=sys.stderr)
for l in run_lines:
print(' RUN: ' + l, file=sys.stderr)
run_lines = common.find_run_lines(test, input_lines)
prefix_list = []
for l in run_lines:
if '|' not in l:
@ -146,9 +131,8 @@ def main():
for prefix in prefixes:
func_dict.update({prefix: dict()})
for prefixes, opt_args in prefix_list:
if args.verbose:
print('Extracted opt cmd: ' + opt_basename + ' ' + opt_args, file=sys.stderr)
print('Extracted FileCheck prefixes: ' + str(prefixes), file=sys.stderr)
common.debug('Extracted opt cmd: ' + opt_basename + ' ' + opt_args)
common.debug('Extracted FileCheck prefixes: ' + str(prefixes))
raw_tool_output = common.invoke_tool(args.opt_binary, opt_args, test)
common.build_function_body_dictionary(
@ -159,8 +143,7 @@ def main():
is_in_function = False
is_in_function_start = False
prefix_set = set([prefix for prefixes, _ in prefix_list for prefix in prefixes])
if args.verbose:
print('Rewriting FileCheck prefixes: %s' % (prefix_set,), file=sys.stderr)
common.debug('Rewriting FileCheck prefixes:', str(prefix_set))
output_lines = []
output_lines.append(autogenerated_note)
@ -207,8 +190,7 @@ def main():
continue
is_in_function = is_in_function_start = True
if args.verbose:
print('Writing %d lines to %s...' % (len(output_lines), test), file=sys.stderr)
common.debug('Writing %d lines to %s...' % (len(output_lines), test))
with open(test, 'wb') as f:
f.writelines(['{}\n'.format(l).encode('utf-8') for l in output_lines])