2019-08-28 02:31:29 +08:00
|
|
|
#!/usr/bin/env python
|
2016-10-08 01:06:34 +08:00
|
|
|
|
|
|
|
|
from __future__ import print_function
|
|
|
|
|
|
|
|
|
|
import argparse
|
2017-06-30 02:56:25 +08:00
|
|
|
import cgi
|
2017-12-15 02:42:42 +08:00
|
|
|
import codecs
|
2017-06-30 02:56:25 +08:00
|
|
|
import errno
|
|
|
|
|
import functools
|
|
|
|
|
from multiprocessing import cpu_count
|
2016-10-08 01:06:34 +08:00
|
|
|
import os.path
|
2016-11-11 09:51:34 +08:00
|
|
|
import re
|
2016-10-08 01:06:34 +08:00
|
|
|
import shutil
|
[opt-viewer] Don't Decode HTML bytes for Python 2
Summary:
D36624 added some python3 compatibility. But that fix has a problem..
With python2 (which is specified by `#!/usr/bin/env python2.7`), if the env variables do not specify the UTF8,
and the source file is UTF8 (contains non-ASCII symbols), then the `.decode('utf-8')` causes the following exception:
```
Reading YAML files...
Rendering HTML files...
8 of 41Traceback (most recent call last):
File "/build/llvm/tools/opt-viewer/opt-viewer.py", line 277, in <module>
print_progress)
File "/build/llvm/tools/opt-viewer/opt-viewer.py", line 213, in generate_report
should_print_progress)
File "/build/llvm/tools/opt-viewer/optpmap.py", line 45, in pmap
result = map(_wrapped_func, func_and_args, *args, **kwargs)
File "/build/llvm/tools/opt-viewer/optpmap.py", line 25, in _wrapped_func
return func(argument)
File "/build/llvm/tools/opt-viewer/opt-viewer.py", line 174, in _render_file
SourceFileRenderer(source_dir, output_dir, filename).render(remarks)
File "/build/llvm/tools/opt-viewer/opt-viewer.py", line 125, in render
self.render_source_lines(self.source_stream, line_remarks)
File "/build/llvm/tools/opt-viewer/opt-viewer.py", line 79, in render_source_lines
</tr>'''.format(**locals()), file=self.stream)
UnicodeEncodeError: 'ascii' codec can't encode character u'\xf4' in position 47: ordinal not in range(128)
```
This is similar to https://bugs.llvm.org/show_bug.cgi?id=33548, which was fixed by https://reviews.llvm.org/D37661
Unlike that fix, here, *removing* `.decode('utf-8')` actually fixes it.
Since i assume that the original fix is needed, i simply made
that fix conditional, since for python2 it actually breaks things.
Reviewers: modocache, anemet
Reviewed By: anemet
Subscribers: fhahn, llvm-commits
Differential Revision: https://reviews.llvm.org/D38289
llvm-svn: 315350
2017-10-11 03:34:15 +08:00
|
|
|
import sys
|
2017-06-30 02:56:25 +08:00
|
|
|
|
2016-11-11 08:51:32 +08:00
|
|
|
from pygments import highlight
|
|
|
|
|
from pygments.lexers.c_cpp import CppLexer
|
|
|
|
|
from pygments.formatters import HtmlFormatter
|
2017-06-30 02:56:25 +08:00
|
|
|
|
|
|
|
|
import optpmap
|
|
|
|
|
import optrecord
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
desc = '''Generate HTML output to visualize optimization records from the YAML files
|
|
|
|
|
generated with -fsave-optimization-record and -fdiagnostics-show-hotness.
|
|
|
|
|
|
|
|
|
|
The tools requires PyYAML and Pygments Python packages.'''
|
|
|
|
|
|
2016-10-08 01:06:34 +08:00
|
|
|
|
2017-01-27 14:39:09 +08:00
|
|
|
# This allows passing the global context to the child processes.
|
|
|
|
|
class Context:
|
2017-03-02 05:35:00 +08:00
|
|
|
def __init__(self, caller_loc = dict()):
|
2017-01-27 14:39:09 +08:00
|
|
|
# Map function names to their source location for function where inlining happened
|
|
|
|
|
self.caller_loc = caller_loc
|
2016-11-15 16:40:51 +08:00
|
|
|
|
2017-01-27 14:39:09 +08:00
|
|
|
context = Context()
|
2016-11-11 14:11:56 +08:00
|
|
|
|
2017-12-07 00:50:50 +08:00
|
|
|
def suppress(remark):
|
|
|
|
|
if remark.Name == 'sil.Specialized':
|
|
|
|
|
return remark.getArgDict()['Function'][0].startswith('\"Swift.')
|
|
|
|
|
elif remark.Name == 'sil.Inlined':
|
|
|
|
|
return remark.getArgDict()['Callee'][0].startswith(('\"Swift.', '\"specialized Swift.'))
|
|
|
|
|
return False
|
|
|
|
|
|
2016-10-08 01:06:34 +08:00
|
|
|
class SourceFileRenderer:
|
2018-01-06 06:05:13 +08:00
|
|
|
def __init__(self, source_dir, output_dir, filename, no_highlight):
|
2018-02-27 05:15:50 +08:00
|
|
|
self.filename = filename
|
2016-11-11 09:08:02 +08:00
|
|
|
existing_filename = None
|
|
|
|
|
if os.path.exists(filename):
|
|
|
|
|
existing_filename = filename
|
|
|
|
|
else:
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
fn = os.path.join(source_dir, filename)
|
2016-11-11 09:08:02 +08:00
|
|
|
if os.path.exists(fn):
|
|
|
|
|
existing_filename = fn
|
|
|
|
|
|
2018-01-06 06:05:13 +08:00
|
|
|
self.no_highlight = no_highlight
|
2017-12-15 02:42:42 +08:00
|
|
|
self.stream = codecs.open(os.path.join(output_dir, optrecord.html_file_name(filename)), 'w', encoding='utf-8')
|
2016-11-11 09:08:02 +08:00
|
|
|
if existing_filename:
|
|
|
|
|
self.source_stream = open(existing_filename)
|
|
|
|
|
else:
|
|
|
|
|
self.source_stream = None
|
|
|
|
|
print('''
|
|
|
|
|
<html>
|
|
|
|
|
<h1>Unable to locate file {}</h1>
|
|
|
|
|
</html>
|
|
|
|
|
'''.format(filename), file=self.stream)
|
|
|
|
|
|
2017-02-10 12:50:18 +08:00
|
|
|
self.html_formatter = HtmlFormatter(encoding='utf-8')
|
2017-02-18 23:13:58 +08:00
|
|
|
self.cpp_lexer = CppLexer(stripnl=False)
|
2016-10-08 01:06:34 +08:00
|
|
|
|
2017-02-18 23:13:58 +08:00
|
|
|
def render_source_lines(self, stream, line_remarks):
|
|
|
|
|
file_text = stream.read()
|
2017-11-30 01:07:41 +08:00
|
|
|
|
2018-01-06 06:05:13 +08:00
|
|
|
if self.no_highlight:
|
2019-01-03 22:40:34 +08:00
|
|
|
if sys.version_info.major >= 3:
|
|
|
|
|
html_highlighted = file_text
|
|
|
|
|
else:
|
|
|
|
|
html_highlighted = file_text.decode('utf-8')
|
2017-11-30 01:07:41 +08:00
|
|
|
else:
|
|
|
|
|
html_highlighted = highlight(
|
2017-08-12 02:05:26 +08:00
|
|
|
file_text,
|
2017-11-30 01:07:41 +08:00
|
|
|
self.cpp_lexer,
|
|
|
|
|
self.html_formatter)
|
[opt-viewer] Don't Decode HTML bytes for Python 2
Summary:
D36624 added some python3 compatibility. But that fix has a problem..
With python2 (which is specified by `#!/usr/bin/env python2.7`), if the env variables do not specify the UTF8,
and the source file is UTF8 (contains non-ASCII symbols), then the `.decode('utf-8')` causes the following exception:
```
Reading YAML files...
Rendering HTML files...
8 of 41Traceback (most recent call last):
File "/build/llvm/tools/opt-viewer/opt-viewer.py", line 277, in <module>
print_progress)
File "/build/llvm/tools/opt-viewer/opt-viewer.py", line 213, in generate_report
should_print_progress)
File "/build/llvm/tools/opt-viewer/optpmap.py", line 45, in pmap
result = map(_wrapped_func, func_and_args, *args, **kwargs)
File "/build/llvm/tools/opt-viewer/optpmap.py", line 25, in _wrapped_func
return func(argument)
File "/build/llvm/tools/opt-viewer/opt-viewer.py", line 174, in _render_file
SourceFileRenderer(source_dir, output_dir, filename).render(remarks)
File "/build/llvm/tools/opt-viewer/opt-viewer.py", line 125, in render
self.render_source_lines(self.source_stream, line_remarks)
File "/build/llvm/tools/opt-viewer/opt-viewer.py", line 79, in render_source_lines
</tr>'''.format(**locals()), file=self.stream)
UnicodeEncodeError: 'ascii' codec can't encode character u'\xf4' in position 47: ordinal not in range(128)
```
This is similar to https://bugs.llvm.org/show_bug.cgi?id=33548, which was fixed by https://reviews.llvm.org/D37661
Unlike that fix, here, *removing* `.decode('utf-8')` actually fixes it.
Since i assume that the original fix is needed, i simply made
that fix conditional, since for python2 it actually breaks things.
Reviewers: modocache, anemet
Reviewed By: anemet
Subscribers: fhahn, llvm-commits
Differential Revision: https://reviews.llvm.org/D38289
llvm-svn: 315350
2017-10-11 03:34:15 +08:00
|
|
|
|
2017-12-15 02:42:42 +08:00
|
|
|
# Note that the API is different between Python 2 and 3. On
|
|
|
|
|
# Python 3, pygments.highlight() returns a bytes object, so we
|
|
|
|
|
# have to decode. On Python 2, the output is str but since we
|
|
|
|
|
# support unicode characters and the output streams is unicode we
|
|
|
|
|
# decode too.
|
|
|
|
|
html_highlighted = html_highlighted.decode('utf-8')
|
2017-02-18 23:13:58 +08:00
|
|
|
|
2017-11-30 01:07:41 +08:00
|
|
|
# Take off the header and footer, these must be
|
|
|
|
|
# reapplied line-wise, within the page structure
|
|
|
|
|
html_highlighted = html_highlighted.replace('<div class="highlight"><pre>', '')
|
|
|
|
|
html_highlighted = html_highlighted.replace('</pre></div>', '')
|
2017-02-18 23:13:58 +08:00
|
|
|
|
|
|
|
|
for (linenum, html_line) in enumerate(html_highlighted.split('\n'), start=1):
|
2017-12-15 02:42:42 +08:00
|
|
|
print(u'''
|
2016-10-08 01:06:34 +08:00
|
|
|
<tr>
|
|
|
|
|
<td><a name=\"L{linenum}\">{linenum}</a></td>
|
|
|
|
|
<td></td>
|
|
|
|
|
<td></td>
|
2017-02-18 23:13:58 +08:00
|
|
|
<td><div class="highlight"><pre>{html_line}</pre></div></td>
|
2016-10-08 01:06:34 +08:00
|
|
|
</tr>'''.format(**locals()), file=self.stream)
|
|
|
|
|
|
2017-02-18 23:13:58 +08:00
|
|
|
for remark in line_remarks.get(linenum, []):
|
2017-12-07 00:50:50 +08:00
|
|
|
if not suppress(remark):
|
|
|
|
|
self.render_inline_remarks(remark, html_line)
|
2017-02-18 23:13:58 +08:00
|
|
|
|
2016-11-11 09:51:34 +08:00
|
|
|
def render_inline_remarks(self, r, line):
|
2016-11-11 09:25:04 +08:00
|
|
|
inlining_context = r.DemangledFunctionName
|
2017-01-27 14:39:09 +08:00
|
|
|
dl = context.caller_loc.get(r.Function)
|
2016-11-11 09:25:04 +08:00
|
|
|
if dl:
|
2017-07-20 06:04:59 +08:00
|
|
|
dl_dict = dict(list(dl))
|
|
|
|
|
link = optrecord.make_link(dl_dict['File'], dl_dict['Line'] - 2)
|
2016-11-11 09:25:04 +08:00
|
|
|
inlining_context = "<a href={link}>{r.DemangledFunctionName}</a>".format(**locals())
|
|
|
|
|
|
2016-11-11 09:51:34 +08:00
|
|
|
# Column is the number of characters *including* tabs, keep those and
|
|
|
|
|
# replace everything else with spaces.
|
2017-03-01 07:59:46 +08:00
|
|
|
indent = line[:max(r.Column, 1) - 1]
|
2016-11-11 09:51:34 +08:00
|
|
|
indent = re.sub('\S', ' ', indent)
|
2017-02-02 13:49:02 +08:00
|
|
|
|
2019-01-26 01:48:31 +08:00
|
|
|
# Create expanded message and link if we have a multiline message.
|
|
|
|
|
lines = r.message.split('\n')
|
|
|
|
|
if len(lines) > 1:
|
|
|
|
|
expand_link = '<a style="text-decoration: none;" href="" onclick="toggleExpandedMessage(this); return false;">+</a>'
|
|
|
|
|
message = lines[0]
|
|
|
|
|
expand_message = u'''
|
|
|
|
|
<div class="full-info" style="display:none;">
|
|
|
|
|
<div class="col-left"><pre style="display:inline">{}</pre></div>
|
|
|
|
|
<div class="expanded col-left"><pre>{}</pre></div>
|
|
|
|
|
</div>'''.format(indent, '\n'.join(lines[1:]))
|
|
|
|
|
else:
|
|
|
|
|
expand_link = ''
|
|
|
|
|
expand_message = ''
|
|
|
|
|
message = r.message
|
2017-12-15 02:42:42 +08:00
|
|
|
print(u'''
|
2016-10-08 01:06:34 +08:00
|
|
|
<tr>
|
|
|
|
|
<td></td>
|
2016-11-11 14:11:56 +08:00
|
|
|
<td>{r.RelativeHotness}</td>
|
2017-03-03 01:00:59 +08:00
|
|
|
<td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
|
2019-01-26 01:48:31 +08:00
|
|
|
<td><pre style="display:inline">{indent}</pre><span class=\"column-entry-yellow\">{expand_link} {message} </span>{expand_message}</td>
|
2016-11-11 09:25:04 +08:00
|
|
|
<td class=\"column-entry-yellow\">{inlining_context}</td>
|
2016-10-08 01:06:34 +08:00
|
|
|
</tr>'''.format(**locals()), file=self.stream)
|
|
|
|
|
|
|
|
|
|
def render(self, line_remarks):
|
2016-11-11 09:08:02 +08:00
|
|
|
if not self.source_stream:
|
|
|
|
|
return
|
|
|
|
|
|
2016-10-08 01:06:34 +08:00
|
|
|
print('''
|
|
|
|
|
<html>
|
2018-02-27 05:15:50 +08:00
|
|
|
<title>{}</title>
|
2017-12-15 02:55:33 +08:00
|
|
|
<meta charset="utf-8" />
|
2016-10-08 01:06:34 +08:00
|
|
|
<head>
|
|
|
|
|
<link rel='stylesheet' type='text/css' href='style.css'>
|
2019-01-26 01:48:31 +08:00
|
|
|
<script type="text/javascript">
|
|
|
|
|
/* Simple helper to show/hide the expanded message of a remark. */
|
|
|
|
|
function toggleExpandedMessage(e) {{
|
|
|
|
|
var FullTextElems = e.parentElement.parentElement.getElementsByClassName("full-info");
|
|
|
|
|
if (!FullTextElems || FullTextElems.length < 1) {{
|
|
|
|
|
return false;
|
|
|
|
|
}}
|
|
|
|
|
var FullText = FullTextElems[0];
|
|
|
|
|
if (FullText.style.display == 'none') {{
|
|
|
|
|
e.innerHTML = '-';
|
|
|
|
|
FullText.style.display = 'block';
|
|
|
|
|
}} else {{
|
|
|
|
|
e.innerHTML = '+';
|
|
|
|
|
FullText.style.display = 'none';
|
|
|
|
|
}}
|
|
|
|
|
}}
|
|
|
|
|
</script>
|
2016-10-08 01:06:34 +08:00
|
|
|
</head>
|
|
|
|
|
<body>
|
|
|
|
|
<div class="centered">
|
2017-11-14 12:48:18 +08:00
|
|
|
<table class="source">
|
|
|
|
|
<thead>
|
2016-10-08 01:06:34 +08:00
|
|
|
<tr>
|
2017-11-14 12:48:18 +08:00
|
|
|
<th style="width: 2%">Line</td>
|
|
|
|
|
<th style="width: 3%">Hotness</td>
|
|
|
|
|
<th style="width: 10%">Optimization</td>
|
|
|
|
|
<th style="width: 70%">Source</td>
|
|
|
|
|
<th style="width: 15%">Inline Context</td>
|
|
|
|
|
</tr>
|
|
|
|
|
</thead>
|
2018-02-27 05:15:50 +08:00
|
|
|
<tbody>'''.format(os.path.basename(self.filename)), file=self.stream)
|
2017-02-18 23:13:58 +08:00
|
|
|
self.render_source_lines(self.source_stream, line_remarks)
|
|
|
|
|
|
2016-10-08 01:06:34 +08:00
|
|
|
print('''
|
2017-11-14 12:48:18 +08:00
|
|
|
</tbody>
|
2016-10-08 01:06:34 +08:00
|
|
|
</table>
|
|
|
|
|
</body>
|
|
|
|
|
</html>''', file=self.stream)
|
|
|
|
|
|
2016-11-11 12:51:27 +08:00
|
|
|
|
2016-10-08 01:06:34 +08:00
|
|
|
class IndexRenderer:
|
2018-01-06 06:05:13 +08:00
|
|
|
def __init__(self, output_dir, should_display_hotness, max_hottest_remarks_on_index):
|
2017-12-15 02:42:42 +08:00
|
|
|
self.stream = codecs.open(os.path.join(output_dir, 'index.html'), 'w', encoding='utf-8')
|
2017-11-14 12:37:32 +08:00
|
|
|
self.should_display_hotness = should_display_hotness
|
2018-01-06 06:05:13 +08:00
|
|
|
self.max_hottest_remarks_on_index = max_hottest_remarks_on_index
|
2016-10-08 01:06:34 +08:00
|
|
|
|
2017-02-02 13:49:00 +08:00
|
|
|
def render_entry(self, r, odd):
|
2017-02-02 13:49:02 +08:00
|
|
|
escaped_name = cgi.escape(r.DemangledFunctionName)
|
2017-12-15 02:42:42 +08:00
|
|
|
print(u'''
|
2016-10-12 00:19:06 +08:00
|
|
|
<tr>
|
2017-02-02 13:49:00 +08:00
|
|
|
<td class=\"column-entry-{odd}\"><a href={r.Link}>{r.DebugLocString}</a></td>
|
|
|
|
|
<td class=\"column-entry-{odd}\">{r.RelativeHotness}</td>
|
2017-02-02 13:49:02 +08:00
|
|
|
<td class=\"column-entry-{odd}\">{escaped_name}</td>
|
2017-03-03 01:00:59 +08:00
|
|
|
<td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
|
2016-10-12 00:19:06 +08:00
|
|
|
</tr>'''.format(**locals()), file=self.stream)
|
2016-10-08 01:06:34 +08:00
|
|
|
|
|
|
|
|
def render(self, all_remarks):
|
|
|
|
|
print('''
|
|
|
|
|
<html>
|
2017-12-15 02:55:33 +08:00
|
|
|
<meta charset="utf-8" />
|
2016-10-08 01:06:34 +08:00
|
|
|
<head>
|
|
|
|
|
<link rel='stylesheet' type='text/css' href='style.css'>
|
|
|
|
|
</head>
|
|
|
|
|
<body>
|
|
|
|
|
<div class="centered">
|
|
|
|
|
<table>
|
|
|
|
|
<tr>
|
|
|
|
|
<td>Source Location</td>
|
|
|
|
|
<td>Hotness</td>
|
|
|
|
|
<td>Function</td>
|
|
|
|
|
<td>Pass</td>
|
|
|
|
|
</tr>''', file=self.stream)
|
2017-11-14 12:37:32 +08:00
|
|
|
|
|
|
|
|
max_entries = None
|
2018-01-06 06:05:13 +08:00
|
|
|
if self.should_display_hotness:
|
|
|
|
|
max_entries = self.max_hottest_remarks_on_index
|
2017-11-14 12:37:32 +08:00
|
|
|
|
|
|
|
|
for i, remark in enumerate(all_remarks[:max_entries]):
|
2017-12-07 00:50:50 +08:00
|
|
|
if not suppress(remark):
|
|
|
|
|
self.render_entry(remark, i % 2)
|
2016-10-08 01:06:34 +08:00
|
|
|
print('''
|
|
|
|
|
</table>
|
|
|
|
|
</body>
|
|
|
|
|
</html>''', file=self.stream)
|
|
|
|
|
|
|
|
|
|
|
2019-02-07 02:43:37 +08:00
|
|
|
def _render_file(source_dir, output_dir, ctx, no_highlight, entry, filter_):
|
2017-01-27 14:39:09 +08:00
|
|
|
global context
|
|
|
|
|
context = ctx
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
filename, remarks = entry
|
2018-01-06 06:05:13 +08:00
|
|
|
SourceFileRenderer(source_dir, output_dir, filename, no_highlight).render(remarks)
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
def map_remarks(all_remarks):
|
|
|
|
|
# Set up a map between function names and their source location for
|
|
|
|
|
# function where inlining happened
|
2017-06-30 02:47:31 +08:00
|
|
|
for remark in optrecord.itervalues(all_remarks):
|
2017-03-02 05:35:00 +08:00
|
|
|
if isinstance(remark, optrecord.Passed) and remark.Pass == "inline" and remark.Name == "Inlined":
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
for arg in remark.Args:
|
2017-07-20 06:04:59 +08:00
|
|
|
arg_dict = dict(list(arg))
|
|
|
|
|
caller = arg_dict.get('Caller')
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
if caller:
|
2017-07-13 12:19:13 +08:00
|
|
|
try:
|
2017-07-20 06:04:59 +08:00
|
|
|
context.caller_loc[caller] = arg_dict['DebugLoc']
|
2017-07-13 12:19:13 +08:00
|
|
|
except KeyError:
|
|
|
|
|
pass
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
|
|
|
|
|
|
2017-06-30 02:56:25 +08:00
|
|
|
def generate_report(all_remarks,
|
|
|
|
|
file_remarks,
|
|
|
|
|
source_dir,
|
|
|
|
|
output_dir,
|
2018-01-06 06:05:13 +08:00
|
|
|
no_highlight,
|
2017-06-30 02:56:25 +08:00
|
|
|
should_display_hotness,
|
2018-01-06 06:05:13 +08:00
|
|
|
max_hottest_remarks_on_index,
|
2017-06-30 02:56:25 +08:00
|
|
|
num_jobs,
|
|
|
|
|
should_print_progress):
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
try:
|
|
|
|
|
os.makedirs(output_dir)
|
|
|
|
|
except OSError as e:
|
|
|
|
|
if e.errno == errno.EEXIST and os.path.isdir(output_dir):
|
|
|
|
|
pass
|
|
|
|
|
else:
|
|
|
|
|
raise
|
|
|
|
|
|
2017-06-30 02:56:25 +08:00
|
|
|
if should_print_progress:
|
2018-02-27 05:15:47 +08:00
|
|
|
print('Rendering index page...')
|
2017-03-02 05:35:00 +08:00
|
|
|
if should_display_hotness:
|
2017-06-30 02:47:31 +08:00
|
|
|
sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.Hotness, r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function), reverse=True)
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
else:
|
2017-06-30 02:47:31 +08:00
|
|
|
sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function))
|
2018-01-06 06:05:13 +08:00
|
|
|
IndexRenderer(output_dir, should_display_hotness, max_hottest_remarks_on_index).render(sorted_remarks)
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
|
|
|
|
|
shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)),
|
|
|
|
|
"style.css"), output_dir)
|
|
|
|
|
|
2018-02-27 05:15:47 +08:00
|
|
|
_render_file_bound = functools.partial(_render_file, source_dir, output_dir, context, no_highlight)
|
|
|
|
|
if should_print_progress:
|
|
|
|
|
print('Rendering HTML files...')
|
|
|
|
|
optpmap.pmap(_render_file_bound,
|
|
|
|
|
file_remarks.items(),
|
|
|
|
|
num_jobs,
|
|
|
|
|
should_print_progress)
|
|
|
|
|
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
|
2018-01-06 06:05:13 +08:00
|
|
|
def main():
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
parser = argparse.ArgumentParser(description=desc)
|
2017-07-18 02:00:41 +08:00
|
|
|
parser.add_argument(
|
|
|
|
|
'yaml_dirs_or_files',
|
|
|
|
|
nargs='+',
|
|
|
|
|
help='List of optimization record files or directories searched '
|
|
|
|
|
'for optimization record files.')
|
2017-06-30 00:20:31 +08:00
|
|
|
parser.add_argument(
|
|
|
|
|
'--output-dir',
|
|
|
|
|
'-o',
|
|
|
|
|
default='html',
|
|
|
|
|
help='Path to a directory where generated HTML files will be output. '
|
|
|
|
|
'If the directory does not already exist, it will be created. '
|
|
|
|
|
'"%(default)s" by default.')
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
parser.add_argument(
|
|
|
|
|
'--jobs',
|
|
|
|
|
'-j',
|
2018-01-06 06:05:13 +08:00
|
|
|
default=None,
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
type=int,
|
2017-06-11 05:33:27 +08:00
|
|
|
help='Max job count (defaults to %(default)s, the current CPU count)')
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
parser.add_argument(
|
2017-11-28 03:00:22 +08:00
|
|
|
'--source-dir',
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
'-s',
|
|
|
|
|
default='',
|
|
|
|
|
help='set source directory')
|
2017-06-30 02:56:25 +08:00
|
|
|
parser.add_argument(
|
|
|
|
|
'--no-progress-indicator',
|
|
|
|
|
'-n',
|
|
|
|
|
action='store_true',
|
|
|
|
|
default=False,
|
|
|
|
|
help='Do not display any indicator of how many YAML files were read '
|
|
|
|
|
'or rendered into HTML.')
|
2017-11-14 12:37:32 +08:00
|
|
|
parser.add_argument(
|
|
|
|
|
'--max-hottest-remarks-on-index',
|
|
|
|
|
default=1000,
|
|
|
|
|
type=int,
|
|
|
|
|
help='Maximum number of the hottest remarks to appear on the index page')
|
2017-11-30 01:07:41 +08:00
|
|
|
parser.add_argument(
|
|
|
|
|
'--no-highlight',
|
|
|
|
|
action='store_true',
|
|
|
|
|
default=False,
|
|
|
|
|
help='Do not use a syntax highlighter when rendering the source code')
|
|
|
|
|
parser.add_argument(
|
|
|
|
|
'--demangler',
|
|
|
|
|
help='Set the demangler to be used (defaults to %s)' % optrecord.Remark.default_demangler)
|
2018-01-06 06:05:13 +08:00
|
|
|
|
2019-02-07 02:43:37 +08:00
|
|
|
parser.add_argument(
|
|
|
|
|
'--filter',
|
|
|
|
|
default='',
|
|
|
|
|
help='Only display remarks from passes matching filter expression')
|
|
|
|
|
|
2018-01-06 06:05:13 +08:00
|
|
|
# Do not make this a global variable. Values needed to be propagated through
|
|
|
|
|
# to individual classes and functions to be portable with multiprocessing across
|
|
|
|
|
# Windows and non-Windows.
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
args = parser.parse_args()
|
|
|
|
|
|
2017-06-30 02:56:25 +08:00
|
|
|
print_progress = not args.no_progress_indicator
|
2017-11-30 01:07:41 +08:00
|
|
|
if args.demangler:
|
|
|
|
|
optrecord.Remark.set_demangler(args.demangler)
|
2017-07-18 02:00:41 +08:00
|
|
|
|
2017-09-29 13:20:53 +08:00
|
|
|
files = optrecord.find_opt_files(*args.yaml_dirs_or_files)
|
2017-07-18 02:00:41 +08:00
|
|
|
if not files:
|
|
|
|
|
parser.error("No *.opt.yaml files found")
|
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
2017-06-30 02:56:25 +08:00
|
|
|
all_remarks, file_remarks, should_display_hotness = \
|
2019-02-07 02:43:37 +08:00
|
|
|
optrecord.gather_results(files, args.jobs, print_progress, args.filter)
|
[opt-viewer] Put critical items in parallel
Summary:
Put opt-viewer critical items in parallel
Patch by Brian Cain!
Requires features from Python 2.7
**Performance**
Below are performance results across various configurations. These were taken on an i5-5200U (dual core + HT). They were taken with a small subset of the YAML output of building Python 3.6.0b3 with LTO+PGO. 60 YAML files.
"multiprocessing" is the current submission contents. "baseline" is as of 544f14c6b2a07a94168df31833dba9dc35fd8289 (I think this is aka r287505).
"ImportError" vs "class<...CLoader>" below are just confirming the expected configuration (with/without CLoader).
The below was measured on AMD A8-5500B (4 cores) with 224 input YAML files, showing a ~1.75x speed increase over the baseline with libYAML. I suspect it would scale well on high-end servers.
```
**************************************** MULTIPROCESSING ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
489.42user 5.53system 2:38.03elapsed 313%CPU (0avgtext+0avgdata 400308maxresident)k
0inputs+31392outputs (0major+473540minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
78.69user 5.45system 0:32.63elapsed 257%CPU (0avgtext+0avgdata 398560maxresident)k
0inputs+31392outputs (0major+542022minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
154.27user 8.12system 0:53.83elapsed 301%CPU (0avgtext+0avgdata 627960maxresident)k
808inputs+30376outputs (0major+727994minor)pagefaults 0swaps
**************************************** BASELINE ****************************************
PyYAML:
Traceback (most recent call last):
File "<string>", line 1, in <module>
ImportError: cannot import name CLoader
Python 2.7.10
358.08user 4.05system 6:08.37elapsed 98%CPU (0avgtext+0avgdata 315004maxresident)k
0inputs+31392outputs (0major+85252minor)pagefaults 0swaps
PyYAML+libYAML:
<class 'yaml.cyaml.CLoader'>
Python 2.7.10
50.32user 3.30system 0:56.59elapsed 94%CPU (0avgtext+0avgdata 307296maxresident)k
0inputs+31392outputs (0major+79335minor)pagefaults 0swaps
PyPy/PyYAML:
Traceback (most recent call last):
File "<builtin>/app_main.py", line 75, in run_toplevel
File "<builtin>/app_main.py", line 601, in run_it
File "<string>", line 1, in <module>
ImportError: cannot import name 'CLoader'
Python 2.7.9 (2.6.0+dfsg-3, Jul 04 2015, 05:43:17)
[PyPy 2.6.0 with GCC 4.9.3]
72.94user 5.18system 1:23.41elapsed 93%CPU (0avgtext+0avgdata 455312maxresident)k
0inputs+30392outputs (0major+110280minor)pagefaults 0swaps
```
Reviewers: fhahn, anemet
Reviewed By: anemet
Subscribers: llvm-commits, mehdi_amini
Differential Revision: https://reviews.llvm.org/D26967
llvm-svn: 293261
2017-01-27 14:38:31 +08:00
|
|
|
|
|
|
|
|
map_remarks(all_remarks)
|
|
|
|
|
|
2017-06-30 02:56:25 +08:00
|
|
|
generate_report(all_remarks,
|
|
|
|
|
file_remarks,
|
|
|
|
|
args.source_dir,
|
|
|
|
|
args.output_dir,
|
2018-01-06 06:05:13 +08:00
|
|
|
args.no_highlight,
|
2017-06-30 02:56:25 +08:00
|
|
|
should_display_hotness,
|
2018-01-06 06:05:13 +08:00
|
|
|
args.max_hottest_remarks_on_index,
|
2017-06-30 02:56:25 +08:00
|
|
|
args.jobs,
|
|
|
|
|
print_progress)
|
2018-01-06 06:05:13 +08:00
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
main()
|