maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

[Dexter][NFC] Add Debugger Controller To Dexter 

Add DebuggerControllerBase and DefaultController to Dexter
  implements a new architecture that supports new and novel ways of running
  a debugger under dexter.
  Current implementation adds the original default behaviour via the new
  architecture via the DefaultController, this should have NFC.

Reviewers: Orlando

Differential Revision: https://reviews.llvm.org/D76926
This commit is contained in:
Tom Weaver 2020-04-20 15:46:55 +01:00
parent ecf313c01d
commit 9cf9710bb0
15 changed files with 233 additions and 182 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
debuginfo-tests/dexter/dex/command/ParseCommand.py
View File

@ -13,7 +13,7 @@ import os
import unittest
from copy import copy
from collections import defaultdict
from collections import defaultdict, OrderedDict
from dex.utils.Exceptions import CommandParseError
@ -26,7 +26,8 @@ from dex.command.commands.DexExpectWatchValue import DexExpectWatchValue
from dex.command.commands.DexLabel import DexLabel
from dex.command.commands.DexUnreachable import DexUnreachable
from dex.command.commands.DexWatch import DexWatch
from dex.utils import Timer
from dex.utils.Exceptions import CommandParseError, DebuggerException
def _get_valid_commands():
"""Return all top level DExTer test commands.
@ -262,9 +263,7 @@ def _find_all_commands_in_file(path, file_lines, valid_commands):
raise format_parse_err(msg, path, file_lines, err_point)
return dict(commands)
def find_all_commands(source_files):
def _find_all_commands(source_files):
commands = defaultdict(dict)
valid_commands = _get_valid_commands()
for source_file in source_files:
@ -277,6 +276,21 @@ def find_all_commands(source_files):
return dict(commands)
def get_command_infos(source_files):
with Timer('parsing commands'):
try:
commands = _find_all_commands(source_files)
command_infos = OrderedDict()
for command_type in commands:
for command in commands[command_type].values():
if command_type not in command_infos:
command_infos[command_type] = []
command_infos[command_type].append(command)
return OrderedDict(command_infos)
except CommandParseError as e:
msg = 'parser error: <d>{}({}):</> {}\n{}\n{}\n'.format(
e.filename, e.lineno, e.info, e.src, e.caret)
raise DebuggerException(msg)
class TestParseCommand(unittest.TestCase):
class MockCmd(CommandBase):

2
debuginfo-tests/dexter/dex/command/__init__.py
View File

@ -5,5 +5,5 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
from dex.command.ParseCommand import find_all_commands
from dex.command.ParseCommand import get_command_infos
from dex.command.StepValueInfo import StepValueInfo

9
debuginfo-tests/dexter/dex/command/commands/DexExpectStepOrder.py
View File

@ -6,6 +6,7 @@
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
from dex.command.CommandBase import CommandBase
from dex.dextIR import LocIR
from dex.dextIR import ValueIR
class DexExpectStepOrder(CommandBase):
@ -28,11 +29,9 @@ class DexExpectStepOrder(CommandBase):
def get_name():
return __class__.__name__
def eval(self, debugger):
step_info = debugger.get_step_info()
loc = step_info.current_location
return {'DexExpectStepOrder': ValueIR(expression=str(loc.lineno),
value=str(debugger.step_index), type_name=None,
def eval(self, step_info):
return {'DexExpectStepOrder': ValueIR(expression=str(step_info.current_location.lineno),
value=str(step_info.step_index), type_name=None,
error_string=None,
could_evaluate=True,
is_optimized_away=True,

2
debuginfo-tests/dexter/dex/command/commands/DexUnreachable.py
View File

@ -26,7 +26,7 @@ class DexUnreachable(CommandBase):
def get_name():
return __class__.__name__
def eval(self, debugger):
def eval(self, step_info):
# If we're ever called, at all, then we're evaluating a line that has
# been marked as unreachable. Which means a failure.
vir = ValueIR(expression="Unreachable",

80
debuginfo-tests/dexter/dex/debugger/DebuggerBase.py
View File

@ -7,10 +7,7 @@
"""Base class for all debugger interface implementations."""
import abc
from itertools import chain
import os
import sys
import time
import traceback
from dex.dextIR import DebuggerIR, ValueIR
@ -20,14 +17,11 @@ from dex.utils.ReturnCode import ReturnCode
class DebuggerBase(object, metaclass=abc.ABCMeta):
def __init__(self, context, step_collection):
def __init__(self, context):
self.context = context
self.steps = step_collection
self._interface = None
self.has_loaded = False
self._loading_error = NotYetLoadedDebuggerException()
self.watches = set()
try:
self._interface = self._load_interface()
self.has_loaded = True
@ -35,13 +29,10 @@ class DebuggerBase(object, metaclass=abc.ABCMeta):
except DebuggerException:
self._loading_error = sys.exc_info()
self.step_index = 0
def __enter__(self):
try:
self._custom_init()
self.clear_breakpoints()
self.add_breakpoints()
except DebuggerException:
self._loading_error = sys.exc_info()
return self
@ -86,31 +77,6 @@ class DebuggerBase(object, metaclass=abc.ABCMeta):
tb = ''.join(tb).splitlines(True)
return tb
def add_breakpoints(self):
for s in self.context.options.source_files:
with open(s, 'r') as fp:
num_lines = len(fp.readlines())
for line in range(1, num_lines + 1):
self.add_breakpoint(s, line)
def _update_step_watches(self, step_info):
loc = step_info.current_location
watch_cmds = ['DexUnreachable', 'DexExpectStepOrder']
towatch = chain.from_iterable(self.steps.commands[x]
for x in watch_cmds
if x in self.steps.commands)
try:
# Iterate over all watches of the types named in watch_cmds
for watch in towatch:
if (os.path.exists(loc.path)
and os.path.samefile(watch.path, loc.path)
and watch.lineno == loc.lineno):
result = watch.eval(self)
step_info.watches.update(result)
break
except KeyError:
pass
def _sanitize_function_name(self, name): # pylint: disable=no-self-use
"""If the function name returned by the debugger needs any post-
processing to make it fit (for example, if it includes a byte offset),
@ -118,48 +84,6 @@ class DebuggerBase(object, metaclass=abc.ABCMeta):
"""
return name
def start(self):
self.steps.clear_steps()
self.launch()
for command_obj in chain.from_iterable(self.steps.commands.values()):
self.watches.update(command_obj.get_watches())
max_steps = self.context.options.max_steps
for _ in range(max_steps):
while self.is_running:
pass
if self.is_finished:
break
self.step_index += 1
step_info = self.get_step_info()
if step_info.current_frame:
self._update_step_watches(step_info)
self.steps.new_step(self.context, step_info)
if self.in_source_file(step_info):
self.step()
else:
self.go()
time.sleep(self.context.options.pause_between_steps)
else:
raise DebuggerException(
'maximum number of steps reached ({})'.format(max_steps))
def in_source_file(self, step_info):
if not step_info.current_frame:
return False
if not step_info.current_location.path:
return False
if not os.path.exists(step_info.current_location.path):
return False
return any(os.path.samefile(step_info.current_location.path, f) \
for f in self.context.options.source_files)
@abc.abstractmethod
def _load_interface(self):
pass
@ -209,7 +133,7 @@ class DebuggerBase(object, metaclass=abc.ABCMeta):
pass
@abc.abstractmethod
def get_step_info(self):
def get_step_info(self, watches, step_index):
pass
@abc.abstractproperty

27
debuginfo-tests/dexter/dex/debugger/DebuggerControllers/DebuggerControllerBase.py Normal file
View File

@ -0,0 +1,27 @@
# DExTer : Debugging Experience Tester
# ~~~~~~ ~ ~~ ~ ~~
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Default class for controlling debuggers."""
import abc
class DebuggerControllerBase(object, metaclass=abc.ABCMeta):
@abc.abstractclassmethod
def _run_debugger_custom(self):
"""Specify your own implementation of run_debugger_custom in your own
controller.
"""
pass
def run_debugger(self, debugger):
"""Responsible for correctly launching and tearing down the debugger.
"""
self.debugger = debugger
with self.debugger:
self._run_debugger_custom()
# We may need to pickle this debugger controller after running the
# debugger. Debuggers are not picklable objects, so set to None.
self.debugger = None

90
debuginfo-tests/dexter/dex/debugger/DebuggerControllers/DefaultController.py Normal file
View File

@ -0,0 +1,90 @@
# DExTer : Debugging Experience Tester
# ~~~~~~ ~ ~~ ~ ~~
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Base class for controlling debuggers."""
from itertools import chain
import os
import time
from dex.debugger.DebuggerControllers.DebuggerControllerBase import DebuggerControllerBase
from dex.utils.Exceptions import DebuggerException
class DefaultController(DebuggerControllerBase):
def __init__(self, context, step_collection):
self.context = context
self.step_collection = step_collection
self.watches = set()
self.step_index = 0
def _update_step_watches(self, step_info):
watch_cmds = ['DexUnreachable', 'DexExpectStepOrder']
towatch = chain.from_iterable(self.step_collection.commands[x]
for x in watch_cmds
if x in self.step_collection.commands)
try:
# Iterate over all watches of the types named in watch_cmds
for watch in towatch:
loc = step_info.current_location
if (os.path.exists(loc.path)
and os.path.samefile(watch.path, loc.path)
and watch.lineno == loc.lineno):
result = watch.eval(step_info)
step_info.watches.update(result)
break
except KeyError:
pass
def _break_point_all_lines(self):
for s in self.context.options.source_files:
with open(s, 'r') as fp:
num_lines = len(fp.readlines())
for line in range(1, num_lines + 1):
self.debugger.add_breakpoint(s, line)
def _in_source_file(self, step_info):
if not step_info.current_frame:
return False
if not step_info.current_location.path:
return False
if not os.path.exists(step_info.current_location.path):
return False
return any(os.path.samefile(step_info.current_location.path, f) \
for f in self.context.options.source_files)
def _run_debugger_custom(self):
self.step_collection.debugger = self.debugger.debugger_info
self._break_point_all_lines()
self.debugger.launch()
for command_obj in chain.from_iterable(self.step_collection.commands.values()):
self.watches.update(command_obj.get_watches())
max_steps = self.context.options.max_steps
for _ in range(max_steps):
while self.debugger.is_running:
pass
if self.debugger.is_finished:
break
self.step_index += 1
step_info = self.debugger.get_step_info(self.watches, self.step_index)
if step_info.current_frame:
self._update_step_watches(step_info)
self.step_collection.new_step(self.context, step_info)
if self._in_source_file(step_info):
self.debugger.step()
else:
self.debugger.go()
time.sleep(self.context.options.pause_between_steps)
else:
raise DebuggerException(
'maximum number of steps reached ({})'.format(max_steps))

70
debuginfo-tests/dexter/dex/debugger/Debuggers.py
View File

@ -13,13 +13,15 @@ import subprocess
import sys
from tempfile import NamedTemporaryFile
from dex.command import find_all_commands
from dex.command import get_command_infos
from dex.dextIR import DextIR
from dex.utils import get_root_directory, Timer
from dex.utils.Environment import is_native_windows
from dex.utils.Exceptions import CommandParseError, DebuggerException
from dex.utils.Exceptions import ToolArgumentError
from dex.utils.Warning import warn
from dex.utils.Exceptions import DebuggerException
from dex.debugger.DebuggerControllers.DefaultController import DefaultController
from dex.debugger.dbgeng.dbgeng import DbgEng
from dex.debugger.lldb.LLDB import LLDB
@ -133,55 +135,26 @@ def handle_debugger_tool_options(context, defaults): # noqa
_warn_meaningless_option(context, '--show-debugger')
def _get_command_infos(context):
commands = find_all_commands(context.options.source_files)
command_infos = OrderedDict()
for command_type in commands:
for command in commands[command_type].values():
if command_type not in command_infos:
command_infos[command_type] = []
command_infos[command_type].append(command)
return OrderedDict(command_infos)
def empty_debugger_steps(context):
return DextIR(
executable_path=context.options.executable,
source_paths=context.options.source_files,
dexter_version=context.version)
def get_debugger_steps(context):
step_collection = empty_debugger_steps(context)
with Timer('parsing commands'):
try:
step_collection.commands = _get_command_infos(context)
except CommandParseError as e:
msg = 'parser error: <d>{}({}):</> {}\n{}\n{}\n'.format(
e.filename, e.lineno, e.info, e.src, e.caret)
raise DebuggerException(msg)
def run_debugger_subprocess(debugger_controller, working_dir_path):
with NamedTemporaryFile(
dir=context.working_directory.path, delete=False) as fp:
pickle.dump(step_collection, fp, protocol=pickle.HIGHEST_PROTOCOL)
steps_path = fp.name
with NamedTemporaryFile(
dir=context.working_directory.path, delete=False, mode='wb') as fp:
pickle.dump(context.options, fp, protocol=pickle.HIGHEST_PROTOCOL)
options_path = fp.name
dir=working_dir_path, delete=False, mode='wb') as fp:
pickle.dump(debugger_controller, fp, protocol=pickle.HIGHEST_PROTOCOL)
controller_path = fp.name
dexter_py = os.path.basename(sys.argv[0])
if not os.path.isfile(dexter_py):
dexter_py = os.path.join(get_root_directory(), '..', dexter_py)
assert os.path.isfile(dexter_py)
with NamedTemporaryFile(dir=context.working_directory.path) as fp:
with NamedTemporaryFile(dir=working_dir_path) as fp:
args = [
sys.executable, dexter_py, 'run-debugger-internal-', steps_path,
options_path, '--working-directory', context.working_directory.path,
'--unittest=off', '--indent-timer-level={}'.format(Timer.indent + 2)
sys.executable,
dexter_py,
'run-debugger-internal-',
controller_path,
'--working-directory={}'.format(working_dir_path),
'--unittest=off',
'--indent-timer-level={}'.format(Timer.indent + 2)
]
try:
with Timer('running external debugger process'):
@ -189,10 +162,10 @@ def get_debugger_steps(context):
except subprocess.CalledProcessError as e:
raise DebuggerException(e)
with open(steps_path, 'rb') as fp:
step_collection = pickle.load(fp)
with open(controller_path, 'rb') as fp:
debugger_controller = pickle.load(fp)
return step_collection
return debugger_controller
class Debuggers(object):
@ -207,10 +180,9 @@ class Debuggers(object):
def __init__(self, context):
self.context = context
def load(self, key, step_collection=None):
def load(self, key):
with Timer('load {}'.format(key)):
return Debuggers.potential_debuggers()[key](self.context,
step_collection)
return Debuggers.potential_debuggers()[key](self.context)
def _populate_debugger_cache(self):
debuggers = []

2
debuginfo-tests/dexter/dex/debugger/__init__.py
View File

@ -6,3 +6,5 @@
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
from dex.debugger.Debuggers import Debuggers
from dex.debugger.DebuggerControllers.DebuggerControllerBase import DebuggerControllerBase
from dex.debugger.DebuggerControllers.DefaultController import DefaultController

6
debuginfo-tests/dexter/dex/debugger/dbgeng/dbgeng.py
View File

@ -96,7 +96,7 @@ class DbgEng(DebuggerBase):
# We never go -- we always single step.
pass
def get_step_info(self):
def get_step_info(self, watches, step_index):
frames = self.step_info
state_frames = []
@ -118,12 +118,12 @@ class DbgEng(DebuggerBase):
watches={})
for expr in map(
lambda watch, idx=i: self.evaluate_expression(watch, idx),
self.watches):
watches):
state_frame.watches[expr.expression] = expr
state_frames.append(state_frame)
return StepIR(
step_index=self.step_index, frames=dex_frames,
step_index=step_index, frames=dex_frames,
stop_reason=StopReason.STEP,
program_state=ProgramState(state_frames))

6
debuginfo-tests/dexter/dex/debugger/lldb/LLDB.py
View File

@ -124,7 +124,7 @@ class LLDB(DebuggerBase):
self._process.Continue()
return ReturnCode.OK
def get_step_info(self):
def get_step_info(self, watches, step_index):
frames = []
state_frames = []
@ -164,7 +164,7 @@ class LLDB(DebuggerBase):
watches={})
for expr in map(
lambda watch, idx=i: self.evaluate_expression(watch, idx),
self.watches):
watches):
state_frame.watches[expr.expression] = expr
state_frames.append(state_frame)
@ -175,7 +175,7 @@ class LLDB(DebuggerBase):
reason = self._translate_stop_reason(self._thread.GetStopReason())
return StepIR(
step_index=self.step_index, frames=frames, stop_reason=reason,
step_index=step_index, frames=frames, stop_reason=reason,
program_state=ProgramState(state_frames))
@property

6
debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py
View File

@ -131,7 +131,7 @@ class VisualStudio(DebuggerBase, metaclass=abc.ABCMeta): # pylint: disable=abst
raise Error('attempted to access stack frame {} out of {}'
.format(idx, len(stack_frames)))
def get_step_info(self):
def get_step_info(self, watches, step_index):
thread = self._debugger.CurrentThread
stackframes = thread.StackFrames
@ -154,7 +154,7 @@ class VisualStudio(DebuggerBase, metaclass=abc.ABCMeta): # pylint: disable=abst
is_inlined=frame.is_inlined,
watches={})
for watch in self.watches:
for watch in watches:
state_frame.watches[watch] = self.evaluate_expression(
watch, idx)
@ -174,7 +174,7 @@ class VisualStudio(DebuggerBase, metaclass=abc.ABCMeta): # pylint: disable=abst
program_state = ProgramState(frames=state_frames)
return StepIR(
step_index=self.step_index, frames=frames, stop_reason=reason,
step_index=step_index, frames=frames, stop_reason=reason,
program_state=program_state)
@property

25
debuginfo-tests/dexter/dex/tools/clang_opt_bisect/Tool.py
View File

@ -13,7 +13,10 @@ import re
import pickle
from dex.builder import run_external_build_script
from dex.debugger.Debuggers import empty_debugger_steps, get_debugger_steps
from dex.command.ParseCommand import get_command_infos
from dex.debugger.Debuggers import run_debugger_subprocess
from dex.debugger.DebuggerControllers.DefaultController import DefaultController
from dex.dextIR.DextIR import DextIR
from dex.heuristic import Heuristic
from dex.tools import TestToolBase
from dex.utils.Exceptions import DebuggerException, Error
@ -84,6 +87,16 @@ class Tool(TestToolBase):
"supported " % options.builder)
super(Tool, self).handle_options(defaults)
def _init_debugger_controller(self):
step_collection = DextIR(
executable_path=self.context.options.executable,
source_paths=self.context.options.source_files,
dexter_version=self.context.version)
step_collection.commands = get_command_infos(
self.context.options.source_files)
debugger_controller = DefaultController(self.context, step_collection)
return debugger_controller
def _run_test(self, test_name): # noqa
options = self.context.options
@ -123,9 +136,15 @@ class Tool(TestToolBase):
pass_info = (0, None, None)
try:
steps = get_debugger_steps(self.context)
debugger_controller =self._init_debugger_controller()
debugger_controller = run_debugger_subprocess(
self.context, debugger_controller)
steps = debugger_controller.step_collection
except DebuggerException:
steps = empty_debugger_steps(self.context)
steps = DextIR(
executable_path=self.context.options.executable,
source_paths=self.context.options.source_files,
dexter_version=self.context.version)
steps.builder = builderIR

46
debuginfo-tests/dexter/dex/tools/run_debugger_internal_/Tool.py
View File

@ -17,58 +17,46 @@ from dex.utils import Timer
from dex.utils.Exceptions import DebuggerException, Error
from dex.utils.ReturnCode import ReturnCode
class Tool(ToolBase):
def __init__(self, *args, **kwargs):
self.controller_path = None
self.debugger_controller = None
self.options = None
super(Tool, self).__init__(*args, **kwargs)
self.dextIR = None
@property
def name(self):
return 'DExTer run debugger internal'
def add_tool_arguments(self, parser, defaults):
parser.add_argument('dextIR_path', type=str, help='dextIR file')
parser.add_argument(
'pickled_options', type=str, help='pickled options file')
'controller_path',
type=str,
help='pickled debugger controller file')
def handle_options(self, defaults):
with open(self.context.options.dextIR_path, 'rb') as fp:
self.dextIR = pickle.load(fp)
with open(self.context.options.pickled_options, 'rb') as fp:
poptions = pickle.load(fp)
poptions.working_directory = (
self.context.options.working_directory[:])
poptions.unittest = self.context.options.unittest
poptions.dextIR_path = self.context.options.dextIR_path
self.context.options = poptions
Timer.display = self.context.options.time_report
with open(self.context.options.controller_path, 'rb') as fp:
self.debugger_controller = pickle.load(fp)
self.controller_path = self.context.options.controller_path
self.context = self.debugger_controller.context
self.options = self.context.options
Timer.display = self.options.time_report
def go(self) -> ReturnCode:
options = self.context.options
with Timer('loading debugger'):
debugger = Debuggers(self.context).load(options.debugger,
self.dextIR)
self.dextIR.debugger = debugger.debugger_info
debugger = Debuggers(self.context).load(self.options.debugger)
with Timer('running debugger'):
if not debugger.is_available:
msg = '<d>could not load {}</> ({})\n'.format(
debugger.name, debugger.loading_error)
if options.verbose:
if self.options.verbose:
msg = '{}\n {}'.format(
msg, ' '.join(debugger.loading_error_trace))
raise Error(msg)
with debugger:
try:
debugger.start()
except DebuggerException as e:
raise Error(e)
self.debugger_controller.run_debugger(debugger)
with open(self.context.options.dextIR_path, 'wb') as fp:
pickle.dump(self.dextIR, fp)
with open(self.controller_path, 'wb') as fp:
pickle.dump(self.debugger_controller, fp)
return ReturnCode.OK

20
debuginfo-tests/dexter/dex/tools/test/Tool.py
View File

@ -13,7 +13,10 @@ import pickle
import shutil
from dex.builder import run_external_build_script
from dex.debugger.Debuggers import get_debugger_steps
from dex.command.ParseCommand import get_command_infos
from dex.debugger.Debuggers import run_debugger_subprocess
from dex.debugger.DebuggerControllers.DefaultController import DefaultController
from dex.dextIR.DextIR import DextIR
from dex.heuristic import Heuristic
from dex.tools import TestToolBase
from dex.utils.Exceptions import DebuggerException
@ -128,10 +131,23 @@ class Tool(TestToolBase):
executable_file=options.executable)
return builderIR
def _init_debugger_controller(self):
step_collection = DextIR(
executable_path=self.context.options.executable,
source_paths=self.context.options.source_files,
dexter_version=self.context.version)
step_collection.commands = get_command_infos(
self.context.options.source_files)
debugger_controller = DefaultController(self.context, step_collection)
return debugger_controller
def _get_steps(self, builderIR):
"""Generate a list of debugger steps from a test case.
"""
steps = get_debugger_steps(self.context)
debugger_controller = self._init_debugger_controller()
debugger_controller = run_debugger_subprocess(
debugger_controller, self.context.working_directory.path)
steps = debugger_controller.step_collection
steps.builder = builderIR
return steps