Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!6016 Fix the unsafe python code in profiler 

Merge pull request !6016 from gzhcv/bugfix_bak
This commit is contained in:
mindspore-ci-bot 2020-09-19 23:44:01 +08:00 committed by Gitee
parent 861602bce9 3792180ee9
commit e6803a12b5
9 changed files with 42 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
mindspore/profiler/parser/aicpu_data_parser.py
View File

@ -16,6 +16,7 @@
The parser for AI CPU preprocess data.
"""
import os
import stat
from mindspore.profiler.common.util import fwrite_format, get_file_join_name
from mindspore import log as logger
@ -96,7 +97,7 @@ class DataPreProcessParser:
ai_cpu_str = str(ai_cpu_data.read().replace(b'\n\x00', b' ___ ')
.replace(b'\x00', b' ___ '))[2:-1]
ai_cpu_lines = ai_cpu_str.split(" ___ ")
os.chmod(self._source_file_name, stat.S_IREAD | stat.S_IWRITE)
result_list = list()
ai_cpu_total_time_summary = 0
# Node serial number.

4
mindspore/profiler/parser/framework_parser.py
View File

@ -18,6 +18,7 @@ import enum
import json
import os
import re
import stat
from mindspore.profiler.common.exceptions.exceptions import \
ProfilerPathErrorException, ProfilerDirNotFoundException, \
@ -447,6 +448,7 @@ class FrameworkParser:
def _parse_task_files(self):
"""Parse the framework task files."""
for path in self._framework_path['task']:
path = validate_and_normalize_path(path)
with open(path, 'r') as file:
for task_info in file:
infos = task_info.strip('\n').split(' ')
@ -490,6 +492,7 @@ class FrameworkParser:
value.append(key)
value.extend(none_list)
csv_writer.writerow(value)
os.chmod(self._save_path, stat.S_IREAD | stat.S_IWRITE)
def _parse_one_row_graph_info(self, row_info):
"""
@ -591,6 +594,7 @@ class FrameworkParser:
def _parse_point_files(self):
"""Parse the framework point files."""
for path in self._framework_path['point']:
path = validate_and_normalize_path(path)
with open(path, 'r') as file:
for point_info in file:
infos = point_info.strip('\n').split(' ')

7
mindspore/profiler/parser/hwts_log_parser.py
View File

@ -17,7 +17,8 @@ import os
import struct
from mindspore.profiler.common.util import fwrite_format, get_file_join_name
from mindspore import log as logger
from mindspore.profiler.common.validator.validate_path import \
validate_and_normalize_path
class HWTSLogParser:
"""
@ -68,8 +69,10 @@ class HWTSLogParser:
result_data = ""
self._source_flie_name = validate_and_normalize_path(self._source_flie_name)
with open(self._source_flie_name, 'rb') as hwts_data:
while True:
# read 64 bit data
line = hwts_data.read(64)
if line:
if not line.strip():
@ -77,6 +80,8 @@ class HWTSLogParser:
else:
break
byte_first_four = struct.unpack('BBHHH', line[0:8])
# byte_first[0:4] refers to count. byte_first[4] refers to is_warn_res0_0v.
# byte_first[5:8] refers to the type of ms.
byte_first = bin(byte_first_four[0]).replace('0b', '').zfill(8)
ms_type = byte_first[-3:]
is_warn_res0_ov = byte_first[4]

11
mindspore/profiler/parser/integrator.py
View File

@ -16,6 +16,7 @@
import csv
import json
import os
import stat
from decimal import Decimal
from mindspore import log as logger
@ -91,6 +92,7 @@ class Integrator:
self._profiling_dir,
self._file_name_framework.format(self._device_id)
)
framework_file = validate_and_normalize_path(framework_file)
if not os.path.isfile(framework_file):
return
@ -130,6 +132,7 @@ class Integrator:
self._profiling_dir,
self._file_name_aicore_detail_time.format(self._device_id)
)
aicore_detail_file = validate_and_normalize_path(aicore_detail_file)
if not os.path.isfile(aicore_detail_file):
return
@ -169,6 +172,7 @@ class Integrator:
self._profiling_dir,
self._file_name_aicpu_time.format(self._device_id)
)
aicpu_file = validate_and_normalize_path(aicpu_file)
if not os.path.isfile(aicpu_file):
return
@ -197,6 +201,7 @@ class Integrator:
self._profiling_dir,
self._file_name_aicore_type_time.format(self._device_id)
)
op_type_file_path = validate_and_normalize_path(op_type_file_path)
if not os.path.isfile(op_type_file_path):
logger.warning('The file <%s> does not exist.', op_type_file_path)
return
@ -217,6 +222,8 @@ class Integrator:
self._profiling_dir,
self._file_name_framework.format(self._device_id)
)
op_detail_file_path = validate_and_normalize_path(op_detail_file_path)
framework_file_path = validate_and_normalize_path(framework_file_path)
if not os.path.isfile(op_detail_file_path):
logger.warning('The file <%s> does not exist.', op_detail_file_path)
return
@ -251,6 +258,7 @@ class Integrator:
if not file_path:
logger.error("Failed to find parsed trace time file.")
raise ProfilerFileNotFoundException('parsed step trace time file')
file_path = validate_and_normalize_path(file_path)
with open(file_path, 'r') as handle:
csv_reader = csv.reader(handle)
self.__column__ = next(csv_reader)
@ -261,6 +269,7 @@ class Integrator:
def _load_point_info(self):
"""Load point info."""
file_path = os.path.join(self._profiling_dir, 'step_trace_point_info.json')
file_path = validate_and_normalize_path(file_path)
if os.path.isfile(file_path):
with open(file_path, 'r', encoding='utf-8') as file:
try:
@ -540,6 +549,7 @@ class BaseTimelineGenerator:
break
json_file.write(',')
json_file.write(']')
os.chmod(display_file_path, stat.S_IREAD | stat.S_IWRITE)
except (IOError, OSError) as err:
logger.error('Error occurred when write timeline display file: %s', err)
raise ProfilerIOException
@ -556,6 +566,7 @@ class BaseTimelineGenerator:
try:
with open(timeline_summary_file_path, 'w') as json_file:
json.dump(self._timeline_summary, json_file)
os.chmod(timeline_summary_file_path, stat.S_IREAD | stat.S_IWRITE)
except (IOError, OSError) as err:
logger.error('Error occurred when write timeline summary file: %s', err)
raise ProfilerIOException

5
mindspore/profiler/parser/minddata_parser.py
View File

@ -17,7 +17,8 @@ import os
from mindspore.profiler.common.util import get_file_join_name, fwrite_format
from mindspore import log as logger
from mindspore.profiler.common.validator.validate_path import \
validate_and_normalize_path
class MinddataParser:
"""Minddata Aicpu Parser."""
@ -34,6 +35,7 @@ class MinddataParser:
"""
result = list()
try:
minddata_aicpu_source_path = validate_and_normalize_path(minddata_aicpu_source_path)
with open(minddata_aicpu_source_path) as source_data_file:
source_data = source_data_file.read()
step_data = source_data.split("\x00")
@ -77,6 +79,7 @@ class MinddataParser:
device_id (str): the device id.
"""
col_names = ["node_name", "start_time", "end_time", "queue_size"]
source_path = validate_and_normalize_path(source_path)
minddata_aicpu_source_path = get_file_join_name(
input_path=source_path, file_name='DATA_PREPROCESS.AICPUMI')
if not minddata_aicpu_source_path:

2
mindspore/profiler/parser/minddata_pipeline_parser.py
View File

@ -16,6 +16,7 @@
import csv
import json
import os
import stat
from queue import Queue
from mindspore.profiler.common.exceptions.exceptions import \
@ -167,6 +168,7 @@ class MinddataPipelineParser:
self._parse_and_save_op_info(
csv_writer, op_id_info_cache, sample_interval
)
os.chmod(self._save_path, stat.S_IREAD | stat.S_IWRITE)
def _parse_and_save_op_info(self, csv_writer, op_id_info_cache,
sample_interval):

2
mindspore/profiler/parser/optime_parser.py
View File

@ -14,6 +14,7 @@
# ============================================================================
"""Op compute time files parser."""
import os
import stat
from mindspore.profiler.common.util import fwrite_format
from mindspore.profiler.common.exceptions.exceptions import ProfilerFileNotFoundException, \
ProfilerIOException
@ -163,6 +164,7 @@ class OPComputeTimeParser:
for timeline in timeline_data:
timeline = [str(item) for item in timeline]
f_obj.write(','.join(timeline) + '\n')
os.chmod(file_path, stat.S_IREAD | stat.S_IWRITE)
except (IOError, OSError) as err:
logger.error('Error occurred when writing intermediate timeline file: %s', err)
raise ProfilerIOException

3
mindspore/profiler/parser/step_trace_parser.py
View File

@ -25,6 +25,8 @@ from mindspore.profiler.common.exceptions.exceptions import ProfilerPathErrorExc
JobIdMismatchException, ProfilerIOException
from mindspore import log
from mindspore.profiler.common.util import get_summary_for_step_trace
from mindspore.profiler.common.validator.validate_path import \
validate_and_normalize_path
StepTraceStruct = namedtuple(
'TrainingTraceStruct', ['tag_id', 'task_id', 'stream_id', 'sys_count']
@ -199,6 +201,7 @@ class StepTraceParser:
log.info("Start to parse step trace file.")
event_info = {}
for source_file in source_files:
source_file = validate_and_normalize_path(source_file)
with open(source_file, 'rb') as handler:
content = handler.read()
for step_trace in self._get_next_step_trace(content, event_info):

12
mindspore/profiler/profiling.py
View File

@ -152,11 +152,10 @@ class Profiler:
# parse hwts.log.data.45.dev file, and get task profiling data
hwts_output_filename = self._hwts_output_filename_target + self._dev_id + ".txt"
hwts_output_filename = os.path.join(self._output_path, hwts_output_filename)
source_path = validate_and_normalize_path(source_path)
hwts_output_filename = validate_and_normalize_path(hwts_output_filename)
hwtslog_parser = HWTSLogParser(source_path, hwts_output_filename)
result = hwtslog_parser.execute()
if not result:
logger.error("Profiling: fail to parse hwts log file.")
return
_ = hwtslog_parser.execute()
# parse Framework file, and get the relation of op and tasks
framework_parser = FrameworkParser(job_id, self._dev_id, self._output_path)
@ -169,6 +168,7 @@ class Profiler:
# get op compute time from hwts data and framework data, write output_op_compute_time.txt
opcompute_output_filename = self._opcompute_output_filename_target + self._dev_id + ".txt"
opcompute_output_filename = os.path.join(self._output_path, opcompute_output_filename)
opcompute_output_filename = validate_and_normalize_path(opcompute_output_filename)
optime_parser = OPComputeTimeParser(
hwts_output_filename, opcompute_output_filename,
op_task_dict, self._output_path, self._dev_id
@ -178,6 +178,7 @@ class Profiler:
# parse DATA_PREPROCESS.dev.AICPU file, write output_data_preprocess_aicpu_x.txt
output_data_preprocess_aicpu = self._aicpu_op_output_filename_target + self._dev_id + ".txt"
output_data_preprocess_aicpu = os.path.join(self._output_path, output_data_preprocess_aicpu)
output_data_preprocess_aicpu = validate_and_normalize_path(output_data_preprocess_aicpu)
aicpu_data_parser = DataPreProcessParser(source_path, output_data_preprocess_aicpu)
aicpu_data_parser.execute()
@ -230,10 +231,13 @@ class Profiler:
self._output_path,
'step_trace_point_info.json'
)
step_trace_intermediate_file_path = validate_and_normalize_path(step_trace_intermediate_file_path)
point_info_file_path = validate_and_normalize_path(point_info_file_path)
# whether keep the first step
skip_first_step_flag = framework_parser.check_op_name(INIT_OP_NAME)
point_info = framework_parser.point_info
# parser the step trace files and save the result to disk
source_path = validate_and_normalize_path(source_path)
parser = StepTraceParser(input_dir=source_path,
output_file_path=step_trace_intermediate_file_path,
job_id=self._job_id_env,