diff --git a/mindspore/profiler/parser/aicpu_data_parser.py b/mindspore/profiler/parser/aicpu_data_parser.py index 19b6f1426da..7cdb3571381 100644 --- a/mindspore/profiler/parser/aicpu_data_parser.py +++ b/mindspore/profiler/parser/aicpu_data_parser.py @@ -16,6 +16,7 @@ The parser for AI CPU preprocess data. """ import os +import stat from mindspore.profiler.common.util import fwrite_format, get_file_join_name from mindspore import log as logger @@ -96,7 +97,7 @@ class DataPreProcessParser: ai_cpu_str = str(ai_cpu_data.read().replace(b'\n\x00', b' ___ ') .replace(b'\x00', b' ___ '))[2:-1] ai_cpu_lines = ai_cpu_str.split(" ___ ") - + os.chmod(self._source_file_name, stat.S_IREAD | stat.S_IWRITE) result_list = list() ai_cpu_total_time_summary = 0 # Node serial number. diff --git a/mindspore/profiler/parser/framework_parser.py b/mindspore/profiler/parser/framework_parser.py index 4215d441e45..949a91056e6 100644 --- a/mindspore/profiler/parser/framework_parser.py +++ b/mindspore/profiler/parser/framework_parser.py @@ -18,6 +18,7 @@ import enum import json import os import re +import stat from mindspore.profiler.common.exceptions.exceptions import \ ProfilerPathErrorException, ProfilerDirNotFoundException, \ @@ -447,6 +448,7 @@ class FrameworkParser: def _parse_task_files(self): """Parse the framework task files.""" for path in self._framework_path['task']: + path = validate_and_normalize_path(path) with open(path, 'r') as file: for task_info in file: infos = task_info.strip('\n').split(' ') @@ -490,6 +492,7 @@ class FrameworkParser: value.append(key) value.extend(none_list) csv_writer.writerow(value) + os.chmod(self._save_path, stat.S_IREAD | stat.S_IWRITE) def _parse_one_row_graph_info(self, row_info): """ @@ -591,6 +594,7 @@ class FrameworkParser: def _parse_point_files(self): """Parse the framework point files.""" for path in self._framework_path['point']: + path = validate_and_normalize_path(path) with open(path, 'r') as file: for point_info in file: infos = point_info.strip('\n').split(' ') diff --git a/mindspore/profiler/parser/hwts_log_parser.py b/mindspore/profiler/parser/hwts_log_parser.py index af882fc8350..cbcca7e5456 100644 --- a/mindspore/profiler/parser/hwts_log_parser.py +++ b/mindspore/profiler/parser/hwts_log_parser.py @@ -17,7 +17,8 @@ import os import struct from mindspore.profiler.common.util import fwrite_format, get_file_join_name from mindspore import log as logger - +from mindspore.profiler.common.validator.validate_path import \ + validate_and_normalize_path class HWTSLogParser: """ @@ -68,8 +69,10 @@ class HWTSLogParser: result_data = "" + self._source_flie_name = validate_and_normalize_path(self._source_flie_name) with open(self._source_flie_name, 'rb') as hwts_data: while True: + # read 64 bit data line = hwts_data.read(64) if line: if not line.strip(): @@ -77,6 +80,8 @@ class HWTSLogParser: else: break byte_first_four = struct.unpack('BBHHH', line[0:8]) + # byte_first[0:4] refers to count. byte_first[4] refers to is_warn_res0_0v. + # byte_first[5:8] refers to the type of ms. byte_first = bin(byte_first_four[0]).replace('0b', '').zfill(8) ms_type = byte_first[-3:] is_warn_res0_ov = byte_first[4] diff --git a/mindspore/profiler/parser/integrator.py b/mindspore/profiler/parser/integrator.py index 584ba3181a6..a7c29cf6121 100644 --- a/mindspore/profiler/parser/integrator.py +++ b/mindspore/profiler/parser/integrator.py @@ -16,6 +16,7 @@ import csv import json import os +import stat from decimal import Decimal from mindspore import log as logger @@ -91,6 +92,7 @@ class Integrator: self._profiling_dir, self._file_name_framework.format(self._device_id) ) + framework_file = validate_and_normalize_path(framework_file) if not os.path.isfile(framework_file): return @@ -130,6 +132,7 @@ class Integrator: self._profiling_dir, self._file_name_aicore_detail_time.format(self._device_id) ) + aicore_detail_file = validate_and_normalize_path(aicore_detail_file) if not os.path.isfile(aicore_detail_file): return @@ -169,6 +172,7 @@ class Integrator: self._profiling_dir, self._file_name_aicpu_time.format(self._device_id) ) + aicpu_file = validate_and_normalize_path(aicpu_file) if not os.path.isfile(aicpu_file): return @@ -197,6 +201,7 @@ class Integrator: self._profiling_dir, self._file_name_aicore_type_time.format(self._device_id) ) + op_type_file_path = validate_and_normalize_path(op_type_file_path) if not os.path.isfile(op_type_file_path): logger.warning('The file <%s> does not exist.', op_type_file_path) return @@ -217,6 +222,8 @@ class Integrator: self._profiling_dir, self._file_name_framework.format(self._device_id) ) + op_detail_file_path = validate_and_normalize_path(op_detail_file_path) + framework_file_path = validate_and_normalize_path(framework_file_path) if not os.path.isfile(op_detail_file_path): logger.warning('The file <%s> does not exist.', op_detail_file_path) return @@ -251,6 +258,7 @@ class Integrator: if not file_path: logger.error("Failed to find parsed trace time file.") raise ProfilerFileNotFoundException('parsed step trace time file') + file_path = validate_and_normalize_path(file_path) with open(file_path, 'r') as handle: csv_reader = csv.reader(handle) self.__column__ = next(csv_reader) @@ -261,6 +269,7 @@ class Integrator: def _load_point_info(self): """Load point info.""" file_path = os.path.join(self._profiling_dir, 'step_trace_point_info.json') + file_path = validate_and_normalize_path(file_path) if os.path.isfile(file_path): with open(file_path, 'r', encoding='utf-8') as file: try: @@ -540,6 +549,7 @@ class BaseTimelineGenerator: break json_file.write(',') json_file.write(']') + os.chmod(display_file_path, stat.S_IREAD | stat.S_IWRITE) except (IOError, OSError) as err: logger.error('Error occurred when write timeline display file: %s', err) raise ProfilerIOException @@ -556,6 +566,7 @@ class BaseTimelineGenerator: try: with open(timeline_summary_file_path, 'w') as json_file: json.dump(self._timeline_summary, json_file) + os.chmod(timeline_summary_file_path, stat.S_IREAD | stat.S_IWRITE) except (IOError, OSError) as err: logger.error('Error occurred when write timeline summary file: %s', err) raise ProfilerIOException diff --git a/mindspore/profiler/parser/minddata_parser.py b/mindspore/profiler/parser/minddata_parser.py index 583126dc732..a200acc9bcf 100644 --- a/mindspore/profiler/parser/minddata_parser.py +++ b/mindspore/profiler/parser/minddata_parser.py @@ -17,7 +17,8 @@ import os from mindspore.profiler.common.util import get_file_join_name, fwrite_format from mindspore import log as logger - +from mindspore.profiler.common.validator.validate_path import \ + validate_and_normalize_path class MinddataParser: """Minddata Aicpu Parser.""" @@ -34,6 +35,7 @@ class MinddataParser: """ result = list() try: + minddata_aicpu_source_path = validate_and_normalize_path(minddata_aicpu_source_path) with open(minddata_aicpu_source_path) as source_data_file: source_data = source_data_file.read() step_data = source_data.split("\x00") @@ -77,6 +79,7 @@ class MinddataParser: device_id (str): the device id. """ col_names = ["node_name", "start_time", "end_time", "queue_size"] + source_path = validate_and_normalize_path(source_path) minddata_aicpu_source_path = get_file_join_name( input_path=source_path, file_name='DATA_PREPROCESS.AICPUMI') if not minddata_aicpu_source_path: diff --git a/mindspore/profiler/parser/minddata_pipeline_parser.py b/mindspore/profiler/parser/minddata_pipeline_parser.py index ea0c9ae366d..390a62cdb71 100644 --- a/mindspore/profiler/parser/minddata_pipeline_parser.py +++ b/mindspore/profiler/parser/minddata_pipeline_parser.py @@ -16,6 +16,7 @@ import csv import json import os +import stat from queue import Queue from mindspore.profiler.common.exceptions.exceptions import \ @@ -167,6 +168,7 @@ class MinddataPipelineParser: self._parse_and_save_op_info( csv_writer, op_id_info_cache, sample_interval ) + os.chmod(self._save_path, stat.S_IREAD | stat.S_IWRITE) def _parse_and_save_op_info(self, csv_writer, op_id_info_cache, sample_interval): diff --git a/mindspore/profiler/parser/optime_parser.py b/mindspore/profiler/parser/optime_parser.py index 842376fcf38..2725d7cc154 100644 --- a/mindspore/profiler/parser/optime_parser.py +++ b/mindspore/profiler/parser/optime_parser.py @@ -14,6 +14,7 @@ # ============================================================================ """Op compute time files parser.""" import os +import stat from mindspore.profiler.common.util import fwrite_format from mindspore.profiler.common.exceptions.exceptions import ProfilerFileNotFoundException, \ ProfilerIOException @@ -163,6 +164,7 @@ class OPComputeTimeParser: for timeline in timeline_data: timeline = [str(item) for item in timeline] f_obj.write(','.join(timeline) + '\n') + os.chmod(file_path, stat.S_IREAD | stat.S_IWRITE) except (IOError, OSError) as err: logger.error('Error occurred when writing intermediate timeline file: %s', err) raise ProfilerIOException diff --git a/mindspore/profiler/parser/step_trace_parser.py b/mindspore/profiler/parser/step_trace_parser.py index b39820d4bcd..2bb35ae274c 100644 --- a/mindspore/profiler/parser/step_trace_parser.py +++ b/mindspore/profiler/parser/step_trace_parser.py @@ -25,6 +25,8 @@ from mindspore.profiler.common.exceptions.exceptions import ProfilerPathErrorExc JobIdMismatchException, ProfilerIOException from mindspore import log from mindspore.profiler.common.util import get_summary_for_step_trace +from mindspore.profiler.common.validator.validate_path import \ + validate_and_normalize_path StepTraceStruct = namedtuple( 'TrainingTraceStruct', ['tag_id', 'task_id', 'stream_id', 'sys_count'] @@ -199,6 +201,7 @@ class StepTraceParser: log.info("Start to parse step trace file.") event_info = {} for source_file in source_files: + source_file = validate_and_normalize_path(source_file) with open(source_file, 'rb') as handler: content = handler.read() for step_trace in self._get_next_step_trace(content, event_info): diff --git a/mindspore/profiler/profiling.py b/mindspore/profiler/profiling.py index 3d3e1b6c679..be23abc31d7 100644 --- a/mindspore/profiler/profiling.py +++ b/mindspore/profiler/profiling.py @@ -152,11 +152,10 @@ class Profiler: # parse hwts.log.data.45.dev file, and get task profiling data hwts_output_filename = self._hwts_output_filename_target + self._dev_id + ".txt" hwts_output_filename = os.path.join(self._output_path, hwts_output_filename) + source_path = validate_and_normalize_path(source_path) + hwts_output_filename = validate_and_normalize_path(hwts_output_filename) hwtslog_parser = HWTSLogParser(source_path, hwts_output_filename) - result = hwtslog_parser.execute() - if not result: - logger.error("Profiling: fail to parse hwts log file.") - return + _ = hwtslog_parser.execute() # parse Framework file, and get the relation of op and tasks framework_parser = FrameworkParser(job_id, self._dev_id, self._output_path) @@ -169,6 +168,7 @@ class Profiler: # get op compute time from hwts data and framework data, write output_op_compute_time.txt opcompute_output_filename = self._opcompute_output_filename_target + self._dev_id + ".txt" opcompute_output_filename = os.path.join(self._output_path, opcompute_output_filename) + opcompute_output_filename = validate_and_normalize_path(opcompute_output_filename) optime_parser = OPComputeTimeParser( hwts_output_filename, opcompute_output_filename, op_task_dict, self._output_path, self._dev_id @@ -178,6 +178,7 @@ class Profiler: # parse DATA_PREPROCESS.dev.AICPU file, write output_data_preprocess_aicpu_x.txt output_data_preprocess_aicpu = self._aicpu_op_output_filename_target + self._dev_id + ".txt" output_data_preprocess_aicpu = os.path.join(self._output_path, output_data_preprocess_aicpu) + output_data_preprocess_aicpu = validate_and_normalize_path(output_data_preprocess_aicpu) aicpu_data_parser = DataPreProcessParser(source_path, output_data_preprocess_aicpu) aicpu_data_parser.execute() @@ -230,10 +231,13 @@ class Profiler: self._output_path, 'step_trace_point_info.json' ) + step_trace_intermediate_file_path = validate_and_normalize_path(step_trace_intermediate_file_path) + point_info_file_path = validate_and_normalize_path(point_info_file_path) # whether keep the first step skip_first_step_flag = framework_parser.check_op_name(INIT_OP_NAME) point_info = framework_parser.point_info # parser the step trace files and save the result to disk + source_path = validate_and_normalize_path(source_path) parser = StepTraceParser(input_dir=source_path, output_file_path=step_trace_intermediate_file_path, job_id=self._job_id_env,