diff --git a/tests/ut/data/profiler_data/JOB1/Framework.host.vm.graph_desc_info.0.slice_0 b/tests/ut/data/profiler_data/JOB1/Framework.host.vm.graph_desc_info.0.slice_0 new file mode 100644 index 0000000000..9b3e7b322c --- /dev/null +++ b/tests/ut/data/profiler_data/JOB1/Framework.host.vm.graph_desc_info.0.slice_0 @@ -0,0 +1,4 @@ +op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224" +op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16" +op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16" +op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16" diff --git a/tests/ut/data/profiler_data/JOB1/Framework.host.vm.graph_desc_info.0.slice_0.done b/tests/ut/data/profiler_data/JOB1/Framework.host.vm.graph_desc_info.0.slice_0.done new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/ut/data/profiler_data/JOB1/Framework.host.vm.task_desc_info.0.slice_0 b/tests/ut/data/profiler_data/JOB1/Framework.host.vm.task_desc_info.0.slice_0 new file mode 100644 index 0000000000..e786b35e22 --- /dev/null +++ b/tests/ut/data/profiler_data/JOB1/Framework.host.vm.task_desc_info.0.slice_0 @@ -0,0 +1,4 @@ +Default/Cast-op6 32 51517 0 +Default/TransData-op7 32 51518 0 +Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 51519 0 +Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 51522 0 diff --git a/tests/ut/data/profiler_data/JOB1/pipeline_profiling_0.json b/tests/ut/data/profiler_data/JOB1/pipeline_profiling_0.json new file mode 100644 index 0000000000..2f5fdca724 --- /dev/null +++ b/tests/ut/data/profiler_data/JOB1/pipeline_profiling_0.json @@ -0,0 +1,55 @@ +{ + "sampling_interval": 10, + "op_info": [ + { + "op_id": 4, + "op_type": "TFReader", + "num_workers": 4, + "metrics": null, + "children": [3] + }, + { + "op_id": 3, + "op_type": "TFReader", + "num_workers": 4, + "metrics": { + "output_queue": { + "size": [10, 20, 30], + "length": 64 + } + }, + "children": null + }, + { + "op_id": 2, + "op_type": "TFReader", + "num_workers": 4, + "metrics": { + "output_queue": { + "size": [10, 20, 30], + "length": 64 + } + }, + "children": null + }, + { + "op_id": 1, + "op_type": "Shuffle", + "num_workers": 1, + "metrics": { + "output_queue": { + "size": [10, 20, 30], + "length": 64 + } + }, + "children": [2, 4] + }, + { + "op_id": 0, + "op_type": "Batch", + "num_workers": 4, + "metrics": null, + "children": [1] + } + ] +} \ No newline at end of file diff --git a/tests/ut/data/profiler_data/JOB1/training_trace.46.dev.profiler_default_tag.0.slice_0 b/tests/ut/data/profiler_data/JOB1/training_trace.46.dev.profiler_default_tag.0.slice_0 new file mode 100644 index 0000000000..2fd8ba0bc9 Binary files /dev/null and b/tests/ut/data/profiler_data/JOB1/training_trace.46.dev.profiler_default_tag.0.slice_0 differ diff --git a/tests/ut/data/profiler_data/JOB2/pipeline_profiling_0.json b/tests/ut/data/profiler_data/JOB2/pipeline_profiling_0.json new file mode 100644 index 0000000000..1357a35c83 --- /dev/null +++ b/tests/ut/data/profiler_data/JOB2/pipeline_profiling_0.json @@ -0,0 +1,48 @@ +{ + "sampling_interval": 10, + "op_info": [ + { + "op_id": 3, + "op_type": "TFReader", + "num_workers": 4, + "metrics": { + "output_queue": { + "size": [10, 20, 30], + "length": 64 + } + }, + "children": null + }, + { + "op_id": 2, + "op_type": "TFReader", + "num_workers": 4, + "metrics": { + "output_queue": { + "size": [10, 20, 30], + "length": 64 + } + }, + "children": null + }, + { + "op_id": 1, + "op_type": "Shuffle", + "num_workers": 1, + "metrics": { + "output_queue": { + "size": [10, 20, 30], + "length": 64 + } + }, + "children": [2, 3] + }, + { + "op_id": 0, + "op_type": "Batch", + "num_workers": 4, + "metrics": null, + "children": [1] + } + ] +} \ No newline at end of file diff --git a/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.graph_desc_info.0.slice_0 b/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.graph_desc_info.0.slice_0 new file mode 100644 index 0000000000..9b3e7b322c --- /dev/null +++ b/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.graph_desc_info.0.slice_0 @@ -0,0 +1,4 @@ +op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224" +op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16" +op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16" +op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16" diff --git a/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.point.0.slice_0 b/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.point.0.slice_0 new file mode 100644 index 0000000000..01bcf6f3ca --- /dev/null +++ b/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.point.0.slice_0 @@ -0,0 +1,2 @@ +1 Default/Cast-op6 +2 Default/TransData-op7 diff --git a/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.task_desc_info.0.slice_0 b/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.task_desc_info.0.slice_0 new file mode 100644 index 0000000000..e49673789f --- /dev/null +++ b/tests/ut/data/profiler_data/JOB4/data/Framework.host.vm.task_desc_info.0.slice_0 @@ -0,0 +1,4 @@ +Default/Cast-op6 32 1 0 +Default/TransData-op7 32 2 0 +Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 3 0 +Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 4 0 diff --git a/tests/ut/data/profiler_data/JOB_AICPU/data/DATA_PREPROCESS.dev.AICPU.0.slice_0 b/tests/ut/data/profiler_data/JOB_AICPU/data/DATA_PREPROCESS.dev.AICPU.0.slice_0 new file mode 100644 index 0000000000..30fe89d6e9 Binary files /dev/null and b/tests/ut/data/profiler_data/JOB_AICPU/data/DATA_PREPROCESS.dev.AICPU.0.slice_0 differ diff --git a/tests/ut/data/profiler_data/JOB_AICPU/expect/output_data_preprocess_aicpu_0.txt b/tests/ut/data/profiler_data/JOB_AICPU/expect/output_data_preprocess_aicpu_0.txt new file mode 100644 index 0000000000..fe83966d02 --- /dev/null +++ b/tests/ut/data/profiler_data/JOB_AICPU/expect/output_data_preprocess_aicpu_0.txt @@ -0,0 +1,5 @@ +serial_number node_type_name total_time(ms) dispatch_time(ms) run_start run_end +1 InitData 1.567 0.1 2298200409 2298200538 +2 GetNext 0.989 0.087 2302769932 2302769980 +3 TruncatedNormal 1.566 0.105 4098200409 4098200538 +AI CPU Total Time(ms): 4.122000 diff --git a/tests/ut/data/profiler_data/container/0/data/Framework.host.vm.graph_desc_info.0.JOB2.slice_0 b/tests/ut/data/profiler_data/container/0/data/Framework.host.vm.graph_desc_info.0.JOB2.slice_0 new file mode 100644 index 0000000000..9b3e7b322c --- /dev/null +++ b/tests/ut/data/profiler_data/container/0/data/Framework.host.vm.graph_desc_info.0.JOB2.slice_0 @@ -0,0 +1,4 @@ +op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224" +op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16" +op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16" +op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16" diff --git a/tests/ut/data/profiler_data/container/0/data/Framework.host.vm.task_desc_info.0.JOB2.slice_0 b/tests/ut/data/profiler_data/container/0/data/Framework.host.vm.task_desc_info.0.JOB2.slice_0 new file mode 100644 index 0000000000..e786b35e22 --- /dev/null +++ b/tests/ut/data/profiler_data/container/0/data/Framework.host.vm.task_desc_info.0.JOB2.slice_0 @@ -0,0 +1,4 @@ +Default/Cast-op6 32 51517 0 +Default/TransData-op7 32 51518 0 +Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 51519 0 +Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 51522 0 diff --git a/tests/ut/data/profiler_data/profiler/aicore_intermediate_1_detail.csv b/tests/ut/data/profiler_data/profiler/aicore_intermediate_1_detail.csv new file mode 100755 index 0000000000..ec117c5346 --- /dev/null +++ b/tests/ut/data/profiler_data/profiler/aicore_intermediate_1_detail.csv @@ -0,0 +1,11 @@ +full_op_time,execution_time +Default/AtomicAddrClean-op104,0.00133 +Default/AtomicAddrClean-op105,0.000987 +Default/AtomicAddrClean-op106,0.001129 +Default/Cast-op10,0.00466 +Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op12,0.002366 +Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/gradConv2D/Cast-op53,0.004879 +Default/TransData-op11,0.006366 +Gradients/Default/network-WithLossCell/_backbone-LeNet5/gradReshape/TransData-op44,0.006782 +Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Conv2D-op13,0.05651 +Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/MatMul-op9,0.370864 diff --git a/tests/ut/data/profiler_data/profiler/aicore_intermediate_1_type.csv b/tests/ut/data/profiler_data/profiler/aicore_intermediate_1_type.csv new file mode 100755 index 0000000000..56bf368a6c --- /dev/null +++ b/tests/ut/data/profiler_data/profiler/aicore_intermediate_1_type.csv @@ -0,0 +1,6 @@ +op_type,execution_time,execution_frequency,percent +AtomicAddrClean,0.007283,6,0.49 +Cast,0.053395,13,3.63 +TransData,0.121800,5,8.23 +Conv2D,0.063656,2,4.33 +MatMul,1.085982,9,73.80 diff --git a/tests/ut/data/profiler_data/profiler/framework_raw_0.csv b/tests/ut/data/profiler_data/profiler/framework_raw_0.csv new file mode 100755 index 0000000000..762bc69346 --- /dev/null +++ b/tests/ut/data/profiler_data/profiler/framework_raw_0.csv @@ -0,0 +1,5 @@ +task_id,stream_id,block_dim,full_op_name,op_name,op_type,subgraph,op_info +51517,0,32,Default/Cast-op6,Cast-op6,Cast,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,3,224,224""}, ""output_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,3,224,224""}}" +51518,0,32,Default/TransData-op7,TransData-op7,TransData,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,3,224,224""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,224,224,16""}}" +51519,0,32,Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5,Cast-op5,Cast,Default,"{""input_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""49,4,16,16""}, ""output_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""49,4,16,16""}}" +51522,0,4,Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28,Cast-op28,Cast,Default,"{""input_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""4,4,16,16""}, ""output_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""4,4,16,16""}}" diff --git a/tests/ut/data/profiler_data/profiler/framework_raw_1.csv b/tests/ut/data/profiler_data/profiler/framework_raw_1.csv new file mode 100755 index 0000000000..5b60ecef49 --- /dev/null +++ b/tests/ut/data/profiler_data/profiler/framework_raw_1.csv @@ -0,0 +1,11 @@ +task_id,stream_id,block_dim,full_op_name,op_name,op_type,subgraph,op_info +30290,0,1,Default/AtomicAddrClean-op104,AtomicAddrClean-op104,AtomicAddrClean,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": """"}}" +30295,0,1,Default/AtomicAddrClean-op105,AtomicAddrClean-op105,AtomicAddrClean,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""10""}}" +30300,0,1,Default/AtomicAddrClean-op106,AtomicAddrClean-op106,AtomicAddrClean,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""84""}}" +30268,0,32,Default/Cast-op10,Cast-op10,Cast,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,1,32,32""}, ""output_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32""}}" +30271,0,9,Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op12,Cast-op12,Cast,Default,"{""input_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""25,1,16,16""}, ""output_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""25,1,16,16""}}" +30320,0,32,Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/gradConv2D/Cast-op53,Cast-op53,Cast,Gradients,"{""input_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,1,28,28,16""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,28,28,16""}}" +30269,0,32,Default/TransData-op11,TransData-op11,TransData,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32""}}" +30308,0,32,Gradients/Default/network-WithLossCell/_backbone-LeNet5/gradReshape/TransData-op44,TransData-op44,TransData,Gradients,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,16,5,5""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,5,5,16""}}" +30272,0,32,Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Conv2D-op13,Conv2D-op13,Conv2D,Default,"{""input_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32,16""}, ""input_1"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""25,1,16,16""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,28,28,16""}}" +30286,0,1,Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/MatMul-op9,MatMul-op9,MatMul,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,120""}, ""input_1"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""84,120""}, ""input_2"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""84""}, ""output_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,84""}}" diff --git a/tests/ut/data/profiler_data/profiler/minddata_pipeline_raw_0.csv b/tests/ut/data/profiler_data/profiler/minddata_pipeline_raw_0.csv new file mode 100644 index 0000000000..57f8ddaf68 --- /dev/null +++ b/tests/ut/data/profiler_data/profiler/minddata_pipeline_raw_0.csv @@ -0,0 +1,5 @@ +op_id,op_type,num_workers,output_queue_size,output_queue_average_size,output_queue_length,output_queue_usage_rate,sample_interval,parent_id,children_id +0,Batch,4,,,,,10,,[1] +1,Shuffle,1,"[10, 20, 30]",20.0,64,0.3125,10,0,"[2, 3]" +2,TFReader,4,"[10, 20, 30]",20.0,64,0.3125,10,1, +3,TFReader,4,"[10, 20, 30]",20.0,64,0.3125,10,1, diff --git a/tests/ut/data/profiler_data/profiler/pipeline_profiling_1.json b/tests/ut/data/profiler_data/profiler/pipeline_profiling_1.json new file mode 100644 index 0000000000..2f5fdca724 --- /dev/null +++ b/tests/ut/data/profiler_data/profiler/pipeline_profiling_1.json @@ -0,0 +1,55 @@ +{ + "sampling_interval": 10, + "op_info": [ + { + "op_id": 4, + "op_type": "TFReader", + "num_workers": 4, + "metrics": null, + "children": [3] + }, + { + "op_id": 3, + "op_type": "TFReader", + "num_workers": 4, + "metrics": { + "output_queue": { + "size": [10, 20, 30], + "length": 64 + } + }, + "children": null + }, + { + "op_id": 2, + "op_type": "TFReader", + "num_workers": 4, + "metrics": { + "output_queue": { + "size": [10, 20, 30], + "length": 64 + } + }, + "children": null + }, + { + "op_id": 1, + "op_type": "Shuffle", + "num_workers": 1, + "metrics": { + "output_queue": { + "size": [10, 20, 30], + "length": 64 + } + }, + "children": [2, 4] + }, + { + "op_id": 0, + "op_type": "Batch", + "num_workers": 4, + "metrics": null, + "children": [1] + } + ] +} \ No newline at end of file diff --git a/tests/ut/data/profiler_data/profiler/step_trace_raw_0_detail_time.csv b/tests/ut/data/profiler_data/profiler/step_trace_raw_0_detail_time.csv new file mode 100755 index 0000000000..0e923ccad8 --- /dev/null +++ b/tests/ut/data/profiler_data/profiler/step_trace_raw_0_detail_time.csv @@ -0,0 +1,22 @@ +step_num,start_point,end_point,total,fp_point,bp_point,iteration_interval,fp_and_bp,tail,stream_10_parallel_0_start_point,stream_10_parallel_0_end_point,stream_10_parallel_0,stream_10_parallel_1_start_point,stream_10_parallel_1_end_point,stream_10_parallel_1,stream_10_parallel_2_start_point,stream_10_parallel_2_end_point,stream_10_parallel_2,stream_11_parallel_0_start_point,stream_11_parallel_0_end_point,stream_11_parallel_0 +1,45000030081,45004033128,4003047,45000030081,45001733025,0,1702944,2300103,45000042679,45000060275,17596,45001048152,45001346254,298102,45002247411,45002448354,200943,45000049687,45000075987,26300 +2,45004033128,45017085658,13052530,45013070937,45014785314,9037809,1714377,2300344,45013085379,45013105429,20050,45014087119,45014385136,298017,45015297166,45015504449,207283,45013084925,45013118334,33409 +3,45017085658,45030119392,13033734,45026116231,45027818443,9030573,1702212,2300949,45026131909,45026150554,18645,45027134392,45027430418,296026,45028337093,45028537767,200674,45026129217,45026160937,31720 +4,45030119392,45043158607,13039215,45039152348,45040856975,9032956,1704627,2301632,45039169890,45039188966,19076,45040169338,45040466770,297432,45041374122,45041567754,193632,45039171681,45039193865,22184 +5,45043158607,45056198128,13039521,45052190932,45053898028,9032325,1707096,2300100,45052207675,45052222642,14967,45053204442,45053505540,301098,45054413207,45054616536,203329,45052201931,45052237599,35668 +6,45056198128,45069239564,13041436,45065233106,45066939463,9034978,1706357,2300101,45065245482,45065272534,27052,45066248423,45066546419,297996,45067455113,45067659145,204032,45065245817,45065279896,34079 +7,45069239564,45082281383,13041819,45078274997,45079980193,9035433,1705196,2301190,45078293910,45078312935,19025,45079287754,45079593841,306087,45080492957,45080691395,198438,45078292067,45078322277,30210 +8,45082281383,45095336378,13054995,45091321488,45093036084,9040105,1714596,2300294,45091338628,45091359138,20510,45092338469,45092638994,300525,45093554195,45093747470,193275,45091341356,45091369667,28311 +9,45095336378,45108372225,13035847,45104363079,45106071009,9026701,1707930,2301216,45104374524,45104400088,25564,45105378751,45105683029,304278,45106587481,45106785336,197855,45104382131,45104410852,28721 +10,45108372225,45121412413,13040188,45117401873,45119111301,9029648,1709428,2301112,45117417721,45117439668,21947,45118413083,45118718050,304967,45119629347,45119829996,200649,45117421502,45117446718,25216 +11,45121412413,45134477662,13065249,45130459598,45132175723,9047185,1716125,2301939,45130478168,45130498936,20768,45131477957,45131775220,297263,45132691645,45132893707,202062,45130470285,45130501652,31367 +12,45134477662,45147533298,13055636,45143521860,45145232553,9044198,1710693,2300745,45143533787,45143557293,23506,45144533554,45144841545,307991,45145744997,45145952255,207258,45143537383,45143563466,26083 +13,45147533298,45160588134,13054836,45156570201,45158286694,9036903,1716493,2301440,45156581069,45156609506,28437,45157581617,45157880841,299224,45158806166,45158999875,193709,45156589050,45156615664,26614 +14,45160588134,45173640064,13051930,45169625906,45171339426,9037772,1713520,2300638,45169637432,45169661754,24322,45170639482,45170940949,301467,45171853721,45172056606,202885,45169644605,45169673410,28805 +15,45173640064,45186671634,13031570,45182666696,45184371430,9026632,1704734,2300204,45182678355,45182698471,20116,45183679568,45183981082,301514,45184887156,45185083035,195879,45182680062,45182708455,28393 +16,45186671634,45199720448,13048814,45195714716,45197420410,9043082,1705694,2300038,45195728993,45195754646,25653,45196732493,45197028048,295555,45197934921,45198139237,204316,45195733069,45195764102,31033 +17,45199720448,45212762605,13042157,45208758416,45210460864,9037968,1702448,2301741,45208771010,45208790367,19357,45209773548,45210074988,301440,45210978277,45211173577,195300,45208773143,45208803280,30137 +18,45212762605,45225814601,13051996,45221801814,45223514580,9039209,1712766,2300021,45221815911,45221839644,23733,45222819211,45223114544,295333,45224031469,45224234043,202574,45221812106,45221849103,36997 +19,45225814601,45238848430,13033829,45234842015,45236548356,9027414,1706341,2300074,45234855444,45234876469,21025,45235853358,45236160825,307467,45237063061,45237260964,197903,45234857141,45234882976,25835 +20,45238848430,45251899738,13051308,45247879385,45249598280,9030955,1718895,2301458,45247896725,45247917316,20591,45248896361,45249193681,297320,45250117916,45250315651,197735,45247894228,45247926723,32495 +-,45121436513,45134482124,13045611,45130471874,45132181322,9035360,1709449,2300802,45130486422,45130508229,21808,45131486785,45131787364,300579,45132697369,45132897305,199936,45130487458,45130517315,29857 diff --git a/tests/ut/data/profiler_data/profiler/step_trace_raw_10_detail_time.csv b/tests/ut/data/profiler_data/profiler/step_trace_raw_10_detail_time.csv new file mode 100755 index 0000000000..9e97e499b2 --- /dev/null +++ b/tests/ut/data/profiler_data/profiler/step_trace_raw_10_detail_time.csv @@ -0,0 +1,42 @@ +step_num,start_point,end_point,total,fp_point,bp_point,iteration_interval,fp_and_bp,tail,stream_10_parallel_0_start_point,stream_10_parallel_0_end_point,stream_10_parallel_0,stream_10_parallel_1_start_point,stream_10_parallel_1_end_point,stream_10_parallel_1,stream_10_parallel_2_start_point,stream_10_parallel_2_end_point,stream_10_parallel_2,stream_11_parallel_0_start_point,stream_11_parallel_0_end_point,stream_11_parallel_0 +1,45000025226,45004034753,4009527,45000025226,45001734362,0,1709136,2300391,45000044023,45000060886,16863,45001043581,45001343373,299792,45002254048,45002452830,198782,45000043807,45000065736,21929 +2,45004034753,45017091420,13056667,45013073790,45014789509,9039037,1715719,2301911,45013085205,45013104210,19005,45014086339,45014393261,306922,45015299546,45015501808,202262,45013085040,45013119810,34770 +3,45017091420,45030144372,13052952,45026123867,45027843651,9032447,1719784,2300721,45026138546,45026154524,15978,45027135742,45027437486,301744,45028363120,45028560901,197781,45026136046,45026171363,35317 +4,45030144372,45043184486,13040114,45039173149,45040883087,9028777,1709938,2301399,45039190927,45039209948,19021,45040185915,45040484897,298982,45041399754,45041594775,195021,45039192768,45039221423,28655 +5,45043184486,45056241064,13056578,45052223555,45053940709,9039069,1717154,2300355,45052241736,45052262186,20450,45053239605,45053540866,301261,45054452604,45054654505,201901,45052233932,45052265774,31842 +6,45056241064,45069291346,13050282,45065278144,45066991121,9037080,1712977,2300225,45065293660,45065316136,22476,45066289480,45066589910,300430,45067511002,45067701731,190729,45065293679,45065321296,27617 +7,45069291346,45082344927,13053581,45078335376,45080043268,9044030,1707892,2301659,45078353164,45078365382,12218,45079354748,45079648384,293636,45080557453,45080760374,202921,45078353030,45078384530,31500 +8,45082344927,45095382554,13037627,45091368697,45093080797,9023770,1712100,2301757,45091381244,45091405208,23964,45092382630,45092684285,301655,45093590961,45093796698,205737,45091381199,45091413840,32641 +9,45095382554,45108433947,13051393,45104419947,45106132133,9037393,1712186,2301814,45104432587,45104457476,24889,45105431458,45105735476,304018,45106651213,45106845305,194092,45104435207,45104466677,31470 +10,45108433947,45121486591,13052644,45117469353,45119185969,9035406,1716616,2300622,45117483627,45117504869,21242,45118483411,45118788540,305129,45119696660,45119898575,201915,45117485587,45117510985,25398 +11,45121486591,45134546571,13059980,45130528618,45132244809,9042027,1716191,2301762,45130539730,45130561122,21392,45131538695,45131846715,308020,45132759789,45132960848,201059,45130545378,45130569412,24034 +12,45134546571,45147608222,13061651,45143597023,45145307273,9050452,1710250,2300949,45143615771,45143631460,15689,45144610592,45144910736,300144,45145818642,45146024326,205684,45143613528,45143640223,26695 +13,45147608222,45160663790,13055568,45156648696,45158362923,9040474,1714227,2300867,45156663193,45156685466,22273,45157661576,45157963074,301498,45158881212,45159074431,193219,45156667038,45156694912,27874 +14,45160663790,45173707626,13043836,45169694535,45171407246,9030745,1712711,2300380,45169710667,45169727936,17269,45170705802,45171013806,308004,45171924100,45172120273,196173,45169708524,45169739038,30514 +15,45173707626,45186754860,13047234,45182750254,45184454036,9042628,1703782,2300824,45182765445,45182789799,24354,45183761335,45184065169,303834,45184973312,45185170444,197132,45182769451,45182799598,30147 +16,45186754860,45199798718,13043858,45195792271,45197497908,9037411,1705637,2300810,45195804771,45195827915,23144,45196804016,45197108243,304227,45198013357,45198209858,196501,45195806656,45195841674,35018 +17,45199798718,45212854993,13056275,45208834355,45210553378,9035637,1719023,2301615,45208850179,45208865588,15409,45209851018,45210151436,300418,45211073169,45211271792,198623,45208847052,45208876998,29946 +18,45212854993,45225893712,13038719,45221888939,45223593704,9033946,1704765,2300008,45221901732,45221924983,23251,45222908795,45223203590,294795,45224105803,45224313354,207551,45221899792,45221938802,39010 +19,45225893712,45238941242,13047530,45234926295,45236640454,9032583,1714159,2300788,45234938628,45234957237,18609,45235942710,45236239983,297273,45237159532,45237356140,196608,45234938330,45234976170,37840 +20,45238941242,45251979177,13037935,45247977674,45249678116,9036432,1700442,2301061,45247990919,45248013476,22557,45248991451,45249294742,303291,45250195733,45250395760,200027,45247988950,45248024969,36019 +21,45251979177,45265018752,13039575,45261005416,45262718472,9026239,1713056,2300280,0,0,0,0,0,0,0,0,0,0,0,0 +22,45265018752,45278062782,13044030,45274047185,45275762095,9028433,1714910,2300687,0,0,0,0,0,0,0,0,0,0,0,0 +23,45278062782,45291105708,13042926,45287094000,45288805223,9031218,1711223,2300485,0,0,0,0,0,0,0,0,0,0,0,0 +24,45291105708,45304155918,13050210,45300150844,45301854040,9045136,1703196,2301878,0,0,0,0,0,0,0,0,0,0,0,0 +25,45304155918,45317206695,13050777,45313191948,45314905714,9036030,1713766,2300981,0,0,0,0,0,0,0,0,0,0,0,0 +26,45317206695,45330265105,13058410,45326256021,45327964581,9049326,1708560,2300524,0,0,0,0,0,0,0,0,0,0,0,0 +27,45330265105,45343324012,13058907,45339305124,45341023739,9040019,1718615,2300273,0,0,0,0,0,0,0,0,0,0,0,0 +28,45343324012,45356374571,13050559,45352366211,45354073401,9042199,1707190,2301170,0,0,0,0,0,0,0,0,0,0,0,0 +29,45356374571,45369429514,13054943,45365417827,45367128283,9043256,1710456,2301231,0,0,0,0,0,0,0,0,0,0,0,0 +30,45369429514,45382479199,13049685,45378476397,45380177297,9046883,1700900,2301902,0,0,0,0,0,0,0,0,0,0,0,0 +31,45382479199,45395530376,13051177,45391510137,45393229377,9030938,1719240,2300999,0,0,0,0,0,0,0,0,0,0,0,0 +32,45395530376,45408571765,13041389,45404559082,45406270720,9028706,1711638,2301045,0,0,0,0,0,0,0,0,0,0,0,0 +33,45408571765,45421635175,13063410,45417619223,45419334221,9047458,1714998,2300954,0,0,0,0,0,0,0,0,0,0,0,0 +34,45421635175,45434672219,13037044,45430669445,45432371312,9034270,1701867,2300907,0,0,0,0,0,0,0,0,0,0,0,0 +35,45434672219,45447714036,13041817,45443704548,45445413852,9032329,1709304,2300184,0,0,0,0,0,0,0,0,0,0,0,0 +36,45447714036,45460765153,13051117,45456753675,45458463701,9039639,1710026,2301452,0,0,0,0,0,0,0,0,0,0,0,0 +37,45460765153,45473829105,13063952,45469808281,45471527400,9043128,1719119,2301705,0,0,0,0,0,0,0,0,0,0,0,0 +38,45473829105,45486884190,13055085,45482867237,45484583534,9038132,1716297,2300656,0,0,0,0,0,0,0,0,0,0,0,0 +39,45486884190,45499928571,13044381,45495917628,45497627921,9033438,1710293,2300650,0,0,0,0,0,0,0,0,0,0,0,0 +40,45499928571,45512973815,13045244,45508968990,45510673699,9040419,1704709,2300116,0,0,0,0,0,0,0,0,0,0,0,0 +-,45251983006,45265032725,13049720,45261020353,45262731761,9037347,1711408,2300964,21986676455,21986686280,9825,21987163213,21987310272,147058,21987754537,21987851587,97050,21986676441,21986691731,15290 diff --git a/tests/ut/python/profiler/__init__.py b/tests/ut/python/profiler/__init__.py new file mode 100644 index 0000000000..586589132c --- /dev/null +++ b/tests/ut/python/profiler/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Unit test for profiler.""" +import os + +RAW_DATA_BASE = os.path.realpath(os.path.join(os.path.dirname(__file__), '../../data/profiler_data')) +RAW_DATA = os.path.realpath(os.path.join(RAW_DATA_BASE, 'JOB1')) +RAW_DATA_JOB2 = os.path.realpath(os.path.join(RAW_DATA_BASE, 'JOB2')) +PROFILER_DIR = os.path.realpath(os.path.join(RAW_DATA_BASE, 'profiler')) diff --git a/tests/ut/python/profiler/parser/__init__.py b/tests/ut/python/profiler/parser/__init__.py new file mode 100644 index 0000000000..e30774307c --- /dev/null +++ b/tests/ut/python/profiler/parser/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ diff --git a/tests/ut/python/profiler/parser/test_aicpu_parser.py b/tests/ut/python/profiler/parser/test_aicpu_parser.py new file mode 100644 index 0000000000..47e3bc13aa --- /dev/null +++ b/tests/ut/python/profiler/parser/test_aicpu_parser.py @@ -0,0 +1,74 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test the aicpu parser.""" +import os +import tempfile +import shutil + +from unittest import TestCase + +from mindspore.profiler.parser.aicpu_data_parser import DataPreProcessParser + + +def get_result(file_path): + """ + Get result from the aicpu file. + + Args: + file_path (str): The aicpu file path. + + Returns: + list[list], the parsed aicpu information. + """ + result = [] + try: + file = open(file_path, 'r') + result.append(file.read()) + return result + finally: + if file: + file.close() + + +class TestAicpuParser(TestCase): + """Test the class of Aicpu Parser.""" + + def setUp(self) -> None: + """Initialization before test case execution.""" + self.profiling_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), + '../../../data/profiler_data/' + 'JOB_AICPU/data')) + self.expect_dir = os.path.realpath(os.path.join(os.path.dirname(__file__), + '../../../data/profiler_data/' + 'JOB_AICPU/expect')) + self.output_path = tempfile.mkdtemp(prefix='output_data_preprocess_aicpu_') + self.output_file = os.path.join(self.output_path, 'output_data_preprocess_aicpu_0.txt') + self.expect_file = os.path.join(self.expect_dir, 'output_data_preprocess_aicpu_0.txt') + + def test_aicpu_parser(self): + """Test the class of Aicpu Parser.""" + data = DataPreProcessParser(self.profiling_dir, self.output_file) + data.execute() + expect_result = get_result(self.expect_file) + result = get_result(self.output_file) + shutil.rmtree(self.output_path) + assert expect_result == result + + def test_aicpu_parser_file_not_exist(self): + """Test the class of Aicpu Parser.""" + profiling_dir = os.path.realpath(os.path.join(self.profiling_dir, 'data')) + data = DataPreProcessParser(profiling_dir, self.output_file) + data.execute() + shutil.rmtree(self.output_path) diff --git a/tests/ut/python/profiler/parser/test_framework_parser.py b/tests/ut/python/profiler/parser/test_framework_parser.py new file mode 100644 index 0000000000..d37bd19dd9 --- /dev/null +++ b/tests/ut/python/profiler/parser/test_framework_parser.py @@ -0,0 +1,128 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test the framework parser module.""" +import csv +import os +import shutil +import tempfile +from unittest import mock + +import pytest + +from mindspore.profiler.common.exceptions.exceptions import \ + ProfilerFileNotFoundException +from mindspore.profiler.parser.framework_parser import FrameworkParser +from tests.ut.python.profiler import PROFILER_DIR, RAW_DATA_BASE + + +def get_framework_result(file_path): + """ + Get framework result from the framework file. + + Args: + file_path (str): The framework file path. + + Returns: + list[list], the parsed framework information. + """ + result = [] + with open(file_path, 'r') as file: + csv_reader = csv.reader(file) + for row in csv_reader: + result.append(row) + return result + + +class TestFrameworkParser: + """Test the class of `FrameworkParser`.""" + def setup_method(self): + """Initialization before test case execution.""" + with mock.patch.object(FrameworkParser, '_raw_data_dir', RAW_DATA_BASE): + self._output_path_1 = tempfile.mkdtemp(prefix='test_framework_parser_') + self._parser_1 = FrameworkParser('JOB1', '0', self._output_path_1) + self._output_path_2 = tempfile.mkdtemp(prefix='test_framework_parser_') + self._parser_2 = FrameworkParser('JOB2', '0', self._output_path_2) + self._output_path_4 = tempfile.mkdtemp(prefix='test_framework_parser_') + self._parser_4 = FrameworkParser('JOB4', '0', self._output_path_4) + + def teardown_method(self) -> None: + """Clear up after test case execution.""" + shutil.rmtree(self._output_path_1) + shutil.rmtree(self._output_path_2) + shutil.rmtree(self._output_path_4) + + def test_save_path(self): + """Test the querying save path function.""" + expect_result = os.path.join(self._output_path_1, 'framework_raw_0.csv') + assert expect_result == self._parser_1.save_path + + expect_result = os.path.join(self._output_path_2, 'framework_raw_0.csv') + assert expect_result == self._parser_2.save_path + + def test_point_info(self): + """Test the querying point info function.""" + expect_result = { + 1: 'Default/Cast-op6', + 2: 'Default/TransData-op7' + } + assert expect_result == self._parser_4.point_info + + def test_to_task_id_full_op_name_dict(self): + """Test the querying task id and full operator name dict function.""" + expect_result = { + '51517': 'Default/Cast-op6', + '51518': 'Default/TransData-op7', + '51519': 'Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5', + '51522': 'Default/network-WithLossCell/_backbone-ResNet/' + 'layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28' + } + assert expect_result == self._parser_1.to_task_id_full_op_name_dict() + assert expect_result == self._parser_2.to_task_id_full_op_name_dict() + + expect_result = { + '0_1': 'Default/Cast-op6', + '0_2': 'Default/TransData-op7', + '0_3': 'Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5', + '0_4': 'Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/' + '0-ResidualBlock/conv1-Conv2d/Cast-op28' + } + assert expect_result == self._parser_4.to_task_id_full_op_name_dict() + + def test_parse(self): + """Test the parse function.""" + expect_framework_file = os.path.join(PROFILER_DIR, 'framework_raw_0.csv') + expect_framework_file = os.path.realpath(expect_framework_file) + expect_result = get_framework_result(expect_framework_file) + + self._parser_1.parse() + framework_file = os.path.join(self._output_path_1, 'framework_raw_0.csv') + result = get_framework_result(framework_file) + assert expect_result == result + + self._parser_2.parse() + framework_file = os.path.join(self._output_path_2, 'framework_raw_0.csv') + result = get_framework_result(framework_file) + assert expect_result == result + + @mock.patch('os.listdir') + @mock.patch('os.path.isdir') + def test_create_framework_parser_fail_1(self, *args): + """Test the function of fail to create framework parser.""" + args[0].return_value = True + args[1].return_value = [] + with pytest.raises(ProfilerFileNotFoundException) as exc_info: + FrameworkParser('JOB1', '0') + assert exc_info.value.error_code == '50546084' + assert exc_info.value.message == 'The file not found.' diff --git a/tests/ut/python/profiler/parser/test_minddata_pipeline_parser.py b/tests/ut/python/profiler/parser/test_minddata_pipeline_parser.py new file mode 100644 index 0000000000..65ca7d7717 --- /dev/null +++ b/tests/ut/python/profiler/parser/test_minddata_pipeline_parser.py @@ -0,0 +1,93 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Test the minddata pipeline parser module.""" +import csv +import os +import shutil +import tempfile + +from mindspore.profiler.parser.minddata_pipeline_parser import \ + MinddataPipelineParser +from tests.ut.python.profiler import PROFILER_DIR, RAW_DATA, RAW_DATA_JOB2 + + +def get_minddata_pipeline_result(file_path): + """ + Get minddata pipeline result from the minddata pipeline file. + + Args: + file_path (str): The minddata pipeline file path. + + Returns: + list[list], the parsed minddata pipeline information. + """ + result = [] + with open(file_path, 'r') as file: + csv_reader = csv.reader(file) + for row in csv_reader: + result.append(row) + return result + + +class TestMinddataPipelineParser: + """Test the class of `MinddataPipelineParser`.""" + def setup_method(self): + """Initialization before test case execution.""" + self._output_path_1 = tempfile.mkdtemp( + prefix='test_minddata_pipeline_parser_' + ) + self._parser_1 = MinddataPipelineParser( + RAW_DATA, '0', self._output_path_1 + ) + + self._output_path_2 = tempfile.mkdtemp( + prefix='test_minddata_pipeline_parser_' + ) + self._parser_2 = MinddataPipelineParser( + RAW_DATA_JOB2, '0', self._output_path_2 + ) + + def teardown_method(self) -> None: + """Clear up after test case execution.""" + shutil.rmtree(self._output_path_1) + shutil.rmtree(self._output_path_2) + + def test_save_path(self): + """Test the querying save path function.""" + expect_result = os.path.join( + self._output_path_1, 'minddata_pipeline_raw_0.csv' + ) + assert expect_result == self._parser_1.save_path + + def test_parse(self): + """Test the parse function.""" + expect_pipeline_file = os.path.join( + PROFILER_DIR, 'minddata_pipeline_raw_0.csv' + ) + expect_result = get_minddata_pipeline_result(expect_pipeline_file) + + self._parser_1.parse() + pipeline_file = os.path.join( + self._output_path_1, 'minddata_pipeline_raw_0.csv' + ) + result = get_minddata_pipeline_result(pipeline_file) + assert expect_result == result + + self._parser_2.parse() + pipeline_file = os.path.join( + self._output_path_2, 'minddata_pipeline_raw_0.csv' + ) + result = get_minddata_pipeline_result(pipeline_file) + assert expect_result == result