!3648 Add some testing cases for mindspore.profiler
Merge pull request !3648 from 张毅辉/zyh_profiler_test
This commit is contained in:
Gitee
commit
bac1781539
|
|
@ -0,0 +1,4 @@
|
|||
op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224"
|
||||
op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16"
|
||||
op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16"
|
||||
op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16"
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
Default/Cast-op6 32 51517 0
|
||||
Default/TransData-op7 32 51518 0
|
||||
Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 51519 0
|
||||
Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 51522 0
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
{
|
||||
"sampling_interval": 10,
|
||||
"op_info": [
|
||||
{
|
||||
"op_id": 4,
|
||||
"op_type": "TFReader",
|
||||
"num_workers": 4,
|
||||
"metrics": null,
|
||||
"children": [3]
|
||||
},
|
||||
{
|
||||
"op_id": 3,
|
||||
"op_type": "TFReader",
|
||||
"num_workers": 4,
|
||||
"metrics": {
|
||||
"output_queue": {
|
||||
"size": [10, 20, 30],
|
||||
"length": 64
|
||||
}
|
||||
},
|
||||
"children": null
|
||||
},
|
||||
{
|
||||
"op_id": 2,
|
||||
"op_type": "TFReader",
|
||||
"num_workers": 4,
|
||||
"metrics": {
|
||||
"output_queue": {
|
||||
"size": [10, 20, 30],
|
||||
"length": 64
|
||||
}
|
||||
},
|
||||
"children": null
|
||||
},
|
||||
{
|
||||
"op_id": 1,
|
||||
"op_type": "Shuffle",
|
||||
"num_workers": 1,
|
||||
"metrics": {
|
||||
"output_queue": {
|
||||
"size": [10, 20, 30],
|
||||
"length": 64
|
||||
}
|
||||
},
|
||||
"children": [2, 4]
|
||||
},
|
||||
{
|
||||
"op_id": 0,
|
||||
"op_type": "Batch",
|
||||
"num_workers": 4,
|
||||
"metrics": null,
|
||||
"children": [1]
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
|
|
@ -0,0 +1,48 @@
|
|||
{
|
||||
"sampling_interval": 10,
|
||||
"op_info": [
|
||||
{
|
||||
"op_id": 3,
|
||||
"op_type": "TFReader",
|
||||
"num_workers": 4,
|
||||
"metrics": {
|
||||
"output_queue": {
|
||||
"size": [10, 20, 30],
|
||||
"length": 64
|
||||
}
|
||||
},
|
||||
"children": null
|
||||
},
|
||||
{
|
||||
"op_id": 2,
|
||||
"op_type": "TFReader",
|
||||
"num_workers": 4,
|
||||
"metrics": {
|
||||
"output_queue": {
|
||||
"size": [10, 20, 30],
|
||||
"length": 64
|
||||
}
|
||||
},
|
||||
"children": null
|
||||
},
|
||||
{
|
||||
"op_id": 1,
|
||||
"op_type": "Shuffle",
|
||||
"num_workers": 1,
|
||||
"metrics": {
|
||||
"output_queue": {
|
||||
"size": [10, 20, 30],
|
||||
"length": 64
|
||||
}
|
||||
},
|
||||
"children": [2, 3]
|
||||
},
|
||||
{
|
||||
"op_id": 0,
|
||||
"op_type": "Batch",
|
||||
"num_workers": 4,
|
||||
"metrics": null,
|
||||
"children": [1]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224"
|
||||
op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16"
|
||||
op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16"
|
||||
op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16"
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
1 Default/Cast-op6
|
||||
2 Default/TransData-op7
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
Default/Cast-op6 32 1 0
|
||||
Default/TransData-op7 32 2 0
|
||||
Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 3 0
|
||||
Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 4 0
|
||||
Binary file not shown.
|
|
@ -0,0 +1,5 @@
|
|||
serial_number node_type_name total_time(ms) dispatch_time(ms) run_start run_end
|
||||
1 InitData 1.567 0.1 2298200409 2298200538
|
||||
2 GetNext 0.989 0.087 2302769932 2302769980
|
||||
3 TruncatedNormal 1.566 0.105 4098200409 4098200538
|
||||
AI CPU Total Time(ms): 4.122000
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
op_name:Default/Cast-op6 op_type:Cast input_id:0 input_format:DefaultFormat input_data_type:40 input_shape:"32,3,224,224" output_id:0 output_format:DefaultFormat output_data_type:39 output_shape:"32,3,224,224"
|
||||
op_name:Default/TransData-op7 op_type:TransData input_id:0 input_format:DefaultFormat input_data_type:39 input_shape:"32,3,224,224" output_id:0 output_format:NC1HWC0 output_data_type:39 output_shape:"32,1,224,224,16"
|
||||
op_name:Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"49,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"49,4,16,16"
|
||||
op_name:Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 op_type:Cast input_id:0 input_format:FracZ input_data_type:40 input_shape:"4,4,16,16" output_id:0 output_format:FracZ output_data_type:39 output_shape:"4,4,16,16"
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
Default/Cast-op6 32 51517 0
|
||||
Default/TransData-op7 32 51518 0
|
||||
Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5 32 51519 0
|
||||
Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28 4 51522 0
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
full_op_time,execution_time
|
||||
Default/AtomicAddrClean-op104,0.00133
|
||||
Default/AtomicAddrClean-op105,0.000987
|
||||
Default/AtomicAddrClean-op106,0.001129
|
||||
Default/Cast-op10,0.00466
|
||||
Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op12,0.002366
|
||||
Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/gradConv2D/Cast-op53,0.004879
|
||||
Default/TransData-op11,0.006366
|
||||
Gradients/Default/network-WithLossCell/_backbone-LeNet5/gradReshape/TransData-op44,0.006782
|
||||
Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Conv2D-op13,0.05651
|
||||
Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/MatMul-op9,0.370864
|
||||
|
|
|
@ -0,0 +1,6 @@
|
|||
op_type,execution_time,execution_frequency,percent
|
||||
AtomicAddrClean,0.007283,6,0.49
|
||||
Cast,0.053395,13,3.63
|
||||
TransData,0.121800,5,8.23
|
||||
Conv2D,0.063656,2,4.33
|
||||
MatMul,1.085982,9,73.80
|
||||
|
|
|
@ -0,0 +1,5 @@
|
|||
task_id,stream_id,block_dim,full_op_name,op_name,op_type,subgraph,op_info
|
||||
51517,0,32,Default/Cast-op6,Cast-op6,Cast,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,3,224,224""}, ""output_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,3,224,224""}}"
|
||||
51518,0,32,Default/TransData-op7,TransData-op7,TransData,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,3,224,224""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,224,224,16""}}"
|
||||
51519,0,32,Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5,Cast-op5,Cast,Default,"{""input_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""49,4,16,16""}, ""output_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""49,4,16,16""}}"
|
||||
51522,0,4,Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28,Cast-op28,Cast,Default,"{""input_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""4,4,16,16""}, ""output_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""4,4,16,16""}}"
|
||||
|
|
|
@ -0,0 +1,11 @@
|
|||
task_id,stream_id,block_dim,full_op_name,op_name,op_type,subgraph,op_info
|
||||
30290,0,1,Default/AtomicAddrClean-op104,AtomicAddrClean-op104,AtomicAddrClean,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": """"}}"
|
||||
30295,0,1,Default/AtomicAddrClean-op105,AtomicAddrClean-op105,AtomicAddrClean,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""10""}}"
|
||||
30300,0,1,Default/AtomicAddrClean-op106,AtomicAddrClean-op106,AtomicAddrClean,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""84""}}"
|
||||
30268,0,32,Default/Cast-op10,Cast-op10,Cast,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,1,32,32""}, ""output_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32""}}"
|
||||
30271,0,9,Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Cast-op12,Cast-op12,Cast,Default,"{""input_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""25,1,16,16""}, ""output_0"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""25,1,16,16""}}"
|
||||
30320,0,32,Gradients/Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/gradConv2D/Cast-op53,Cast-op53,Cast,Gradients,"{""input_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,1,28,28,16""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,28,28,16""}}"
|
||||
30269,0,32,Default/TransData-op11,TransData-op11,TransData,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32""}}"
|
||||
30308,0,32,Gradients/Default/network-WithLossCell/_backbone-LeNet5/gradReshape/TransData-op44,TransData-op44,TransData,Gradients,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,16,5,5""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,5,5,16""}}"
|
||||
30272,0,32,Default/network-WithLossCell/_backbone-LeNet5/conv1-Conv2d/Conv2D-op13,Conv2D-op13,Conv2D,Default,"{""input_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,32,32,16""}, ""input_1"": {""format"": ""FracZ"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""25,1,16,16""}, ""output_0"": {""format"": ""NC1HWC0"", ""data_type"": ""NUMBER_TYPE_FLOAT16"", ""shape"": ""32,1,28,28,16""}}"
|
||||
30286,0,1,Default/network-WithLossCell/_backbone-LeNet5/fc3-Dense/MatMul-op9,MatMul-op9,MatMul,Default,"{""input_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,120""}, ""input_1"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""84,120""}, ""input_2"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""84""}, ""output_0"": {""format"": ""DefaultFormat"", ""data_type"": ""NUMBER_TYPE_FLOAT32"", ""shape"": ""32,84""}}"
|
||||
|
|
|
@ -0,0 +1,5 @@
|
|||
op_id,op_type,num_workers,output_queue_size,output_queue_average_size,output_queue_length,output_queue_usage_rate,sample_interval,parent_id,children_id
|
||||
0,Batch,4,,,,,10,,[1]
|
||||
1,Shuffle,1,"[10, 20, 30]",20.0,64,0.3125,10,0,"[2, 3]"
|
||||
2,TFReader,4,"[10, 20, 30]",20.0,64,0.3125,10,1,
|
||||
3,TFReader,4,"[10, 20, 30]",20.0,64,0.3125,10,1,
|
||||
|
|
|
@ -0,0 +1,55 @@
|
|||
{
|
||||
"sampling_interval": 10,
|
||||
"op_info": [
|
||||
{
|
||||
"op_id": 4,
|
||||
"op_type": "TFReader",
|
||||
"num_workers": 4,
|
||||
"metrics": null,
|
||||
"children": [3]
|
||||
},
|
||||
{
|
||||
"op_id": 3,
|
||||
"op_type": "TFReader",
|
||||
"num_workers": 4,
|
||||
"metrics": {
|
||||
"output_queue": {
|
||||
"size": [10, 20, 30],
|
||||
"length": 64
|
||||
}
|
||||
},
|
||||
"children": null
|
||||
},
|
||||
{
|
||||
"op_id": 2,
|
||||
"op_type": "TFReader",
|
||||
"num_workers": 4,
|
||||
"metrics": {
|
||||
"output_queue": {
|
||||
"size": [10, 20, 30],
|
||||
"length": 64
|
||||
}
|
||||
},
|
||||
"children": null
|
||||
},
|
||||
{
|
||||
"op_id": 1,
|
||||
"op_type": "Shuffle",
|
||||
"num_workers": 1,
|
||||
"metrics": {
|
||||
"output_queue": {
|
||||
"size": [10, 20, 30],
|
||||
"length": 64
|
||||
}
|
||||
},
|
||||
"children": [2, 4]
|
||||
},
|
||||
{
|
||||
"op_id": 0,
|
||||
"op_type": "Batch",
|
||||
"num_workers": 4,
|
||||
"metrics": null,
|
||||
"children": [1]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
step_num,start_point,end_point,total,fp_point,bp_point,iteration_interval,fp_and_bp,tail,stream_10_parallel_0_start_point,stream_10_parallel_0_end_point,stream_10_parallel_0,stream_10_parallel_1_start_point,stream_10_parallel_1_end_point,stream_10_parallel_1,stream_10_parallel_2_start_point,stream_10_parallel_2_end_point,stream_10_parallel_2,stream_11_parallel_0_start_point,stream_11_parallel_0_end_point,stream_11_parallel_0
|
||||
1,45000030081,45004033128,4003047,45000030081,45001733025,0,1702944,2300103,45000042679,45000060275,17596,45001048152,45001346254,298102,45002247411,45002448354,200943,45000049687,45000075987,26300
|
||||
2,45004033128,45017085658,13052530,45013070937,45014785314,9037809,1714377,2300344,45013085379,45013105429,20050,45014087119,45014385136,298017,45015297166,45015504449,207283,45013084925,45013118334,33409
|
||||
3,45017085658,45030119392,13033734,45026116231,45027818443,9030573,1702212,2300949,45026131909,45026150554,18645,45027134392,45027430418,296026,45028337093,45028537767,200674,45026129217,45026160937,31720
|
||||
4,45030119392,45043158607,13039215,45039152348,45040856975,9032956,1704627,2301632,45039169890,45039188966,19076,45040169338,45040466770,297432,45041374122,45041567754,193632,45039171681,45039193865,22184
|
||||
5,45043158607,45056198128,13039521,45052190932,45053898028,9032325,1707096,2300100,45052207675,45052222642,14967,45053204442,45053505540,301098,45054413207,45054616536,203329,45052201931,45052237599,35668
|
||||
6,45056198128,45069239564,13041436,45065233106,45066939463,9034978,1706357,2300101,45065245482,45065272534,27052,45066248423,45066546419,297996,45067455113,45067659145,204032,45065245817,45065279896,34079
|
||||
7,45069239564,45082281383,13041819,45078274997,45079980193,9035433,1705196,2301190,45078293910,45078312935,19025,45079287754,45079593841,306087,45080492957,45080691395,198438,45078292067,45078322277,30210
|
||||
8,45082281383,45095336378,13054995,45091321488,45093036084,9040105,1714596,2300294,45091338628,45091359138,20510,45092338469,45092638994,300525,45093554195,45093747470,193275,45091341356,45091369667,28311
|
||||
9,45095336378,45108372225,13035847,45104363079,45106071009,9026701,1707930,2301216,45104374524,45104400088,25564,45105378751,45105683029,304278,45106587481,45106785336,197855,45104382131,45104410852,28721
|
||||
10,45108372225,45121412413,13040188,45117401873,45119111301,9029648,1709428,2301112,45117417721,45117439668,21947,45118413083,45118718050,304967,45119629347,45119829996,200649,45117421502,45117446718,25216
|
||||
11,45121412413,45134477662,13065249,45130459598,45132175723,9047185,1716125,2301939,45130478168,45130498936,20768,45131477957,45131775220,297263,45132691645,45132893707,202062,45130470285,45130501652,31367
|
||||
12,45134477662,45147533298,13055636,45143521860,45145232553,9044198,1710693,2300745,45143533787,45143557293,23506,45144533554,45144841545,307991,45145744997,45145952255,207258,45143537383,45143563466,26083
|
||||
13,45147533298,45160588134,13054836,45156570201,45158286694,9036903,1716493,2301440,45156581069,45156609506,28437,45157581617,45157880841,299224,45158806166,45158999875,193709,45156589050,45156615664,26614
|
||||
14,45160588134,45173640064,13051930,45169625906,45171339426,9037772,1713520,2300638,45169637432,45169661754,24322,45170639482,45170940949,301467,45171853721,45172056606,202885,45169644605,45169673410,28805
|
||||
15,45173640064,45186671634,13031570,45182666696,45184371430,9026632,1704734,2300204,45182678355,45182698471,20116,45183679568,45183981082,301514,45184887156,45185083035,195879,45182680062,45182708455,28393
|
||||
16,45186671634,45199720448,13048814,45195714716,45197420410,9043082,1705694,2300038,45195728993,45195754646,25653,45196732493,45197028048,295555,45197934921,45198139237,204316,45195733069,45195764102,31033
|
||||
17,45199720448,45212762605,13042157,45208758416,45210460864,9037968,1702448,2301741,45208771010,45208790367,19357,45209773548,45210074988,301440,45210978277,45211173577,195300,45208773143,45208803280,30137
|
||||
18,45212762605,45225814601,13051996,45221801814,45223514580,9039209,1712766,2300021,45221815911,45221839644,23733,45222819211,45223114544,295333,45224031469,45224234043,202574,45221812106,45221849103,36997
|
||||
19,45225814601,45238848430,13033829,45234842015,45236548356,9027414,1706341,2300074,45234855444,45234876469,21025,45235853358,45236160825,307467,45237063061,45237260964,197903,45234857141,45234882976,25835
|
||||
20,45238848430,45251899738,13051308,45247879385,45249598280,9030955,1718895,2301458,45247896725,45247917316,20591,45248896361,45249193681,297320,45250117916,45250315651,197735,45247894228,45247926723,32495
|
||||
-,45121436513,45134482124,13045611,45130471874,45132181322,9035360,1709449,2300802,45130486422,45130508229,21808,45131486785,45131787364,300579,45132697369,45132897305,199936,45130487458,45130517315,29857
|
||||
|
|
|
@ -0,0 +1,42 @@
|
|||
step_num,start_point,end_point,total,fp_point,bp_point,iteration_interval,fp_and_bp,tail,stream_10_parallel_0_start_point,stream_10_parallel_0_end_point,stream_10_parallel_0,stream_10_parallel_1_start_point,stream_10_parallel_1_end_point,stream_10_parallel_1,stream_10_parallel_2_start_point,stream_10_parallel_2_end_point,stream_10_parallel_2,stream_11_parallel_0_start_point,stream_11_parallel_0_end_point,stream_11_parallel_0
|
||||
1,45000025226,45004034753,4009527,45000025226,45001734362,0,1709136,2300391,45000044023,45000060886,16863,45001043581,45001343373,299792,45002254048,45002452830,198782,45000043807,45000065736,21929
|
||||
2,45004034753,45017091420,13056667,45013073790,45014789509,9039037,1715719,2301911,45013085205,45013104210,19005,45014086339,45014393261,306922,45015299546,45015501808,202262,45013085040,45013119810,34770
|
||||
3,45017091420,45030144372,13052952,45026123867,45027843651,9032447,1719784,2300721,45026138546,45026154524,15978,45027135742,45027437486,301744,45028363120,45028560901,197781,45026136046,45026171363,35317
|
||||
4,45030144372,45043184486,13040114,45039173149,45040883087,9028777,1709938,2301399,45039190927,45039209948,19021,45040185915,45040484897,298982,45041399754,45041594775,195021,45039192768,45039221423,28655
|
||||
5,45043184486,45056241064,13056578,45052223555,45053940709,9039069,1717154,2300355,45052241736,45052262186,20450,45053239605,45053540866,301261,45054452604,45054654505,201901,45052233932,45052265774,31842
|
||||
6,45056241064,45069291346,13050282,45065278144,45066991121,9037080,1712977,2300225,45065293660,45065316136,22476,45066289480,45066589910,300430,45067511002,45067701731,190729,45065293679,45065321296,27617
|
||||
7,45069291346,45082344927,13053581,45078335376,45080043268,9044030,1707892,2301659,45078353164,45078365382,12218,45079354748,45079648384,293636,45080557453,45080760374,202921,45078353030,45078384530,31500
|
||||
8,45082344927,45095382554,13037627,45091368697,45093080797,9023770,1712100,2301757,45091381244,45091405208,23964,45092382630,45092684285,301655,45093590961,45093796698,205737,45091381199,45091413840,32641
|
||||
9,45095382554,45108433947,13051393,45104419947,45106132133,9037393,1712186,2301814,45104432587,45104457476,24889,45105431458,45105735476,304018,45106651213,45106845305,194092,45104435207,45104466677,31470
|
||||
10,45108433947,45121486591,13052644,45117469353,45119185969,9035406,1716616,2300622,45117483627,45117504869,21242,45118483411,45118788540,305129,45119696660,45119898575,201915,45117485587,45117510985,25398
|
||||
11,45121486591,45134546571,13059980,45130528618,45132244809,9042027,1716191,2301762,45130539730,45130561122,21392,45131538695,45131846715,308020,45132759789,45132960848,201059,45130545378,45130569412,24034
|
||||
12,45134546571,45147608222,13061651,45143597023,45145307273,9050452,1710250,2300949,45143615771,45143631460,15689,45144610592,45144910736,300144,45145818642,45146024326,205684,45143613528,45143640223,26695
|
||||
13,45147608222,45160663790,13055568,45156648696,45158362923,9040474,1714227,2300867,45156663193,45156685466,22273,45157661576,45157963074,301498,45158881212,45159074431,193219,45156667038,45156694912,27874
|
||||
14,45160663790,45173707626,13043836,45169694535,45171407246,9030745,1712711,2300380,45169710667,45169727936,17269,45170705802,45171013806,308004,45171924100,45172120273,196173,45169708524,45169739038,30514
|
||||
15,45173707626,45186754860,13047234,45182750254,45184454036,9042628,1703782,2300824,45182765445,45182789799,24354,45183761335,45184065169,303834,45184973312,45185170444,197132,45182769451,45182799598,30147
|
||||
16,45186754860,45199798718,13043858,45195792271,45197497908,9037411,1705637,2300810,45195804771,45195827915,23144,45196804016,45197108243,304227,45198013357,45198209858,196501,45195806656,45195841674,35018
|
||||
17,45199798718,45212854993,13056275,45208834355,45210553378,9035637,1719023,2301615,45208850179,45208865588,15409,45209851018,45210151436,300418,45211073169,45211271792,198623,45208847052,45208876998,29946
|
||||
18,45212854993,45225893712,13038719,45221888939,45223593704,9033946,1704765,2300008,45221901732,45221924983,23251,45222908795,45223203590,294795,45224105803,45224313354,207551,45221899792,45221938802,39010
|
||||
19,45225893712,45238941242,13047530,45234926295,45236640454,9032583,1714159,2300788,45234938628,45234957237,18609,45235942710,45236239983,297273,45237159532,45237356140,196608,45234938330,45234976170,37840
|
||||
20,45238941242,45251979177,13037935,45247977674,45249678116,9036432,1700442,2301061,45247990919,45248013476,22557,45248991451,45249294742,303291,45250195733,45250395760,200027,45247988950,45248024969,36019
|
||||
21,45251979177,45265018752,13039575,45261005416,45262718472,9026239,1713056,2300280,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
22,45265018752,45278062782,13044030,45274047185,45275762095,9028433,1714910,2300687,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
23,45278062782,45291105708,13042926,45287094000,45288805223,9031218,1711223,2300485,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
24,45291105708,45304155918,13050210,45300150844,45301854040,9045136,1703196,2301878,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
25,45304155918,45317206695,13050777,45313191948,45314905714,9036030,1713766,2300981,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
26,45317206695,45330265105,13058410,45326256021,45327964581,9049326,1708560,2300524,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
27,45330265105,45343324012,13058907,45339305124,45341023739,9040019,1718615,2300273,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
28,45343324012,45356374571,13050559,45352366211,45354073401,9042199,1707190,2301170,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
29,45356374571,45369429514,13054943,45365417827,45367128283,9043256,1710456,2301231,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
30,45369429514,45382479199,13049685,45378476397,45380177297,9046883,1700900,2301902,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
31,45382479199,45395530376,13051177,45391510137,45393229377,9030938,1719240,2300999,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
32,45395530376,45408571765,13041389,45404559082,45406270720,9028706,1711638,2301045,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
33,45408571765,45421635175,13063410,45417619223,45419334221,9047458,1714998,2300954,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
34,45421635175,45434672219,13037044,45430669445,45432371312,9034270,1701867,2300907,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
35,45434672219,45447714036,13041817,45443704548,45445413852,9032329,1709304,2300184,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
36,45447714036,45460765153,13051117,45456753675,45458463701,9039639,1710026,2301452,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
37,45460765153,45473829105,13063952,45469808281,45471527400,9043128,1719119,2301705,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
38,45473829105,45486884190,13055085,45482867237,45484583534,9038132,1716297,2300656,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
39,45486884190,45499928571,13044381,45495917628,45497627921,9033438,1710293,2300650,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
40,45499928571,45512973815,13045244,45508968990,45510673699,9040419,1704709,2300116,0,0,0,0,0,0,0,0,0,0,0,0
|
||||
-,45251983006,45265032725,13049720,45261020353,45262731761,9037347,1711408,2300964,21986676455,21986686280,9825,21987163213,21987310272,147058,21987754537,21987851587,97050,21986676441,21986691731,15290
|
||||
|
|
|
@ -0,0 +1,21 @@
|
|||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
"""Unit test for profiler."""
|
||||
import os
|
||||
|
||||
RAW_DATA_BASE = os.path.realpath(os.path.join(os.path.dirname(__file__), '../../data/profiler_data'))
|
||||
RAW_DATA = os.path.realpath(os.path.join(RAW_DATA_BASE, 'JOB1'))
|
||||
RAW_DATA_JOB2 = os.path.realpath(os.path.join(RAW_DATA_BASE, 'JOB2'))
|
||||
PROFILER_DIR = os.path.realpath(os.path.join(RAW_DATA_BASE, 'profiler'))
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
"""Test the aicpu parser."""
|
||||
import os
|
||||
import tempfile
|
||||
import shutil
|
||||
|
||||
from unittest import TestCase
|
||||
|
||||
from mindspore.profiler.parser.aicpu_data_parser import DataPreProcessParser
|
||||
|
||||
|
||||
def get_result(file_path):
|
||||
"""
|
||||
Get result from the aicpu file.
|
||||
|
||||
Args:
|
||||
file_path (str): The aicpu file path.
|
||||
|
||||
Returns:
|
||||
list[list], the parsed aicpu information.
|
||||
"""
|
||||
result = []
|
||||
try:
|
||||
file = open(file_path, 'r')
|
||||
result.append(file.read())
|
||||
return result
|
||||
finally:
|
||||
if file:
|
||||
file.close()
|
||||
|
||||
|
||||
class TestAicpuParser(TestCase):
|
||||
"""Test the class of Aicpu Parser."""
|
||||
|
||||
def setUp(self) -> None:
|
||||
"""Initialization before test case execution."""
|
||||
self.profiling_dir = os.path.realpath(os.path.join(os.path.dirname(__file__),
|
||||
'../../../data/profiler_data/'
|
||||
'JOB_AICPU/data'))
|
||||
self.expect_dir = os.path.realpath(os.path.join(os.path.dirname(__file__),
|
||||
'../../../data/profiler_data/'
|
||||
'JOB_AICPU/expect'))
|
||||
self.output_path = tempfile.mkdtemp(prefix='output_data_preprocess_aicpu_')
|
||||
self.output_file = os.path.join(self.output_path, 'output_data_preprocess_aicpu_0.txt')
|
||||
self.expect_file = os.path.join(self.expect_dir, 'output_data_preprocess_aicpu_0.txt')
|
||||
|
||||
def test_aicpu_parser(self):
|
||||
"""Test the class of Aicpu Parser."""
|
||||
data = DataPreProcessParser(self.profiling_dir, self.output_file)
|
||||
data.execute()
|
||||
expect_result = get_result(self.expect_file)
|
||||
result = get_result(self.output_file)
|
||||
shutil.rmtree(self.output_path)
|
||||
assert expect_result == result
|
||||
|
||||
def test_aicpu_parser_file_not_exist(self):
|
||||
"""Test the class of Aicpu Parser."""
|
||||
profiling_dir = os.path.realpath(os.path.join(self.profiling_dir, 'data'))
|
||||
data = DataPreProcessParser(profiling_dir, self.output_file)
|
||||
data.execute()
|
||||
shutil.rmtree(self.output_path)
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
"""Test the framework parser module."""
|
||||
import csv
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from mindspore.profiler.common.exceptions.exceptions import \
|
||||
ProfilerFileNotFoundException
|
||||
from mindspore.profiler.parser.framework_parser import FrameworkParser
|
||||
from tests.ut.python.profiler import PROFILER_DIR, RAW_DATA_BASE
|
||||
|
||||
|
||||
def get_framework_result(file_path):
|
||||
"""
|
||||
Get framework result from the framework file.
|
||||
|
||||
Args:
|
||||
file_path (str): The framework file path.
|
||||
|
||||
Returns:
|
||||
list[list], the parsed framework information.
|
||||
"""
|
||||
result = []
|
||||
with open(file_path, 'r') as file:
|
||||
csv_reader = csv.reader(file)
|
||||
for row in csv_reader:
|
||||
result.append(row)
|
||||
return result
|
||||
|
||||
|
||||
class TestFrameworkParser:
|
||||
"""Test the class of `FrameworkParser`."""
|
||||
def setup_method(self):
|
||||
"""Initialization before test case execution."""
|
||||
with mock.patch.object(FrameworkParser, '_raw_data_dir', RAW_DATA_BASE):
|
||||
self._output_path_1 = tempfile.mkdtemp(prefix='test_framework_parser_')
|
||||
self._parser_1 = FrameworkParser('JOB1', '0', self._output_path_1)
|
||||
self._output_path_2 = tempfile.mkdtemp(prefix='test_framework_parser_')
|
||||
self._parser_2 = FrameworkParser('JOB2', '0', self._output_path_2)
|
||||
self._output_path_4 = tempfile.mkdtemp(prefix='test_framework_parser_')
|
||||
self._parser_4 = FrameworkParser('JOB4', '0', self._output_path_4)
|
||||
|
||||
def teardown_method(self) -> None:
|
||||
"""Clear up after test case execution."""
|
||||
shutil.rmtree(self._output_path_1)
|
||||
shutil.rmtree(self._output_path_2)
|
||||
shutil.rmtree(self._output_path_4)
|
||||
|
||||
def test_save_path(self):
|
||||
"""Test the querying save path function."""
|
||||
expect_result = os.path.join(self._output_path_1, 'framework_raw_0.csv')
|
||||
assert expect_result == self._parser_1.save_path
|
||||
|
||||
expect_result = os.path.join(self._output_path_2, 'framework_raw_0.csv')
|
||||
assert expect_result == self._parser_2.save_path
|
||||
|
||||
def test_point_info(self):
|
||||
"""Test the querying point info function."""
|
||||
expect_result = {
|
||||
1: 'Default/Cast-op6',
|
||||
2: 'Default/TransData-op7'
|
||||
}
|
||||
assert expect_result == self._parser_4.point_info
|
||||
|
||||
def test_to_task_id_full_op_name_dict(self):
|
||||
"""Test the querying task id and full operator name dict function."""
|
||||
expect_result = {
|
||||
'51517': 'Default/Cast-op6',
|
||||
'51518': 'Default/TransData-op7',
|
||||
'51519': 'Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5',
|
||||
'51522': 'Default/network-WithLossCell/_backbone-ResNet/'
|
||||
'layer1-SequentialCell/0-ResidualBlock/conv1-Conv2d/Cast-op28'
|
||||
}
|
||||
assert expect_result == self._parser_1.to_task_id_full_op_name_dict()
|
||||
assert expect_result == self._parser_2.to_task_id_full_op_name_dict()
|
||||
|
||||
expect_result = {
|
||||
'0_1': 'Default/Cast-op6',
|
||||
'0_2': 'Default/TransData-op7',
|
||||
'0_3': 'Default/network-WithLossCell/_backbone-ResNet/conv1-Conv2d/Cast-op5',
|
||||
'0_4': 'Default/network-WithLossCell/_backbone-ResNet/layer1-SequentialCell/'
|
||||
'0-ResidualBlock/conv1-Conv2d/Cast-op28'
|
||||
}
|
||||
assert expect_result == self._parser_4.to_task_id_full_op_name_dict()
|
||||
|
||||
def test_parse(self):
|
||||
"""Test the parse function."""
|
||||
expect_framework_file = os.path.join(PROFILER_DIR, 'framework_raw_0.csv')
|
||||
expect_framework_file = os.path.realpath(expect_framework_file)
|
||||
expect_result = get_framework_result(expect_framework_file)
|
||||
|
||||
self._parser_1.parse()
|
||||
framework_file = os.path.join(self._output_path_1, 'framework_raw_0.csv')
|
||||
result = get_framework_result(framework_file)
|
||||
assert expect_result == result
|
||||
|
||||
self._parser_2.parse()
|
||||
framework_file = os.path.join(self._output_path_2, 'framework_raw_0.csv')
|
||||
result = get_framework_result(framework_file)
|
||||
assert expect_result == result
|
||||
|
||||
@mock.patch('os.listdir')
|
||||
@mock.patch('os.path.isdir')
|
||||
def test_create_framework_parser_fail_1(self, *args):
|
||||
"""Test the function of fail to create framework parser."""
|
||||
args[0].return_value = True
|
||||
args[1].return_value = []
|
||||
with pytest.raises(ProfilerFileNotFoundException) as exc_info:
|
||||
FrameworkParser('JOB1', '0')
|
||||
assert exc_info.value.error_code == '50546084'
|
||||
assert exc_info.value.message == 'The file <Framework> not found.'
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
# Copyright 2020 Huawei Technologies Co., Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
# ============================================================================
|
||||
"""Test the minddata pipeline parser module."""
|
||||
import csv
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
|
||||
from mindspore.profiler.parser.minddata_pipeline_parser import \
|
||||
MinddataPipelineParser
|
||||
from tests.ut.python.profiler import PROFILER_DIR, RAW_DATA, RAW_DATA_JOB2
|
||||
|
||||
|
||||
def get_minddata_pipeline_result(file_path):
|
||||
"""
|
||||
Get minddata pipeline result from the minddata pipeline file.
|
||||
|
||||
Args:
|
||||
file_path (str): The minddata pipeline file path.
|
||||
|
||||
Returns:
|
||||
list[list], the parsed minddata pipeline information.
|
||||
"""
|
||||
result = []
|
||||
with open(file_path, 'r') as file:
|
||||
csv_reader = csv.reader(file)
|
||||
for row in csv_reader:
|
||||
result.append(row)
|
||||
return result
|
||||
|
||||
|
||||
class TestMinddataPipelineParser:
|
||||
"""Test the class of `MinddataPipelineParser`."""
|
||||
def setup_method(self):
|
||||
"""Initialization before test case execution."""
|
||||
self._output_path_1 = tempfile.mkdtemp(
|
||||
prefix='test_minddata_pipeline_parser_'
|
||||
)
|
||||
self._parser_1 = MinddataPipelineParser(
|
||||
RAW_DATA, '0', self._output_path_1
|
||||
)
|
||||
|
||||
self._output_path_2 = tempfile.mkdtemp(
|
||||
prefix='test_minddata_pipeline_parser_'
|
||||
)
|
||||
self._parser_2 = MinddataPipelineParser(
|
||||
RAW_DATA_JOB2, '0', self._output_path_2
|
||||
)
|
||||
|
||||
def teardown_method(self) -> None:
|
||||
"""Clear up after test case execution."""
|
||||
shutil.rmtree(self._output_path_1)
|
||||
shutil.rmtree(self._output_path_2)
|
||||
|
||||
def test_save_path(self):
|
||||
"""Test the querying save path function."""
|
||||
expect_result = os.path.join(
|
||||
self._output_path_1, 'minddata_pipeline_raw_0.csv'
|
||||
)
|
||||
assert expect_result == self._parser_1.save_path
|
||||
|
||||
def test_parse(self):
|
||||
"""Test the parse function."""
|
||||
expect_pipeline_file = os.path.join(
|
||||
PROFILER_DIR, 'minddata_pipeline_raw_0.csv'
|
||||
)
|
||||
expect_result = get_minddata_pipeline_result(expect_pipeline_file)
|
||||
|
||||
self._parser_1.parse()
|
||||
pipeline_file = os.path.join(
|
||||
self._output_path_1, 'minddata_pipeline_raw_0.csv'
|
||||
)
|
||||
result = get_minddata_pipeline_result(pipeline_file)
|
||||
assert expect_result == result
|
||||
|
||||
self._parser_2.parse()
|
||||
pipeline_file = os.path.join(
|
||||
self._output_path_2, 'minddata_pipeline_raw_0.csv'
|
||||
)
|
||||
result = get_minddata_pipeline_result(pipeline_file)
|
||||
assert expect_result == result
|
||||
Loading…
Reference in New Issue