adjust ascend st from level0 to level1

This commit is contained in:
xsmq 2021-05-14 13:07:24 +08:00
parent 1d3d1fa53e
commit a424c3124b
25 changed files with 225 additions and 225 deletions

View File

@ -123,7 +123,7 @@ class ControlGraphSupportNotEqual(Cell):
return out, out2, out3
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -143,7 +143,7 @@ def test_ctrl_if_while_graph_support_not_equal_true():
allclose_nparray(out3, out_me[2].asnumpy(), 0.0001, 0.0001)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -186,7 +186,7 @@ class ControlBprop(Cell):
return x*2, y*3, z, input_data*5.1
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -237,7 +237,7 @@ class InlineBpropTwoInput1(Cell):
return grads[0]*2, grads[1]*2
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -315,7 +315,7 @@ class SideEffectMemoryCellAddnNet(Cell):
return grad_out
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -347,7 +347,7 @@ class SideEffectIOCellAddnNet(Cell):
return grad_out
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -379,7 +379,7 @@ class SideEffectReturnParameterNet(Cell):
return grad_out
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -462,7 +462,7 @@ class SideEffectPrintInHighOrdeAddnNet(Cell):
return grad_out
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -540,7 +540,7 @@ class SideEffectTwoAddnSwitchNet(Cell):
return grad_out
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -582,7 +582,7 @@ class SideEffectGradIfNet(Cell):
return grad_out
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -639,7 +639,7 @@ def test_highgrad_one_input_sec_grad():
assert (dxdx.asnumpy() == np.array([5, 5]).astype(np.float32)).all()
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -39,7 +39,7 @@ class AssignAddNet(nn.Cell):
return self.para
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -63,7 +63,7 @@ class AssignSubNet(nn.Cell):
return self.para
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -87,7 +87,7 @@ class ScatterAddNet(nn.Cell):
return self.input_x
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -112,7 +112,7 @@ class ScatterSubNet(nn.Cell):
return self.input_x
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -137,7 +137,7 @@ class ScatterMulNet(nn.Cell):
return self.input_x
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -162,7 +162,7 @@ class ScatterDivNet(nn.Cell):
return self.input_x
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -187,7 +187,7 @@ class ScatterMaxNet(nn.Cell):
return self.input_x
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -212,7 +212,7 @@ class ScatterMinNet(nn.Cell):
return self.input_x
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -237,7 +237,7 @@ class ScatterUpdateNet(nn.Cell):
return self.input_x
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -262,7 +262,7 @@ class ScatterNdAddNet(nn.Cell):
return self.input_x
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -287,7 +287,7 @@ class ScatterNdSubNet(nn.Cell):
return self.input_x
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -380,7 +380,7 @@ def train_summary_record(test_writer, steps):
return out_me_dict
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -74,7 +74,7 @@ class ApplyAdaMaxNet(nn.Cell):
return self.var, self.m, self.v
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -109,7 +109,7 @@ class ApplyAdadeltaNet(nn.Cell):
return self.var, self.accum, self.accum_update
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -140,7 +140,7 @@ class ApplyAdagrad(nn.Cell):
return self.var, self.accum
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -168,7 +168,7 @@ class ApplyAdagradV2Net(nn.Cell):
return self.var, self.accum
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -197,7 +197,7 @@ class ApplyAddSignNet(nn.Cell):
return self.var, self.m
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -228,7 +228,7 @@ class ApplyCenteredRMSPropNet(nn.Cell):
return self.var
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -295,7 +295,7 @@ class ApplyGradientDescentNet(nn.Cell):
return self.var
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -382,7 +382,7 @@ class ApplyProximalAdagradNet(nn.Cell):
return self.var, self.accum
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -411,7 +411,7 @@ class ApplyProximalGradientDescentNet(nn.Cell):
return self.var
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -469,7 +469,7 @@ class FusedSparseAdamNet(nn.Cell):
return self.var, self.m, self.v
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -607,7 +607,7 @@ class SparseApplyAdagradNet(nn.Cell):
return self.var, self.accum
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -636,7 +636,7 @@ class SparseApplyAdagradV2Net(nn.Cell):
return self.var, self.accum
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -759,7 +759,7 @@ class SGDNet(nn.Cell):
return self.var
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -791,7 +791,7 @@ class ApplyProximalAdagradConstantNet(nn.Cell):
return self.depend(self.const, optimizer)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -822,7 +822,7 @@ class MulSGDNet(nn.Cell):
return out
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -149,7 +149,7 @@ class CompositePoissonNet(nn.Cell):
return s1, s2, s3
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -204,7 +204,7 @@ class StandardNormalNet(nn.Cell):
return s1, s2, s3
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -260,7 +260,7 @@ class GammaNet(nn.Cell):
return s1, s2, s3
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -289,7 +289,7 @@ class PoissonNet(nn.Cell):
return s1, s2, s3
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -317,7 +317,7 @@ class UniformIntNet(nn.Cell):
return s1, s2, s3
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -346,7 +346,7 @@ class UniformRealNet(nn.Cell):
return s1, s2, s3
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -423,7 +423,7 @@ class RandomCategoricalNet(nn.Cell):
return s1, s2, s3
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -990,7 +990,7 @@ def test_if_by_if_return_inside_grad():
pynative_output = net(idx, end, x)
assert np.allclose(graph_output[0].asnumpy(), pynative_output[0].asnumpy(), 0.0001, 0.0001)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -1165,7 +1165,7 @@ def test_if_by_if_forward_control_inside_net():
pynative_output = net(idx, end, x)
assert np.allclose(graph_output.asnumpy(), pynative_output.asnumpy(), 0.0001, 0.0001)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -1207,7 +1207,7 @@ def test_if_by_if_forward_use_namespace():
pynative_output = net(idx, end, x)
assert np.allclose(graph_output.asnumpy(), pynative_output.asnumpy(), 0.0001, 0.0001)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -1253,7 +1253,7 @@ def test_if_by_if_forward_use_global_op():
pynative_output = net(idx, end, x)
assert np.allclose(graph_output.asnumpy(), pynative_output.asnumpy(), 0.0001, 0.0001)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -1286,7 +1286,7 @@ def test_for_with_if_by_if_forward():
pynative_output = net(idx, end, x)
assert np.allclose(graph_output.asnumpy(), pynative_output.asnumpy(), 0.0001, 0.0001)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -1322,7 +1322,7 @@ def test_for_with_if_by_if_forward_namespace():
assert np.allclose(graph_output.asnumpy(), pynative_output.asnumpy(), 0.0001, 0.0001)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -1368,7 +1368,7 @@ def test_if_by_if_forward_const_branch_inner():
pynative_output = net(idx, end, x)
assert np.allclose(graph_output.asnumpy(), pynative_output.asnumpy(), 0.0001, 0.0001)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -58,7 +58,7 @@ class TestGradCAM:
self.net = SimpleAvgLinear()
self.data = ms.Tensor(np.random.random(size=(1, 1, 4, 4)), ms.float32)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -83,7 +83,7 @@ class TestGradCAM:
res = aggregation(gap_grad * activation)
assert np.allclose(res.asnumpy(), attribution.asnumpy(), atol=1e-5, rtol=1e-3)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -51,7 +51,7 @@ class TestGradient:
self.relu = P.ReLU()
self.abs_ = P.Abs()
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -73,7 +73,7 @@ class Net(nn.Cell):
return x
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training

View File

@ -1,63 +1,63 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import os
import pytest
from tests.st.model_zoo_tests import utils
@pytest.mark.level0
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_arm_ascend_training
@pytest.mark.env_single
def test_DeeplabV3_voc2007():
cur_path = os.path.dirname(os.path.abspath(__file__))
model_path = "{}/../../../../model_zoo/official/cv".format(cur_path)
model_name = "deeplabv3"
utils.copy_files(model_path, cur_path, model_name)
cur_model_path = os.path.join(cur_path, model_name)
old_list = ['/PATH_TO_DATA/vocaug/vocaug_mindrecord/vocaug_mindrecord0',
'/PATH/TO/PRETRAIN_MODEL']
new_list = [os.path.join(utils.data_root, "voc/voc2012/mindrecord_train/vocaug_mindrecord0"),
os.path.join(utils.ckpt_root, "deeplabv3/resnet101_ascend.ckpt")]
utils.exec_sed_command(old_list, new_list,
os.path.join(cur_model_path, "scripts/run_distribute_train_s16_r1.sh"))
old_list = ['model.train(args.train_epochs',
'callbacks=cbs']
new_list = ['model.train(70',
'callbacks=cbs, sink_size=2']
utils.exec_sed_command(old_list, new_list, os.path.join(cur_model_path, "train.py"))
exec_network_shell = "cd {}/scripts; sh run_distribute_train_s16_r1.sh {}".format(
model_name, utils.rank_table_path)
ret = os.system(exec_network_shell)
assert ret == 0
cmd = "ps -ef | grep python | grep train.py | grep -v grep"
ret = utils.process_check(100, cmd)
assert ret
log_file = os.path.join(cur_model_path, "scripts/s16_aug_train/device{}/log")
for i in range(8):
per_step_time = utils.get_perf_data(log_file.format(i))
print("per_step_time is", per_step_time)
assert per_step_time < 530.0
loss_list = []
for i in range(8):
loss = utils.get_loss_data_list(log_file.format(i))
print("loss is", loss[-1])
loss_list.append(loss[-1])
assert sum(loss_list) / len(loss_list) < 2.5
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import os
import pytest
from tests.st.model_zoo_tests import utils
@pytest.mark.level1
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_arm_ascend_training
@pytest.mark.env_single
def test_DeeplabV3_voc2007():
cur_path = os.path.dirname(os.path.abspath(__file__))
model_path = "{}/../../../../model_zoo/official/cv".format(cur_path)
model_name = "deeplabv3"
utils.copy_files(model_path, cur_path, model_name)
cur_model_path = os.path.join(cur_path, model_name)
old_list = ['/PATH_TO_DATA/vocaug/vocaug_mindrecord/vocaug_mindrecord0',
'/PATH/TO/PRETRAIN_MODEL']
new_list = [os.path.join(utils.data_root, "voc/voc2012/mindrecord_train/vocaug_mindrecord0"),
os.path.join(utils.ckpt_root, "deeplabv3/resnet101_ascend.ckpt")]
utils.exec_sed_command(old_list, new_list,
os.path.join(cur_model_path, "scripts/run_distribute_train_s16_r1.sh"))
old_list = ['model.train(args.train_epochs',
'callbacks=cbs']
new_list = ['model.train(70',
'callbacks=cbs, sink_size=2']
utils.exec_sed_command(old_list, new_list, os.path.join(cur_model_path, "train.py"))
exec_network_shell = "cd {}/scripts; sh run_distribute_train_s16_r1.sh {}".format(
model_name, utils.rank_table_path)
ret = os.system(exec_network_shell)
assert ret == 0
cmd = "ps -ef | grep python | grep train.py | grep -v grep"
ret = utils.process_check(100, cmd)
assert ret
log_file = os.path.join(cur_model_path, "scripts/s16_aug_train/device{}/log")
for i in range(8):
per_step_time = utils.get_perf_data(log_file.format(i))
print("per_step_time is", per_step_time)
assert per_step_time < 530.0
loss_list = []
for i in range(8):
loss = utils.get_loss_data_list(log_file.format(i))
print("loss is", loss[-1])
loss_list.append(loss[-1])
assert sum(loss_list) / len(loss_list) < 2.5

View File

@ -1,64 +1,64 @@
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import os
import pytest
from tests.st.model_zoo_tests import utils
@pytest.mark.level0
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_arm_ascend_training
@pytest.mark.env_single
def test_gnmtv2_WMT_English_German():
cur_path = os.path.dirname(os.path.abspath(__file__))
model_path = "{}/../../../../model_zoo/official/nlp".format(cur_path)
model_name = "gnmt_v2"
utils.copy_files(model_path, cur_path, model_name)
cur_model_path = os.path.join(cur_path, model_name)
old_list = ['dataset_sink_mode=config.dataset_sink_mode']
new_list = ['dataset_sink_mode=config.dataset_sink_mode, sink_size=100']
utils.exec_sed_command(old_list, new_list, os.path.join(cur_model_path, "train.py"))
old_list = ['"epochs": 6,']
new_list = ['"epochs": 4,']
utils.exec_sed_command(old_list, new_list, os.path.join(cur_model_path, "config/config.json"))
mindrecord_file = "wmt16_de_en/train_tok_mindrecord/train.tok.clean.bpe.32000.en.mindrecord"
exec_network_shell = "cd {}/scripts; sh run_distributed_train_ascend.sh {} {}"\
.format(model_name, utils.rank_table_path, os.path.join(utils.data_root, mindrecord_file))
ret = os.system(exec_network_shell)
assert ret == 0
cmd = "ps -ef | grep python | grep train.py | grep train.tok.clean.bpe | grep -v grep"
ret = utils.process_check(120, cmd)
assert ret
log_file = os.path.join(cur_model_path, "scripts/device{}/log_gnmt_network{}.log")
for i in range(8):
per_step_time = utils.get_perf_data(log_file.format(i, i))
print("per_step_time is", per_step_time)
assert per_step_time < 270.0
log_file = os.path.join(cur_model_path, "scripts/device{}/loss.log")
loss_list = []
for i in range(8):
pattern1 = r"loss\: ([\d\.\+]+)\,"
loss = utils.parse_log_file(pattern1, log_file.format(i))
print("loss is", loss)
loss_list.append(loss[-1])
print("loss_list is", loss_list)
print(sum(loss_list) / len(loss_list))
assert sum(loss_list) / len(loss_list) < 120
# Copyright 2021 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
import os
import pytest
from tests.st.model_zoo_tests import utils
@pytest.mark.level1
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_arm_ascend_training
@pytest.mark.env_single
def test_gnmtv2_WMT_English_German():
cur_path = os.path.dirname(os.path.abspath(__file__))
model_path = "{}/../../../../model_zoo/official/nlp".format(cur_path)
model_name = "gnmt_v2"
utils.copy_files(model_path, cur_path, model_name)
cur_model_path = os.path.join(cur_path, model_name)
old_list = ['dataset_sink_mode=config.dataset_sink_mode']
new_list = ['dataset_sink_mode=config.dataset_sink_mode, sink_size=100']
utils.exec_sed_command(old_list, new_list, os.path.join(cur_model_path, "train.py"))
old_list = ['"epochs": 6,']
new_list = ['"epochs": 4,']
utils.exec_sed_command(old_list, new_list, os.path.join(cur_model_path, "config/config.json"))
mindrecord_file = "wmt16_de_en/train_tok_mindrecord/train.tok.clean.bpe.32000.en.mindrecord"
exec_network_shell = "cd {}/scripts; sh run_distributed_train_ascend.sh {} {}"\
.format(model_name, utils.rank_table_path, os.path.join(utils.data_root, mindrecord_file))
ret = os.system(exec_network_shell)
assert ret == 0
cmd = "ps -ef | grep python | grep train.py | grep train.tok.clean.bpe | grep -v grep"
ret = utils.process_check(120, cmd)
assert ret
log_file = os.path.join(cur_model_path, "scripts/device{}/log_gnmt_network{}.log")
for i in range(8):
per_step_time = utils.get_perf_data(log_file.format(i, i))
print("per_step_time is", per_step_time)
assert per_step_time < 270.0
log_file = os.path.join(cur_model_path, "scripts/device{}/loss.log")
loss_list = []
for i in range(8):
pattern1 = r"loss\: ([\d\.\+]+)\,"
loss = utils.parse_log_file(pattern1, log_file.format(i))
print("loss is", loss)
loss_list.append(loss[-1])
print("loss_list is", loss_list)
print(sum(loss_list) / len(loss_list))
assert sum(loss_list) / len(loss_list) < 120

View File

@ -24,7 +24,7 @@ from mindspore import Tensor, context, export
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -203,7 +203,7 @@ def test_transformer():
assert per_step_mseconds <= expect_per_step_mseconds + 2
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -84,7 +84,7 @@ class TimeMonitor(Callback):
DATA_DIR = "/home/workspace/mindspore_dataset/coco/coco2017/mindrecord_train/yolov3"
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_single

View File

@ -212,7 +212,7 @@ def test_yolov3_darknet53():
print('==========test case passed===========')
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_arm_ascend_training
@pytest.mark.env_single

View File

@ -245,7 +245,7 @@ def test_bert_precision(enable_graph_kernel=False):
assert np.allclose(loss_scale, expect_loss_scale, 0, 0)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -203,7 +203,7 @@ def create_dataset(data_path, batch_size=32, repeat_size=1,
return mnist_ds
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -974,7 +974,7 @@ def test_empty_like_exception():
mnp.empty_like([[1, 2, 3], [4, 5]])
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training

View File

@ -39,7 +39,7 @@ class GammaMean(nn.Cell):
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.env_onecard
def test_probability_gamma_mean_cdoncentration_rate_rand_2_ndarray():

View File

@ -35,7 +35,7 @@ class NetWithSparseGatherV2(nn.Cell):
def construct(self, indices, label):
return self.gather(self.weight1, indices, self.axis) + self.weight2
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -56,7 +56,7 @@ def test_pynative_ftrl_net():
[[0.6821311, 0.6821311]],
[[0.6821311, 0.6821311]]]))
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -145,7 +145,7 @@ class MSELoss(nn.Cell):
diff = data - label
return self.reduce_mean(self.square(diff), get_axis(diff))
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -168,7 +168,7 @@ def test_loss_scale_fp16_lr_overflow():
assert output_1[0].asnumpy() == output_2[0].asnumpy()
assert output_1[1].asnumpy() == output_2[1].asnumpy() == True
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -211,7 +211,7 @@ def test_loss_scale_fp16_model_train_overflow():
model = Model(net, loss_fn=loss, optimizer=optimizer, metrics=None, loss_scale_manager=scale_manager)
model.train(2, dataset, dataset_sink_mode=False)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -73,7 +73,7 @@ def test_parser_tensor_assign_slice_002():
assert np.allclose(out_me.asnumpy(), input_np_x, rtol=0.01, atol=0.01)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -436,7 +436,7 @@ def test_pynative_hook_outermost_cell_not_change_grad_gpu():
context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU")
pynative_hook_outermost_cell_not_change_grad()
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -451,7 +451,7 @@ def test_pynative_hook_all_cell_record_grad_gpu():
context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU")
pynative_hook_all_cell_record_grad()
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
@ -481,7 +481,7 @@ def test_pynative_hook_mul2_change_input_grad_gpu():
context.set_context(mode=context.PYNATIVE_MODE, device_target="GPU")
pynative_hook_mul2_change_input_grad()
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -604,7 +604,7 @@ def test_tesnsor_augassign_by_list():
# GPU: does not supported op "FloorMod"
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard

View File

@ -48,7 +48,7 @@ class NetWorkSlicePositive(Cell):
return ret0, ret1, ret2, ret3
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -213,7 +213,7 @@ class TensorGetItemByMixedTensorsBasicCase(Cell):
return ret0, ret1, ret2, ret3, ret4, ret5
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -247,7 +247,7 @@ class TensorItemByNone(Cell):
return ret
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -272,7 +272,7 @@ class TensorItemByItem(Cell):
return ret
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -308,7 +308,7 @@ def test_item_by_int():
net(input_3d_ms, index_np_4)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -439,7 +439,7 @@ class TensorSetItemByMixedTensors_2(Cell):
return ret
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -466,7 +466,7 @@ class TensorGetItemByMixedTensorsIndexError(Cell):
return ret
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -493,7 +493,7 @@ class TensorSetItemByOneTensorWithNumber(Cell):
return ret
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -522,7 +522,7 @@ class TensorSetItemByOneTensorWithTensor(Cell):
return ret
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -553,7 +553,7 @@ class TensorSetItemByOneTensorWithTupleOfNumber(Cell):
return ret
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -582,7 +582,7 @@ class TensorSetItemByOneTensorWithTupleOfTensor(Cell):
return ret
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -617,7 +617,7 @@ class TensorSetItemByTensorsWithNumber(Cell):
return ret
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -685,7 +685,7 @@ class TensorSetItemByTensorsWithTensorNumberError(Cell):
return ret
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -714,7 +714,7 @@ class TensorSetItemByTensorsWithTupleOfNumber(Cell):
return ret
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
# GPU op has bug, and has not been fixed.
@ -747,7 +747,7 @@ class TensorSetItemByTensorsWithTupleOfTensor(Cell):
return ret
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
# GPU op has bug, and has not been fixed.
@ -785,7 +785,7 @@ class TensorSetItemByTensorsWithTupleOfTensorNumberError(Cell):
return ret
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -877,7 +877,7 @@ class TensorAssignWithSlice(Cell):
return z
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -927,7 +927,7 @@ def test_tensor_assign_slice_value_2():
assert np.all(z == out.asnumpy())
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -1121,7 +1121,7 @@ def test_tensor_assign_bool_index_0():
assert np.all(out.asnumpy() == res)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -1142,7 +1142,7 @@ def test_tensor_assign_bool_index_1():
assert np.all(out.asnumpy() == res)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -1207,7 +1207,7 @@ def test_tensor_slice_reduce_out_of_bounds_neg():
assert "begin should be in [-6, 6), but got stride: 1, begin: -7." in str(ex.value)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training

View File

@ -236,7 +236,7 @@ class TensorItemSetWithNumber(Cell):
return ret
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -287,7 +287,7 @@ def test_setitem_dim_expand():
setup_testcase(x, cases)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training
@ -341,7 +341,7 @@ def test_itemset_by_number_with_number():
net(input_3d_ms, index_np_4, value_np_2)
@pytest.mark.level0
@pytest.mark.level1
@pytest.mark.platform_arm_ascend_training
@pytest.mark.platform_x86_ascend_training
@pytest.mark.platform_x86_gpu_training