diff --git a/tests/st/model_zoo_tests/bgcf/test_BGCF_amazon_beauty.py b/tests/st/model_zoo_tests/bgcf/test_BGCF_amazon_beauty.py index 5caea24b44b..0b167db106b 100644 --- a/tests/st/model_zoo_tests/bgcf/test_BGCF_amazon_beauty.py +++ b/tests/st/model_zoo_tests/bgcf/test_BGCF_amazon_beauty.py @@ -29,24 +29,24 @@ def test_BGCF_amazon_beauty(): utils.copy_files(model_path, cur_path, model_name) cur_model_path = os.path.join(cur_path, model_name) - old_list = ["--datapath=../data_mr"] - new_list = ["--datapath={}".format(os.path.join(utils.data_root, "amazon_beauty/mindrecord_train"))] - utils.exec_sed_command(old_list, new_list, os.path.join(cur_model_path, "scripts/run_train_ascend.sh")) old_list = ["default=600,"] new_list = ["default=50,"] utils.exec_sed_command(old_list, new_list, os.path.join(cur_model_path, "src/config.py")) - old_list = ["context.set_context(device_id=int(parser.device))"] - new_list = ["context.set_context()"] + old_list = ["context.set_context(device_id=int(parser.device))", + "save_checkpoint("] + new_list = ["context.set_context()", + "pass \\# save_checkpoint("] utils.exec_sed_command(old_list, new_list, os.path.join(cur_model_path, "train.py")) - exec_network_shell = "cd {}/scripts; bash run_train_ascend.sh".format(model_name) + + data_path = os.path.join(utils.data_root, "amazon_beauty/mindrecord_train") + exec_network_shell = "cd {}; python train.py --datapath={} &> log"\ + .format(model_name, data_path) + print("train process is running...") ret = os.system(exec_network_shell) assert ret == 0 + print("train process finished.") - cmd = "ps -ef|grep python |grep train.py|grep amazon_beauty|grep -v grep" - ret = utils.process_check(300, cmd) - assert ret - - log_file = os.path.join(cur_model_path, "scripts/train/log") + log_file = os.path.join(cur_model_path, "log") pattern1 = r"loss ([\d\.\+]+)\," loss_list = utils.parse_log_file(pattern1, log_file) loss_list = loss_list[-5:] diff --git a/tests/st/model_zoo_tests/centernet/test_centernet.py b/tests/st/model_zoo_tests/centernet/test_centernet.py index af0790abad7..e6e81ed1799 100644 --- a/tests/st/model_zoo_tests/centernet/test_centernet.py +++ b/tests/st/model_zoo_tests/centernet/test_centernet.py @@ -28,8 +28,8 @@ def test_center_net(): model_name = "centernet" utils.copy_files(model_path, cur_path, model_name) cur_model_path = os.path.join(cur_path, model_name) - old_list = ['new_repeat_count, dataset'] - new_list = ['5, dataset'] + old_list = ['new_repeat_count, dataset', 'args_opt.data_sink_steps'] + new_list = ['5, dataset', '20'] utils.exec_sed_command(old_list, new_list, os.path.join(cur_model_path, "train.py")) old_list = ["device_ips = {}", "device_ip.strip()", "rank_size = 0", "this_server = server", @@ -47,16 +47,16 @@ def test_center_net(): .format(dataset_path, utils.rank_table_path) os.system(exec_network_shell) cmd = "ps -ef |grep train.py | grep coco | grep -v grep" - ret = utils.process_check(100, cmd) + ret = utils.process_check(120, cmd) assert ret log_file = os.path.join(cur_model_path, "LOG{}/training_log.txt") for i in range(8): per_step_time = utils.get_perf_data(log_file.format(i)) - assert per_step_time < 385 + assert per_step_time < 435 loss_list = [] for i in range(8): loss_cmd = "grep -nr \"outputs are\" {} | awk '{{print $14}}' | awk -F\")\" '{{print $1}}'"\ .format(log_file.format(i)) loss = utils.get_loss_data_list(log_file.format(i), cmd=loss_cmd) loss_list.append(loss[-1]) - assert sum(loss_list) / len(loss_list) < 35.0 + assert sum(loss_list) / len(loss_list) < 58.8 diff --git a/tests/st/model_zoo_tests/deeplabv3/test_DeeplabV3_voc2007.py b/tests/st/model_zoo_tests/deeplabv3/test_DeeplabV3_voc2007.py index 1146564bf14..8a22f4d8381 100644 --- a/tests/st/model_zoo_tests/deeplabv3/test_DeeplabV3_voc2007.py +++ b/tests/st/model_zoo_tests/deeplabv3/test_DeeplabV3_voc2007.py @@ -38,7 +38,7 @@ def test_DeeplabV3_voc2007(): old_list = ['model.train(args.train_epochs', 'callbacks=cbs'] - new_list = ['model.train(70', + new_list = ['model.train(30', 'callbacks=cbs, sink_size=2'] utils.exec_sed_command(old_list, new_list, os.path.join(cur_model_path, "train.py")) @@ -54,10 +54,10 @@ def test_DeeplabV3_voc2007(): for i in range(8): per_step_time = utils.get_perf_data(log_file.format(i)) print("per_step_time is", per_step_time) - assert per_step_time < 530.0 + assert per_step_time < 585.0 loss_list = [] for i in range(8): loss = utils.get_loss_data_list(log_file.format(i)) print("loss is", loss[-1]) loss_list.append(loss[-1]) - assert sum(loss_list) / len(loss_list) < 2.5 + assert sum(loss_list) / len(loss_list) < 3.5 diff --git a/tests/st/model_zoo_tests/gnmt_v2/test_gnmtv2_WMT_English_German.py b/tests/st/model_zoo_tests/gnmt_v2/test_gnmtv2_WMT_English_German.py index 1b94912fd70..38820d4c71a 100644 --- a/tests/st/model_zoo_tests/gnmt_v2/test_gnmtv2_WMT_English_German.py +++ b/tests/st/model_zoo_tests/gnmt_v2/test_gnmtv2_WMT_English_German.py @@ -30,7 +30,7 @@ def test_gnmtv2_WMT_English_German(): cur_model_path = os.path.join(cur_path, model_name) old_list = ['dataset_sink_mode=config.dataset_sink_mode'] - new_list = ['dataset_sink_mode=config.dataset_sink_mode, sink_size=100'] + new_list = ['dataset_sink_mode=config.dataset_sink_mode, sink_size=25'] utils.exec_sed_command(old_list, new_list, os.path.join(cur_model_path, "train.py")) old_list = ['"epochs": 6,'] new_list = ['"epochs": 4,'] @@ -50,7 +50,7 @@ def test_gnmtv2_WMT_English_German(): for i in range(8): per_step_time = utils.get_perf_data(log_file.format(i, i)) print("per_step_time is", per_step_time) - assert per_step_time < 270.0 + assert per_step_time < 330.0 log_file = os.path.join(cur_model_path, "scripts/device{}/loss.log") loss_list = [] @@ -61,4 +61,4 @@ def test_gnmtv2_WMT_English_German(): loss_list.append(loss[-1]) print("loss_list is", loss_list) print(sum(loss_list) / len(loss_list)) - assert sum(loss_list) / len(loss_list) < 120 + assert sum(loss_list) / len(loss_list) < 260 diff --git a/tests/st/model_zoo_tests/ssd_mobilenet_fpn/test_ssd_modelnet_fpn.py b/tests/st/model_zoo_tests/ssd_mobilenet_fpn/test_ssd_modelnet_fpn.py index b284e96b60c..6bc3d8fcc43 100644 --- a/tests/st/model_zoo_tests/ssd_mobilenet_fpn/test_ssd_modelnet_fpn.py +++ b/tests/st/model_zoo_tests/ssd_mobilenet_fpn/test_ssd_modelnet_fpn.py @@ -41,7 +41,7 @@ def test_SSD_mobilenet_v1_fpn_coco2017(): new_list = ["ssd_mobilenet_v1_fpn"] utils.exec_sed_command(old_list, new_list, os.path.join(cur_model_path, "src/config.py")) old_list = ["args_opt.epoch_size", "dataset_sink_mode=dataset_sink_mode"] - new_list = ["5", "dataset_sink_mode=dataset_sink_mode, sink_size=100"] + new_list = ["5", "dataset_sink_mode=dataset_sink_mode, sink_size=20"] utils.exec_sed_command(old_list, new_list, os.path.join(cur_model_path, "train.py")) exec_network_shell = "cd {0}; sh -x scripts/run_distribute_train.sh 8 {1} 0.2 coco {2}"\ @@ -55,10 +55,10 @@ def test_SSD_mobilenet_v1_fpn_coco2017(): for i in range(8): per_step_time = utils.get_perf_data(log_file.format(i)) print("per_step_time is", per_step_time) - assert per_step_time < 545 + assert per_step_time < 580 loss_list = [] for i in range(8): loss = utils.get_loss_data_list(log_file.format(i)) print("loss is", loss[-1]) loss_list.append(loss[-1]) - assert sum(loss_list) / len(loss_list) < 2.72 + assert sum(loss_list) / len(loss_list) < 4.9 diff --git a/tests/st/model_zoo_tests/yolov3_darknet53/test_yolov3_darknet53.py b/tests/st/model_zoo_tests/yolov3_darknet53/test_yolov3_darknet53.py index ed191dc7b76..a8152daa599 100644 --- a/tests/st/model_zoo_tests/yolov3_darknet53/test_yolov3_darknet53.py +++ b/tests/st/model_zoo_tests/yolov3_darknet53/test_yolov3_darknet53.py @@ -245,6 +245,6 @@ def test_yolov3_darknet_8p(): pattern1 = r", *([\d\.]+) imgs/sec" pattern2 = r"loss:*([\d\.]+)," fps_list = utils.parse_log_file(pattern1, train_log_file)[1:] - assert sum(fps_list) / len(fps_list) > 480 + assert sum(fps_list) / len(fps_list) > 370 loss_list = utils.parse_log_file(pattern2, train_log_file) assert loss_list[-1] < 280