diff --git a/model_zoo/official/cv/inceptionv3/scripts/run_distribute_train_gpu.sh b/model_zoo/official/cv/inceptionv3/scripts/run_distribute_train_gpu.sh index 309fde42546..3801169717f 100644 --- a/model_zoo/official/cv/inceptionv3/scripts/run_distribute_train_gpu.sh +++ b/model_zoo/official/cv/inceptionv3/scripts/run_distribute_train_gpu.sh @@ -20,5 +20,5 @@ BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd) CONFIG_FILE="${BASE_PATH}/../default_config_gpu.yaml" mpirun --allow-run-as-root -n 8 --output-filename log_output --merge-stderr-to-stdout \ - python ./train.py --config_path=$CONFIG_FILE --is_distributed --platform 'GPU' \ + python ./train.py --config_path=$CONFIG_FILE --is_distributed=True --platform 'GPU' \ --dataset_path $DATA_DIR --ckpt_path=$CKPT_PATH > train.log 2>&1 & diff --git a/model_zoo/official/nlp/lstm/config_ascend.yaml b/model_zoo/official/nlp/lstm/config_ascend.yaml index 35a7bf21ce8..01d6827c4fb 100644 --- a/model_zoo/official/nlp/lstm/config_ascend.yaml +++ b/model_zoo/official/nlp/lstm/config_ascend.yaml @@ -35,7 +35,7 @@ global_step: 0 preprocess: 'false' aclimdb_path: "/cache/data/aclImdb" glove_path: "/cache/data" -preprocess_path: "./preprocess" +preprocess_path: "/cache/train/preprocess" ckpt_path: './ckpt_lstm/' pre_trained: '' # None device_num: 1 @@ -77,6 +77,5 @@ enable_graph_kernel: 'Accelerate by graph kernel, default is true.' --- device_target: ['Ascend', 'GPU', 'CPU'] distribute: ['true', 'false'] -distribute: ['true', 'false'] enable_graph_kernel: ['true', 'false'] file_format: ['AIR', 'ONNX', 'MINDIR'] \ No newline at end of file diff --git a/model_zoo/official/nlp/lstm/config_ascend_8p.yaml b/model_zoo/official/nlp/lstm/config_ascend_8p.yaml index f4a871cf3f5..21437430292 100644 --- a/model_zoo/official/nlp/lstm/config_ascend_8p.yaml +++ b/model_zoo/official/nlp/lstm/config_ascend_8p.yaml @@ -36,7 +36,7 @@ global_step: 0 preprocess: 'false' aclimdb_path: "/cache/data/aclImdb" glove_path: "/cache/data" -preprocess_path: "./preprocess" +preprocess_path: "/cache/train/preprocess" ckpt_path: './ckpt_lstm/' pre_trained: '' # None device_num: 8 @@ -79,6 +79,5 @@ enable_graph_kernel: 'Accelerate by graph kernel, default is true.' --- device_target: ['Ascend', 'GPU', 'CPU'] distribute: ['true', 'false'] -distribute: ['true', 'false'] enable_graph_kernel: ['true', 'false'] file_format: ['AIR', 'ONNX', 'MINDIR'] diff --git a/model_zoo/official/nlp/lstm/default_config.yaml b/model_zoo/official/nlp/lstm/default_config.yaml index 5be3b87feec..6ae61832028 100644 --- a/model_zoo/official/nlp/lstm/default_config.yaml +++ b/model_zoo/official/nlp/lstm/default_config.yaml @@ -30,7 +30,7 @@ keep_checkpoint_max: 10 preprocess: 'false' aclimdb_path: "/cache/data/aclImdb" glove_path: "/cache/data" -preprocess_path: "./preprocess" +preprocess_path: "/cache/train/preprocess" ckpt_path: './ckpt_lstm/' pre_trained: '' # None device_num: 1 @@ -72,6 +72,5 @@ enable_graph_kernel: 'Accelerate by graph kernel, default is true.' --- device_target: ['Ascend', 'GPU', 'CPU'] distribute: ['true', 'false'] -distribute: ['true', 'false'] enable_graph_kernel: ['true', 'false'] file_format: ['AIR', 'MINDIR'] diff --git a/model_zoo/official/nlp/lstm/eval.py b/model_zoo/official/nlp/lstm/eval.py index f0dd4a21430..b0866c70f3f 100644 --- a/model_zoo/official/nlp/lstm/eval.py +++ b/model_zoo/official/nlp/lstm/eval.py @@ -33,7 +33,6 @@ def modelarts_process(): def eval_lstm(): """ eval lstm """ print('\neval.py config: \n', config) - config.preprocess_path = os.path.join(config.glove_path, config.preprocess_path) context.set_context( mode=context.GRAPH_MODE, diff --git a/model_zoo/official/nlp/lstm/export.py b/model_zoo/official/nlp/lstm/export.py index 086e9b25f99..8cd6128d881 100644 --- a/model_zoo/official/nlp/lstm/export.py +++ b/model_zoo/official/nlp/lstm/export.py @@ -33,7 +33,6 @@ def modelarts_process(): @moxing_wrapper(pre_process=modelarts_process) def export_lstm(): """ export lstm """ - config.preprocess_path = os.path.join(config.glove_path, config.preprocess_path) context.set_context( mode=context.GRAPH_MODE, save_graphs=False, diff --git a/model_zoo/official/nlp/lstm/preprocess.py b/model_zoo/official/nlp/lstm/preprocess.py index 93828bce508..eca269da411 100644 --- a/model_zoo/official/nlp/lstm/preprocess.py +++ b/model_zoo/official/nlp/lstm/preprocess.py @@ -23,7 +23,6 @@ from src.model_utils.config import config if __name__ == '__main__': - config.preprocess_path = os.path.join(config.glove_path, config.preprocess_path) dataset = lstm_create_dataset(config.preprocess_path, config.batch_size, training=False) img_path = os.path.join(config.result_path, "00_data") os.makedirs(img_path) diff --git a/model_zoo/official/nlp/lstm/scripts/run_distribute_train_ascend.sh b/model_zoo/official/nlp/lstm/scripts/run_distribute_train_ascend.sh index 15c9d1b946f..d3182d154c3 100644 --- a/model_zoo/official/nlp/lstm/scripts/run_distribute_train_ascend.sh +++ b/model_zoo/official/nlp/lstm/scripts/run_distribute_train_ascend.sh @@ -48,6 +48,6 @@ do --distribute=true \ --device_num=$RANK_SIZE \ --device_id=$i --rank_id=$i \ - --preprocess=false \ + --preprocess=true \ --preprocess_path=./preprocess > log.txt 2>&1 & done diff --git a/model_zoo/official/nlp/lstm/scripts/run_eval_ascend.sh b/model_zoo/official/nlp/lstm/scripts/run_eval_ascend.sh index 8a26bcdc55a..c3719fb63a6 100644 --- a/model_zoo/official/nlp/lstm/scripts/run_eval_ascend.sh +++ b/model_zoo/official/nlp/lstm/scripts/run_eval_ascend.sh @@ -39,6 +39,6 @@ CONFIG_FILE="${BASE_PATH}/../../config_ascend.yaml" python ../../eval.py \ --config_path=$CONFIG_FILE \ --device_target="Ascend" \ - --preprocess=false \ + --preprocess=true \ --glove_path=$PREPROCESS_DIR \ --ckpt_file=$CKPT_FILE > log.txt 2>&1 & diff --git a/model_zoo/official/nlp/lstm/scripts/run_eval_cpu.sh b/model_zoo/official/nlp/lstm/scripts/run_eval_cpu.sh index 536a279649b..0b628c9f07f 100644 --- a/model_zoo/official/nlp/lstm/scripts/run_eval_cpu.sh +++ b/model_zoo/official/nlp/lstm/scripts/run_eval_cpu.sh @@ -37,6 +37,6 @@ python ../eval.py \ --device_target="CPU" \ --aclimdb_path=$ACLIMDB_DIR \ --glove_path=$GLOVE_DIR \ - --preprocess=false \ + --preprocess=true \ --preprocess_path=./preprocess \ --ckpt_file=$CKPT_FILE > log.txt 2>&1 & diff --git a/model_zoo/official/nlp/lstm/scripts/run_eval_gpu.sh b/model_zoo/official/nlp/lstm/scripts/run_eval_gpu.sh index 2ec5cf1a227..32f151bafe4 100644 --- a/model_zoo/official/nlp/lstm/scripts/run_eval_gpu.sh +++ b/model_zoo/official/nlp/lstm/scripts/run_eval_gpu.sh @@ -40,6 +40,6 @@ python ../eval.py \ --device_target="GPU" \ --aclimdb_path=$ACLIMDB_DIR \ --glove_path=$GLOVE_DIR \ - --preprocess=false \ + --preprocess=true \ --preprocess_path=./preprocess \ --ckpt_file=$CKPT_FILE > log.txt 2>&1 & diff --git a/model_zoo/official/nlp/lstm/scripts/run_train_ascend.sh b/model_zoo/official/nlp/lstm/scripts/run_train_ascend.sh index 4d28f9eb55d..e0b24ca4a97 100644 --- a/model_zoo/official/nlp/lstm/scripts/run_train_ascend.sh +++ b/model_zoo/official/nlp/lstm/scripts/run_train_ascend.sh @@ -40,5 +40,5 @@ python ../../train.py \ --device_target="Ascend" \ --aclimdb_path=$ACLIMDB_DIR \ --glove_path=$GLOVE_DIR \ - --preprocess=false \ + --preprocess=true \ --preprocess_path=./preprocess > log.txt 2>&1 & diff --git a/model_zoo/official/nlp/lstm/scripts/run_train_cpu.sh b/model_zoo/official/nlp/lstm/scripts/run_train_cpu.sh index e7f89023d69..b81901879cc 100644 --- a/model_zoo/official/nlp/lstm/scripts/run_train_cpu.sh +++ b/model_zoo/official/nlp/lstm/scripts/run_train_cpu.sh @@ -36,5 +36,5 @@ python ../train.py \ --device_target="CPU" \ --aclimdb_path=$ACLIMDB_DIR \ --glove_path=$GLOVE_DIR \ - --preprocess=false \ + --preprocess=true \ --preprocess_path=./preprocess > log.txt 2>&1 & diff --git a/model_zoo/official/nlp/lstm/scripts/run_train_gpu.sh b/model_zoo/official/nlp/lstm/scripts/run_train_gpu.sh index 5235db474f8..8dcdd59c216 100644 --- a/model_zoo/official/nlp/lstm/scripts/run_train_gpu.sh +++ b/model_zoo/official/nlp/lstm/scripts/run_train_gpu.sh @@ -39,5 +39,5 @@ python ../train.py \ --device_target="GPU" \ --aclimdb_path=$ACLIMDB_DIR \ --glove_path=$GLOVE_DIR \ - --preprocess=false \ + --preprocess=true \ --preprocess_path=./preprocess > log.txt 2>&1 & diff --git a/model_zoo/official/nlp/lstm/train.py b/model_zoo/official/nlp/lstm/train.py index de78d86b6f6..400591c18e3 100644 --- a/model_zoo/official/nlp/lstm/train.py +++ b/model_zoo/official/nlp/lstm/train.py @@ -39,7 +39,6 @@ def modelarts_pre_process(): def train_lstm(): """ train lstm """ print('\ntrain.py config: \n', config) - config.preprocess_path = os.path.join(config.glove_path, config.preprocess_path) _enable_graph_kernel = config.enable_graph_kernel == "true" and config.device_target == "GPU" context.set_context( @@ -60,6 +59,9 @@ def train_lstm(): device_num=device_num) if config.preprocess == "true": + import shutil + if os.path.exists(config.preprocess_path): + shutil.rmtree(config.preprocess_path) print("============== Starting Data Pre-processing ==============") convert_to_mindrecord(config.embed_size, config.aclimdb_path, config.preprocess_path, config.glove_path)