clould

model_zoo/official/cv/inceptionv3/scripts/run_distribute_train_gpu.sh

@@ -20,5 +20,5 @@ BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd)
 CONFIG_FILE="${BASE_PATH}/../default_config_gpu.yaml"
 
 mpirun --allow-run-as-root -n 8 --output-filename log_output --merge-stderr-to-stdout \
-  python ./train.py --config_path=$CONFIG_FILE --is_distributed --platform 'GPU' \
+  python ./train.py --config_path=$CONFIG_FILE --is_distributed=True --platform 'GPU' \
   --dataset_path $DATA_DIR --ckpt_path=$CKPT_PATH > train.log 2>&1 &

model_zoo/official/nlp/lstm/config_ascend.yaml

@@ -35,7 +35,7 @@ global_step: 0
 preprocess: 'false'
 aclimdb_path: "/cache/data/aclImdb"
 glove_path: "/cache/data"
-preprocess_path: "./preprocess"
+preprocess_path: "/cache/train/preprocess"
 ckpt_path: './ckpt_lstm/'
 pre_trained: '' # None
 device_num: 1
@@ -77,6 +77,5 @@ enable_graph_kernel: 'Accelerate by graph kernel, default is true.'
 ---
 device_target: ['Ascend', 'GPU', 'CPU']
 distribute: ['true', 'false']
-distribute: ['true', 'false']
 enable_graph_kernel: ['true', 'false']
 file_format: ['AIR', 'ONNX', 'MINDIR']

model_zoo/official/nlp/lstm/config_ascend_8p.yaml

@@ -36,7 +36,7 @@ global_step: 0
 preprocess: 'false'
 aclimdb_path: "/cache/data/aclImdb"
 glove_path: "/cache/data"
-preprocess_path: "./preprocess"
+preprocess_path: "/cache/train/preprocess"
 ckpt_path: './ckpt_lstm/'
 pre_trained: '' # None
 device_num: 8
@@ -79,6 +79,5 @@ enable_graph_kernel: 'Accelerate by graph kernel, default is true.'
 ---
 device_target: ['Ascend', 'GPU', 'CPU']
 distribute: ['true', 'false']
-distribute: ['true', 'false']
 enable_graph_kernel: ['true', 'false']
 file_format: ['AIR', 'ONNX', 'MINDIR']

model_zoo/official/nlp/lstm/default_config.yaml

@@ -30,7 +30,7 @@ keep_checkpoint_max: 10
 preprocess: 'false'
 aclimdb_path: "/cache/data/aclImdb"
 glove_path: "/cache/data"
-preprocess_path: "./preprocess"
+preprocess_path: "/cache/train/preprocess"
 ckpt_path: './ckpt_lstm/'
 pre_trained: '' # None
 device_num: 1
@@ -72,6 +72,5 @@ enable_graph_kernel: 'Accelerate by graph kernel, default is true.'
 ---
 device_target: ['Ascend', 'GPU', 'CPU']
 distribute: ['true', 'false']
-distribute: ['true', 'false']
 enable_graph_kernel: ['true', 'false']
 file_format: ['AIR', 'MINDIR']

model_zoo/official/nlp/lstm/eval.py

@@ -33,7 +33,6 @@ def modelarts_process():
 def eval_lstm():
     """ eval lstm """
     print('\neval.py config: \n', config)
-    config.preprocess_path = os.path.join(config.glove_path, config.preprocess_path)
 
     context.set_context(
         mode=context.GRAPH_MODE,

model_zoo/official/nlp/lstm/export.py

@@ -33,7 +33,6 @@ def modelarts_process():
 @moxing_wrapper(pre_process=modelarts_process)
 def export_lstm():
     """ export lstm """
-    config.preprocess_path = os.path.join(config.glove_path, config.preprocess_path)
     context.set_context(
         mode=context.GRAPH_MODE,
         save_graphs=False,

model_zoo/official/nlp/lstm/preprocess.py

@@ -23,7 +23,6 @@ from src.model_utils.config import config
 
 
 if __name__ == '__main__':
-    config.preprocess_path = os.path.join(config.glove_path, config.preprocess_path)
     dataset = lstm_create_dataset(config.preprocess_path, config.batch_size, training=False)
     img_path = os.path.join(config.result_path, "00_data")
     os.makedirs(img_path)

model_zoo/official/nlp/lstm/scripts/run_distribute_train_ascend.sh

@@ -48,6 +48,6 @@ do
         --distribute=true \
         --device_num=$RANK_SIZE \
         --device_id=$i --rank_id=$i \
-        --preprocess=false  \
+        --preprocess=true  \
         --preprocess_path=./preprocess > log.txt 2>&1 &
 done

model_zoo/official/nlp/lstm/scripts/run_eval_ascend.sh

@@ -39,6 +39,6 @@ CONFIG_FILE="${BASE_PATH}/../../config_ascend.yaml"
 python ../../eval.py  \
     --config_path=$CONFIG_FILE \
     --device_target="Ascend" \
-    --preprocess=false  \
+    --preprocess=true  \
     --glove_path=$PREPROCESS_DIR \
     --ckpt_file=$CKPT_FILE > log.txt 2>&1 &

model_zoo/official/nlp/lstm/scripts/run_eval_cpu.sh

@@ -37,6 +37,6 @@ python ../eval.py  \
     --device_target="CPU" \
     --aclimdb_path=$ACLIMDB_DIR \
     --glove_path=$GLOVE_DIR \
-    --preprocess=false  \
+    --preprocess=true  \
     --preprocess_path=./preprocess \
     --ckpt_file=$CKPT_FILE > log.txt 2>&1 &

model_zoo/official/nlp/lstm/scripts/run_eval_gpu.sh

@@ -40,6 +40,6 @@ python ../eval.py  \
     --device_target="GPU" \
     --aclimdb_path=$ACLIMDB_DIR \
     --glove_path=$GLOVE_DIR \
-    --preprocess=false  \
+    --preprocess=true  \
     --preprocess_path=./preprocess \
     --ckpt_file=$CKPT_FILE > log.txt 2>&1 &

model_zoo/official/nlp/lstm/scripts/run_train_ascend.sh

@@ -40,5 +40,5 @@ python ../../train.py  \
     --device_target="Ascend" \
     --aclimdb_path=$ACLIMDB_DIR \
     --glove_path=$GLOVE_DIR \
-    --preprocess=false  \
+    --preprocess=true  \
     --preprocess_path=./preprocess > log.txt 2>&1 &

model_zoo/official/nlp/lstm/scripts/run_train_cpu.sh

@@ -36,5 +36,5 @@ python ../train.py  \
     --device_target="CPU" \
     --aclimdb_path=$ACLIMDB_DIR \
     --glove_path=$GLOVE_DIR \
-    --preprocess=false  \
+    --preprocess=true  \
     --preprocess_path=./preprocess > log.txt 2>&1 &

model_zoo/official/nlp/lstm/scripts/run_train_gpu.sh

@@ -39,5 +39,5 @@ python ../train.py  \
     --device_target="GPU" \
     --aclimdb_path=$ACLIMDB_DIR \
     --glove_path=$GLOVE_DIR \
-    --preprocess=false  \
+    --preprocess=true  \
     --preprocess_path=./preprocess > log.txt 2>&1 &

model_zoo/official/nlp/lstm/train.py

@@ -39,7 +39,6 @@ def modelarts_pre_process():
 def train_lstm():
     """ train lstm """
     print('\ntrain.py config: \n', config)
-    config.preprocess_path = os.path.join(config.glove_path, config.preprocess_path)
 
     _enable_graph_kernel = config.enable_graph_kernel == "true" and config.device_target == "GPU"
     context.set_context(
@@ -60,6 +59,9 @@ def train_lstm():
             device_num=device_num)
 
     if config.preprocess == "true":
+        import shutil
+        if os.path.exists(config.preprocess_path):
+            shutil.rmtree(config.preprocess_path)
         print("============== Starting Data Pre-processing ==============")
         convert_to_mindrecord(config.embed_size, config.aclimdb_path, config.preprocess_path, config.glove_path)