Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

clould

This commit is contained in:
huchunmei 2021-06-24 16:14:41 +08:00
parent 70152adcb3
commit b4d70a3b21
15 changed files with 14 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
model_zoo/official/cv/inceptionv3/scripts/run_distribute_train_gpu.sh
View File

@ -20,5 +20,5 @@ BASE_PATH=$(cd ./"`dirname $0`" || exit; pwd)
CONFIG_FILE="${BASE_PATH}/../default_config_gpu.yaml"
mpirun --allow-run-as-root -n 8 --output-filename log_output --merge-stderr-to-stdout \
python ./train.py --config_path=$CONFIG_FILE --is_distributed --platform 'GPU' \
python ./train.py --config_path=$CONFIG_FILE --is_distributed=True --platform 'GPU' \
--dataset_path $DATA_DIR --ckpt_path=$CKPT_PATH > train.log 2>&1 &

3
model_zoo/official/nlp/lstm/config_ascend.yaml
View File

@ -35,7 +35,7 @@ global_step: 0
preprocess: 'false'
aclimdb_path: "/cache/data/aclImdb"
glove_path: "/cache/data"
preprocess_path: "./preprocess"
preprocess_path: "/cache/train/preprocess"
ckpt_path: './ckpt_lstm/'
pre_trained: '' # None
device_num: 1
@ -77,6 +77,5 @@ enable_graph_kernel: 'Accelerate by graph kernel, default is true.'
---
device_target: ['Ascend', 'GPU', 'CPU']
distribute: ['true', 'false']
distribute: ['true', 'false']
enable_graph_kernel: ['true', 'false']
file_format: ['AIR', 'ONNX', 'MINDIR']

3
model_zoo/official/nlp/lstm/config_ascend_8p.yaml
View File

@ -36,7 +36,7 @@ global_step: 0
preprocess: 'false'
aclimdb_path: "/cache/data/aclImdb"
glove_path: "/cache/data"
preprocess_path: "./preprocess"
preprocess_path: "/cache/train/preprocess"
ckpt_path: './ckpt_lstm/'
pre_trained: '' # None
device_num: 8
@ -79,6 +79,5 @@ enable_graph_kernel: 'Accelerate by graph kernel, default is true.'
---
device_target: ['Ascend', 'GPU', 'CPU']
distribute: ['true', 'false']
distribute: ['true', 'false']
enable_graph_kernel: ['true', 'false']
file_format: ['AIR', 'ONNX', 'MINDIR']

3
model_zoo/official/nlp/lstm/default_config.yaml
View File

@ -30,7 +30,7 @@ keep_checkpoint_max: 10
preprocess: 'false'
aclimdb_path: "/cache/data/aclImdb"
glove_path: "/cache/data"
preprocess_path: "./preprocess"
preprocess_path: "/cache/train/preprocess"
ckpt_path: './ckpt_lstm/'
pre_trained: '' # None
device_num: 1
@ -72,6 +72,5 @@ enable_graph_kernel: 'Accelerate by graph kernel, default is true.'
---
device_target: ['Ascend', 'GPU', 'CPU']
distribute: ['true', 'false']
distribute: ['true', 'false']
enable_graph_kernel: ['true', 'false']
file_format: ['AIR', 'MINDIR']

1
model_zoo/official/nlp/lstm/eval.py
View File

@ -33,7 +33,6 @@ def modelarts_process():
def eval_lstm():
""" eval lstm """
print('\neval.py config: \n', config)
config.preprocess_path = os.path.join(config.glove_path, config.preprocess_path)
context.set_context(
mode=context.GRAPH_MODE,

1
model_zoo/official/nlp/lstm/export.py
View File

@ -33,7 +33,6 @@ def modelarts_process():
@moxing_wrapper(pre_process=modelarts_process)
def export_lstm():
""" export lstm """
config.preprocess_path = os.path.join(config.glove_path, config.preprocess_path)
context.set_context(
mode=context.GRAPH_MODE,
save_graphs=False,

1
model_zoo/official/nlp/lstm/preprocess.py
View File

@ -23,7 +23,6 @@ from src.model_utils.config import config
if __name__ == '__main__':
config.preprocess_path = os.path.join(config.glove_path, config.preprocess_path)
dataset = lstm_create_dataset(config.preprocess_path, config.batch_size, training=False)
img_path = os.path.join(config.result_path, "00_data")
os.makedirs(img_path)

2
model_zoo/official/nlp/lstm/scripts/run_distribute_train_ascend.sh
View File

@ -48,6 +48,6 @@ do
--distribute=true \
--device_num=$RANK_SIZE \
--device_id=$i --rank_id=$i \
--preprocess=false \
--preprocess=true \
--preprocess_path=./preprocess > log.txt 2>&1 &
done

2
model_zoo/official/nlp/lstm/scripts/run_eval_ascend.sh
View File

@ -39,6 +39,6 @@ CONFIG_FILE="${BASE_PATH}/../../config_ascend.yaml"
python ../../eval.py \
--config_path=$CONFIG_FILE \
--device_target="Ascend" \
--preprocess=false \
--preprocess=true \
--glove_path=$PREPROCESS_DIR \
--ckpt_file=$CKPT_FILE > log.txt 2>&1 &

2
model_zoo/official/nlp/lstm/scripts/run_eval_cpu.sh
View File

@ -37,6 +37,6 @@ python ../eval.py \
--device_target="CPU" \
--aclimdb_path=$ACLIMDB_DIR \
--glove_path=$GLOVE_DIR \
--preprocess=false \
--preprocess=true \
--preprocess_path=./preprocess \
--ckpt_file=$CKPT_FILE > log.txt 2>&1 &

2
model_zoo/official/nlp/lstm/scripts/run_eval_gpu.sh
View File

@ -40,6 +40,6 @@ python ../eval.py \
--device_target="GPU" \
--aclimdb_path=$ACLIMDB_DIR \
--glove_path=$GLOVE_DIR \
--preprocess=false \
--preprocess=true \
--preprocess_path=./preprocess \
--ckpt_file=$CKPT_FILE > log.txt 2>&1 &

2
model_zoo/official/nlp/lstm/scripts/run_train_ascend.sh
View File

@ -40,5 +40,5 @@ python ../../train.py \
--device_target="Ascend" \
--aclimdb_path=$ACLIMDB_DIR \
--glove_path=$GLOVE_DIR \
--preprocess=false \
--preprocess=true \
--preprocess_path=./preprocess > log.txt 2>&1 &

2
model_zoo/official/nlp/lstm/scripts/run_train_cpu.sh
View File

@ -36,5 +36,5 @@ python ../train.py \
--device_target="CPU" \
--aclimdb_path=$ACLIMDB_DIR \
--glove_path=$GLOVE_DIR \
--preprocess=false \
--preprocess=true \
--preprocess_path=./preprocess > log.txt 2>&1 &

2
model_zoo/official/nlp/lstm/scripts/run_train_gpu.sh
View File

@ -39,5 +39,5 @@ python ../train.py \
--device_target="GPU" \
--aclimdb_path=$ACLIMDB_DIR \
--glove_path=$GLOVE_DIR \
--preprocess=false \
--preprocess=true \
--preprocess_path=./preprocess > log.txt 2>&1 &

4
model_zoo/official/nlp/lstm/train.py
View File

@ -39,7 +39,6 @@ def modelarts_pre_process():
def train_lstm():
""" train lstm """
print('\ntrain.py config: \n', config)
config.preprocess_path = os.path.join(config.glove_path, config.preprocess_path)
_enable_graph_kernel = config.enable_graph_kernel == "true" and config.device_target == "GPU"
context.set_context(
@ -60,6 +59,9 @@ def train_lstm():
device_num=device_num)
if config.preprocess == "true":
import shutil
if os.path.exists(config.preprocess_path):
shutil.rmtree(config.preprocess_path)
print("============== Starting Data Pre-processing ==============")
convert_to_mindrecord(config.embed_size, config.aclimdb_path, config.preprocess_path, config.glove_path)