p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!10118 raise textrcnn precision when using DynamicRNN 

From: @chenmai1102
Reviewed-by: @guoqi1024,@oacjiewen
Signed-off-by: @guoqi1024
This commit is contained in:
mindspore-ci-bot 2020-12-18 09:58:29 +08:00 committed by Gitee
parent d988e13fb5 d36abebe54
commit 4ce11a930b
5 changed files with 57 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
model_zoo/official/nlp/textrcnn/readme.md
View File

@ -114,13 +114,16 @@ Parameters for both training and evaluation can be set in config.py
```python
'num_epochs': 10, # total training epochs
'batch_size': 64, # training batch size
'cell': 'lstm', # the RNN architecture, can be 'vanilla', 'gru' and 'lstm'.
'cell': 'gru', # the RNN architecture, can be 'vanilla', 'gru' and 'lstm'.
'opt': 'adam', # the optimizer strategy, can be 'adam' or 'momentum'
'ckpt_folder_path': './ckpt', # the path to save the checkpoints
'preprocess_path': './preprocess', # the directory to save the processed data
'preprocess' : 'false', # whethere to preprocess the data
'data_path': './data/', # the path to store the splited data
'lr': 1e-3, # the training learning rate
'lstm_base_lr': 3e-3, # the training learning rate when using lstm as RNN cell
'lstm_decay_rate': 0.9, # lr decay rate when using lstm as RNN cell
'lstm_decay_epoch': 1, # lr decay epoch when using lstm as RNN cell
'emb_path': './word2vec', # the directory to save the embedding file
'embed_size': 300, # the dimension of the word embedding
'save_checkpoint_steps': 149, # per step to save the checkpoint
@ -137,7 +140,7 @@ Parameters for both training and evaluation can be set in config.py
| Dataset | Sentence polarity dataset v1.0 | Sentence polarity dataset v1.0 |
| batch_size | 64 | 64 |
| Accuracy | 0.78 | 0.78 |
| Speed | 78ms/step | 89ms/step |
| Speed | 25ms/step | 77ms/step |
## [ModelZoo Homepage](#contents)

5
model_zoo/official/nlp/textrcnn/src/config.py
View File

@ -23,13 +23,16 @@ textrcnn_cfg = edict({
'neg_dir': 'data/rt-polaritydata/rt-polarity.neg',
'num_epochs': 10,
'batch_size': 64,
'cell': 'lstm',
'cell': 'gru',
'opt': 'adam',
'ckpt_folder_path': './ckpt',
'preprocess_path': './preprocess',
'preprocess': 'false',
'data_path': './data/',
'lr': 1e-3,
'lstm_base_lr': 3e-3,
'lstm_decay_rate': 0.9,
'lstm_decay_epoch': 1,
'emb_path': './word2vec',
'embed_size': 300,
'save_checkpoint_steps': 149,

12
model_zoo/official/nlp/textrcnn/src/textrcnn.py
View File

@ -45,16 +45,16 @@ class textrcnn(nn.Cell):
self.lstm = P.DynamicRNN(forget_bias=0.0)
self.w1_fw = Parameter(
np.random.uniform(-k, k, (self.embed_size + self.num_hiddens, 4 * self.num_hiddens)).astype(
np.float16), name="w1_fw")
self.b1_fw = Parameter(np.random.uniform(-k, k, (4 * self.num_hiddens)).astype(np.float16),
np.float32), name="w1_fw")
self.b1_fw = Parameter(np.random.uniform(-k, k, (4 * self.num_hiddens)).astype(np.float32),
name="b1_fw")
self.w1_bw = Parameter(
np.random.uniform(-k, k, (self.embed_size + self.num_hiddens, 4 * self.num_hiddens)).astype(
np.float16), name="w1_bw")
self.b1_bw = Parameter(np.random.uniform(-k, k, (4 * self.num_hiddens)).astype(np.float16),
np.float32), name="w1_bw")
self.b1_bw = Parameter(np.random.uniform(-k, k, (4 * self.num_hiddens)).astype(np.float32),
name="b1_bw")
self.h1 = Tensor(np.zeros(shape=(1, self.batch_size, self.num_hiddens)).astype(np.float16))
self.c1 = Tensor(np.zeros(shape=(1, self.batch_size, self.num_hiddens)).astype(np.float16))
self.h1 = Tensor(np.zeros(shape=(1, self.batch_size, self.num_hiddens)).astype(np.float32))
self.c1 = Tensor(np.zeros(shape=(1, self.batch_size, self.num_hiddens)).astype(np.float32))
if cell == "vanilla":
self.rnnW_fw = nn.Dense(self.num_hiddens, self.num_hiddens)

29
model_zoo/official/nlp/textrcnn/src/utils.py Normal file
View File

@ -0,0 +1,29 @@
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""training utils"""
from mindspore import dtype as mstype
from mindspore.nn.dynamic_lr import exponential_decay_lr
from mindspore import Tensor
def get_lr(cfg, dataset_size):
if cfg.cell == "lstm":
lr = exponential_decay_lr(cfg.lstm_base_lr, cfg.lstm_decay_rate, dataset_size * cfg.num_epochs,
dataset_size,
cfg.lstm_decay_epoch)
lr_ret = Tensor(lr, mstype.float32)
else:
lr_ret = cfg.lr
return lr_ret

19
model_zoo/official/nlp/textrcnn/train.py
View File

@ -29,6 +29,7 @@ from src.config import textrcnn_cfg as cfg
from src.dataset import create_dataset
from src.dataset import convert_to_mindrecord
from src.textrcnn import textrcnn
from src.utils import get_lr
set_seed(1)
@ -50,25 +51,31 @@ if __name__ == '__main__':
os.mkdir(cfg.preprocess_path)
convert_to_mindrecord(cfg.embed_size, cfg.data_path, cfg.preprocess_path, cfg.emb_path)
if cfg.cell == "vanilla":
print("============ Precision is lower than expected when using vanilla RNN architecture ===========")
embedding_table = np.loadtxt(os.path.join(cfg.preprocess_path, "weight.txt")).astype(np.float32)
network = textrcnn(weight=Tensor(embedding_table), vocab_size=embedding_table.shape[0], \
cell=cfg.cell, batch_size=cfg.batch_size)
cell=cfg.cell, batch_size=cfg.batch_size)
ds_train = create_dataset(cfg.preprocess_path, cfg.batch_size, cfg.num_epochs, True)
step_size = ds_train.get_dataset_size()
loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
lr = get_lr(cfg, step_size)
if cfg.opt == "adam":
opt = nn.Adam(params=network.trainable_params(), learning_rate=cfg.lr)
opt = nn.Adam(params=network.trainable_params(), learning_rate=lr)
elif cfg.opt == "momentum":
opt = nn.Momentum(network.trainable_params(), cfg.lr, cfg.momentum)
opt = nn.Momentum(network.trainable_params(), lr, cfg.momentum)
loss_cb = LossMonitor()
model = Model(network, loss, opt, {'acc': Accuracy()}, amp_level="O3")
print("============== Starting Training ==============")
ds_train = create_dataset(cfg.preprocess_path, cfg.batch_size, cfg.num_epochs, True)
config_ck = CheckpointConfig(save_checkpoint_steps=cfg.save_checkpoint_steps, \
keep_checkpoint_max=cfg.keep_checkpoint_max)
keep_checkpoint_max=cfg.keep_checkpoint_max)
ckpoint_cb = ModelCheckpoint(prefix=cfg.cell, directory=cfg.ckpt_folder_path, config=config_ck)
model.train(cfg.num_epochs, ds_train, callbacks=[ckpoint_cb, loss_cb])
print("train success")