forked from mindspore-Ecosystem/mindspore
336 lines
15 KiB
Python
336 lines
15 KiB
Python
|
# Copyright 2020 Huawei Technologies Co., Ltd
|
||
|
|
#
|
||
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||
|
|
# you may not use this file except in compliance with the License.
|
||
|
|
# You may obtain a copy of the License at
|
||
|
|
#
|
||
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||
|
|
#
|
||
|
|
# Unless required by applicable law or agreed to in writing, software
|
||
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||
|
|
# See the License for the specific language governing permissions and
|
||
|
|
# limitations under the License.
|
||
|
|
# ============================================================================
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
import mindspore.nn as nn
|
||
|
|
from mindspore.common.api import ms_function
|
||
|
|
import numpy as np
|
||
|
|
import mindspore.context as context
|
||
|
|
from mindspore.common.initializer import initializer
|
||
|
|
from mindspore.ops import composite as C
|
||
|
|
from mindspore.ops import operations as P
|
||
|
|
from mindspore.common.tensor import Tensor
|
||
|
|
from mindspore.common.parameter import ParameterTuple, Parameter
|
||
|
|
|
||
|
|
context.set_context(device_target='CPU')
|
||
|
|
|
||
|
|
|
||
|
|
class LstmNet(nn.Cell):
|
||
|
|
def __init__(self, seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout):
|
||
|
|
super(LstmNet, self).__init__()
|
||
|
|
|
||
|
|
num_directions = 1
|
||
|
|
if bidirectional:
|
||
|
|
num_directions = 2
|
||
|
|
|
||
|
|
self.lstm = P.LSTM(input_size, hidden_size, num_layers, has_bias, bidirectional, dropout)
|
||
|
|
input_np = np.array([[[0.6755, -1.6607, 0.1367], [0.4276, -0.7850, -0.3758]],
|
||
|
|
[[-0.6424, -0.6095, 0.6639], [0.7918, 0.4147, -0.5089]],
|
||
|
|
[[-1.5612, 0.0120, -0.7289], [-0.6656, -0.6626, -0.5883]],
|
||
|
|
[[-0.9667, -0.6296, -0.7310], [0.1026, -0.6821, -0.4387]],
|
||
|
|
[[-0.4710, 0.6558, -0.3144], [-0.8449, -0.2184, -0.1806]]
|
||
|
|
]).astype(np.float32)
|
||
|
|
self.x = Parameter(initializer(Tensor(input_np), [seq_len, batch_size, input_size]), name='x')
|
||
|
|
|
||
|
|
self.h = Parameter(initializer(
|
||
|
|
Tensor(
|
||
|
|
np.array([0.1, 0.1, 0.1, 0.1]).reshape((num_layers * num_directions, batch_size, hidden_size)).astype(
|
||
|
|
np.float32)),
|
||
|
|
[num_layers * num_directions, batch_size, hidden_size]), name='h')
|
||
|
|
|
||
|
|
self.c = Parameter(initializer(
|
||
|
|
Tensor(
|
||
|
|
np.array([0.2, 0.2, 0.2, 0.2]).reshape((num_layers * num_directions, batch_size, hidden_size)).astype(
|
||
|
|
np.float32)),
|
||
|
|
[num_layers * num_directions, batch_size, hidden_size]), name='c')
|
||
|
|
|
||
|
|
wih = np.array([[3.4021e-01, -4.6622e-01, 4.5117e-01],
|
||
|
|
[-6.4257e-02, -2.4807e-01, 1.3550e-02], # i
|
||
|
|
[-3.2140e-01, 5.5578e-01, 6.3589e-01],
|
||
|
|
[1.6547e-01, -7.9030e-02, -2.0045e-01],
|
||
|
|
[-6.9863e-01, 5.9773e-01, -3.9062e-01],
|
||
|
|
[-3.0253e-01, -1.9464e-01, 7.0591e-01],
|
||
|
|
[-4.0835e-01, 3.6751e-01, 4.7989e-01],
|
||
|
|
[-5.6894e-01, -5.0359e-01, 4.7491e-01]]).astype(np.float32) # .reshape([1,-1])
|
||
|
|
whh = np.array([[-0.4820, -0.2350],
|
||
|
|
[-0.1195, 0.0519],
|
||
|
|
[0.2162, -0.1178],
|
||
|
|
[0.6237, 0.0711],
|
||
|
|
[0.4511, -0.3961],
|
||
|
|
[-0.5962, 0.0906],
|
||
|
|
[0.1867, -0.1225],
|
||
|
|
[0.1831, 0.0850]]).astype(np.float32) # .reshape([1,-1])
|
||
|
|
wih = wih.transpose((1, 0))
|
||
|
|
whh = whh.transpose((1, 0))
|
||
|
|
bih = np.zeros((1, 8)).astype(np.float32)
|
||
|
|
w_np = np.concatenate((wih, whh, bih), axis=0).reshape([-1, 1, 1])
|
||
|
|
self.w = Parameter(initializer(Tensor(w_np), w_np.shape), name='w')
|
||
|
|
|
||
|
|
@ms_function
|
||
|
|
def construct(self):
|
||
|
|
return self.lstm(self.x, self.h, self.c, self.w)
|
||
|
|
|
||
|
|
@pytest.mark.level0
|
||
|
|
@pytest.mark.platform_x86_cpu
|
||
|
|
@pytest.mark.env_onecard
|
||
|
|
def test_lstm():
|
||
|
|
seq_len = 5
|
||
|
|
batch_size = 2
|
||
|
|
input_size = 3
|
||
|
|
hidden_size = 2
|
||
|
|
num_layers = 1
|
||
|
|
has_bias = True
|
||
|
|
bidirectional = False
|
||
|
|
dropout = 0.0
|
||
|
|
num_directions = 1
|
||
|
|
if bidirectional:
|
||
|
|
num_directions = 2
|
||
|
|
net = LstmNet(seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout)
|
||
|
|
y, h, c, _, _ = net()
|
||
|
|
print(y)
|
||
|
|
print(c)
|
||
|
|
print(h)
|
||
|
|
expect_y = np.array([[[-0.16709016, 0.13125697],
|
||
|
|
[-0.08438572, -0.01969833]],
|
||
|
|
[[-0.2746155, 0.32764038],
|
||
|
|
[-0.06504016, -0.07770399]],
|
||
|
|
[[-0.00140004, 0.17706314],
|
||
|
|
[0.03244496, -0.10135599]],
|
||
|
|
[[0.08328028, 0.06437367],
|
||
|
|
[-0.04133911, -0.11072896]],
|
||
|
|
[[0.19004421, -0.02852732],
|
||
|
|
[0.09138509, -0.00344161]]]
|
||
|
|
)
|
||
|
|
error = np.ones([num_layers, batch_size, hidden_size]) * 1.0e-4
|
||
|
|
diff = y.asnumpy() - expect_y
|
||
|
|
assert np.all(diff < error)
|
||
|
|
assert np.all(-diff < error)
|
||
|
|
#
|
||
|
|
expect_h = np.array([[[0.19004421, -0.02852732],
|
||
|
|
[0.09138509, -0.00344161]]])
|
||
|
|
|
||
|
|
error = np.ones((num_layers * num_directions, batch_size, hidden_size)) * 1.0e-4
|
||
|
|
diff = h.asnumpy() - expect_h
|
||
|
|
assert np.all(diff < error)
|
||
|
|
assert np.all(-diff < error)
|
||
|
|
#
|
||
|
|
expect_c = np.array([[[0.34533143, -0.06313794],
|
||
|
|
[0.169008, -0.00555446]]])
|
||
|
|
error = np.ones((num_layers * num_directions, batch_size, hidden_size)) * 1.0e-4
|
||
|
|
diff = c.asnumpy() - expect_c
|
||
|
|
assert np.all(diff < error)
|
||
|
|
assert np.all(-diff < error)
|
||
|
|
|
||
|
|
|
||
|
|
class MultiLayerBiLstmNet(nn.Cell):
|
||
|
|
def __init__(self, seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout):
|
||
|
|
super(MultiLayerBiLstmNet, self).__init__()
|
||
|
|
|
||
|
|
num_directions = 1
|
||
|
|
if bidirectional:
|
||
|
|
num_directions = 2
|
||
|
|
|
||
|
|
self.lstm = nn.LSTM(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers, has_bias=has_bias,
|
||
|
|
bidirectional=bidirectional, dropout=dropout)
|
||
|
|
|
||
|
|
input_np = np.array([[[-0.1887, -0.4144, -0.0235, 0.7489, 0.7522, 0.5969, 0.3342, 1.2198, 0.6786, -0.9404],
|
||
|
|
[-0.8643, -1.6835, -2.4965, 2.8093, 0.1741, 0.2707, 0.7387, -0.0939, -1.7990, 0.4765]],
|
||
|
|
|
||
|
|
[[-0.5963, -1.2598, -0.7226, 1.1365, -1.7320, -0.7302, 0.1221, -0.2111, -1.6173, -0.0706],
|
||
|
|
[0.8964, 0.1737, -1.0077, -0.1389, 0.4889, 0.4391, 0.7911, 0.3614, -1.9533, -0.9936]],
|
||
|
|
|
||
|
|
[[0.3260, -1.3312, 0.0601, 1.0726, -1.6010, -1.8733, -1.5775, 1.1579, -0.8801, -0.5742],
|
||
|
|
[-2.2998, -0.6344, -0.5409, -0.9221, -0.6500, 0.1206, 1.5215, 0.7517, 1.3691, 2.0021]],
|
||
|
|
|
||
|
|
[[-0.1245, -0.3690, 2.1193, 1.3852, -0.1841, -0.8899, -0.3646, -0.8575, -0.3131, 0.2026],
|
||
|
|
[1.0218, -1.4331, 0.1744, 0.5442, -0.7808, 0.2527, 0.1566, 1.1484, -0.7766, -0.6747]],
|
||
|
|
|
||
|
|
[[-0.6752, 0.9906, -0.4973, 0.3471, -0.1202, -0.4213, 2.0213, 0.0441, 0.9016, 1.0365],
|
||
|
|
[1.2223, -1.3248, 0.1207, -0.8256, 0.1816, 0.7057, -0.3105, 0.5713, 0.2804,
|
||
|
|
-1.0685]]]).astype(np.float32)
|
||
|
|
|
||
|
|
self.x = Parameter(initializer(Tensor(input_np), [seq_len, batch_size, input_size]), name='x')
|
||
|
|
|
||
|
|
self.h0 = Parameter(initializer(
|
||
|
|
Tensor(np.ones((num_directions, batch_size, hidden_size)).astype(np.float32)),
|
||
|
|
[num_directions, batch_size, hidden_size]), name='h0')
|
||
|
|
self.c0 = Parameter(initializer(
|
||
|
|
Tensor(np.ones((num_directions, batch_size, hidden_size)).astype(np.float32)),
|
||
|
|
[num_directions, batch_size, hidden_size]), name='c0')
|
||
|
|
self.h1 = Parameter(initializer(
|
||
|
|
Tensor(np.ones((num_directions, batch_size, hidden_size)).astype(np.float32)),
|
||
|
|
[num_directions, batch_size, hidden_size]), name='h1')
|
||
|
|
self.c1 = Parameter(initializer(
|
||
|
|
Tensor(np.ones((num_directions, batch_size, hidden_size)).astype(np.float32)),
|
||
|
|
[num_directions, batch_size, hidden_size]), name='c1')
|
||
|
|
self.h = ParameterTuple((self.h0, self.h1))
|
||
|
|
self.c = ParameterTuple((self.c0, self.c1))
|
||
|
|
|
||
|
|
@ms_function
|
||
|
|
def construct(self):
|
||
|
|
return self.lstm(self.x, (self.h, self.c))
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.level0
|
||
|
|
@pytest.mark.platform_x86_cpu
|
||
|
|
@pytest.mark.env_onecard
|
||
|
|
def test_multi_layer_bilstm():
|
||
|
|
seq_len = 5
|
||
|
|
batch_size = 2
|
||
|
|
input_size = 10
|
||
|
|
hidden_size = 2
|
||
|
|
num_layers = 2
|
||
|
|
has_bias = True
|
||
|
|
bidirectional = True
|
||
|
|
dropout = 0.0
|
||
|
|
|
||
|
|
num_directions = 1
|
||
|
|
if bidirectional:
|
||
|
|
num_directions = 2
|
||
|
|
|
||
|
|
net = MultiLayerBiLstmNet(seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional,
|
||
|
|
dropout)
|
||
|
|
y, h, c, _, _ = net()
|
||
|
|
print(y)
|
||
|
|
print(h)
|
||
|
|
print(c)
|
||
|
|
|
||
|
|
|
||
|
|
class Grad(nn.Cell):
|
||
|
|
def __init__(self, network):
|
||
|
|
super(Grad, self).__init__()
|
||
|
|
self.network = network
|
||
|
|
self.weights = ParameterTuple(network.trainable_params())
|
||
|
|
self.grad = C.GradOperation('grad',
|
||
|
|
get_by_list=True,
|
||
|
|
sens_param=True)
|
||
|
|
|
||
|
|
@ms_function
|
||
|
|
def construct(self, output_grad):
|
||
|
|
weights = self.weights
|
||
|
|
grads = self.grad(self.network, weights)(output_grad)
|
||
|
|
return grads
|
||
|
|
|
||
|
|
|
||
|
|
class Net(nn.Cell):
|
||
|
|
def __init__(self, seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout):
|
||
|
|
super(Net, self).__init__()
|
||
|
|
|
||
|
|
num_directions = 1
|
||
|
|
if bidirectional:
|
||
|
|
num_directions = 2
|
||
|
|
input_np = np.array([[[-0.5907, 1.0557, 1.7283, 0.6706, -1.2550, -0.5298, -0.2290, -0.6735, 0.8555, 1.4836],
|
||
|
|
[-1.7070, -0.5347, -0.9105, -0.2598, 0.0588, 1.5496, 1.0757, 0.3760, -1.2020, -0.2868]],
|
||
|
|
|
||
|
|
[[0.0151, 0.2126, 0.8090, -0.5292, -2.5590, 0.4279, -0.3081, -1.4706, -0.0498, 1.2301],
|
||
|
|
[0.4165, -0.5391, -0.0996, 0.1928, -0.4909, -0.1255, 0.4444, -1.3687, 1.3096, 0.6553]],
|
||
|
|
|
||
|
|
[[-0.7802, -0.2083, -0.6388, 1.3757, 0.4293, 0.5363, 0.3202, -0.6687, -1.3864, -0.2953],
|
||
|
|
[1.0799, -0.7204, 0.1130, -0.5857, -0.4855, -1.1068, 1.0126, 0.8716, 1.5460, -0.7392]],
|
||
|
|
|
||
|
|
[[2.2645, -0.6586, -0.2227, 1.4290, -0.5006, -1.6576, -0.1793, 0.5319, 0.1360, 0.2707],
|
||
|
|
[-0.4071, 0.1575, 1.4199, -0.9156, 0.1855, 0.4947, 1.0460, -0.6365, 0.1191, -0.6374]],
|
||
|
|
|
||
|
|
[[0.2468, 1.0815, -0.4893, 0.0664, 0.6405, -2.2967, 0.7612, 0.8759, 0.5685, -1.0999],
|
||
|
|
[-0.7272, -1.7750, -0.1164, -0.7159, 0.0061, -0.7839, -1.8329, 0.3434, -0.5634,
|
||
|
|
0.5384]]]).astype(np.float32)
|
||
|
|
|
||
|
|
self.x = Parameter(initializer(Tensor(input_np), [seq_len, batch_size, input_size]), name='x')
|
||
|
|
|
||
|
|
self.h0 = Parameter(initializer(
|
||
|
|
Tensor(np.ones((num_directions, batch_size, hidden_size)).astype(np.float32)),
|
||
|
|
[num_directions, batch_size, hidden_size]), name='h0')
|
||
|
|
|
||
|
|
self.c0 = Parameter(initializer(
|
||
|
|
Tensor(np.ones((num_directions, batch_size, hidden_size)).astype(np.float32)),
|
||
|
|
[num_directions, batch_size, hidden_size]), name='c0')
|
||
|
|
|
||
|
|
wih_l0 = np.array([[0.2300, 0.6668, 0.4703, 0.0425, 0.0464, 0.6825, 0.2249, -0.4315, -0.2449, 0.2964],
|
||
|
|
[-0.2811, -0.3444, 0.2557, -0.5137, -0.5518, 0.1652, -0.6720, 0.1066, 0.3586, 0.6299],
|
||
|
|
[0.5728, -0.1784, 0.5661, 0.4012, 0.3856, -0.1899, 0.3102, 0.3717, -0.5651, 0.1952],
|
||
|
|
[0.1026, -0.0527, 0.1198, -0.3080, 0.2292, 0.5757, -0.3567, -0.2731, -0.0586, -0.2849],
|
||
|
|
[0.2194, -0.1622, 0.3219, -0.3008, -0.3713, -0.3034, -0.2385, 0.0412, -0.5205, 0.0280],
|
||
|
|
[-0.5499, -0.0733, -0.5236, -0.6753, -0.7045, -0.1839, -0.1037, -0.5026, -0.4055, -0.3416],
|
||
|
|
[0.1573, -0.1301, -0.2882, -0.3464, 0.6643, 0.1980, -0.6804, 0.5359, 0.5996, 0.0124],
|
||
|
|
[-0.6436, 0.0587, -0.6520, -0.0471, 0.1667, 0.6042, 0.5752, -0.6296, -0.2976,
|
||
|
|
-0.3757]]).astype(np.float32).reshape([1, -1])
|
||
|
|
|
||
|
|
whh_l0 = np.array([[0.3358, 0.2790],
|
||
|
|
[-0.5355, 0.0989],
|
||
|
|
[-0.1402, 0.5120],
|
||
|
|
[0.1335, 0.1653],
|
||
|
|
[0.3533, -0.3531],
|
||
|
|
[0.4166, -0.4420],
|
||
|
|
[-0.5454, -0.1720],
|
||
|
|
[0.0041, -0.0799]]).astype(np.float32).reshape([1, -1])
|
||
|
|
|
||
|
|
bih_l0 = np.array([0.5518, 0.1083, 0.4829, 0.0607, -0.1770, -0.6944, 0.3059, 0.5354]).astype(
|
||
|
|
np.float32).reshape([1, -1])
|
||
|
|
bhh_l0 = np.array([0.5025, -0.1261, -0.5405, 0.3220, -0.3441, 0.6488, -0.0284, -0.2334]).astype(
|
||
|
|
np.float32).reshape([1, -1])
|
||
|
|
|
||
|
|
w0_np = np.concatenate(
|
||
|
|
(wih_l0, whh_l0, bih_l0 + bhh_l0),
|
||
|
|
axis=1).reshape([-1, 1, 1])
|
||
|
|
self.w0 = Parameter(initializer(Tensor(w0_np), w0_np.shape), name='w0')
|
||
|
|
self.lstm = P.LSTM(input_size=input_size, hidden_size=hidden_size, num_layers=num_layers,
|
||
|
|
has_bias=has_bias, bidirectional=bidirectional, dropout=dropout)
|
||
|
|
|
||
|
|
@ms_function
|
||
|
|
def construct(self):
|
||
|
|
return self.lstm(self.x, self.h0, self.c0, self.w0)[0]
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.level0
|
||
|
|
@pytest.mark.platform_x86_cpu
|
||
|
|
@pytest.mark.env_onecard
|
||
|
|
def test_grad():
|
||
|
|
seq_len = 5
|
||
|
|
batch_size = 2
|
||
|
|
input_size = 10
|
||
|
|
hidden_size = 2
|
||
|
|
num_layers = 1
|
||
|
|
has_bias = True
|
||
|
|
bidirectional = False
|
||
|
|
dropout = 0.0
|
||
|
|
num_directions = 1
|
||
|
|
if bidirectional:
|
||
|
|
num_directions = 2
|
||
|
|
net = Grad(Net(seq_len, batch_size, input_size, hidden_size, num_layers, has_bias, bidirectional, dropout))
|
||
|
|
dy = np.array([[[-3.5471e-01, 7.0540e-01],
|
||
|
|
[2.7161e-01, 1.0865e+00]],
|
||
|
|
|
||
|
|
[[-4.2431e-01, 1.4955e+00],
|
||
|
|
[-4.0418e-01, -2.3282e-01]],
|
||
|
|
|
||
|
|
[[-1.3654e+00, 1.9251e+00],
|
||
|
|
[-4.6481e-01, 1.3138e+00]],
|
||
|
|
|
||
|
|
[[1.2914e+00, -2.3753e-01],
|
||
|
|
[5.3589e-01, -1.0981e-01]],
|
||
|
|
|
||
|
|
[[-1.6032e+00, -1.8818e-01],
|
||
|
|
[1.0065e-01, 9.2045e-01]]]).astype(np.float32)
|
||
|
|
dx, dhx, dcx, dw = net(Tensor(dy))
|
||
|
|
print(dx)
|
||
|
|
print(dhx)
|
||
|
|
print(dcx)
|
||
|
|
print(dw)
|
||
|
|
|
||
|
|
# test_multi_layer_bilstm()
|
||
|
|
# test_lstm()
|
||
|
|
# tf_lstm_test()
|
||
|
|
# test_grad()
|