82 lines
3.2 KiB
Python
82 lines
3.2 KiB
Python
# Copyright 2022 Huawei Technologies Co., Ltd
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
# ============================================================================
|
|
|
|
import numpy as np
|
|
import mindspore.nn as nn
|
|
import mindspore.common.dtype as mstype
|
|
from mindspore.nn import TrainOneStepCell, WithLossCell
|
|
from mindspore import Tensor
|
|
from mindspore.nn import Dense, ReLU
|
|
from mindspore.ops import operations as P
|
|
|
|
|
|
class WeightDecaySchdule(nn.Cell):
|
|
def __init__(self):
|
|
super(WeightDecaySchdule, self).__init__()
|
|
self.weight_decay_list = Tensor([0.001, 0.001, 0.1], mstype.float32)
|
|
|
|
def construct(self, global_step):
|
|
return self.weight_decay_list[global_step]
|
|
|
|
|
|
class Net(nn.Cell):
|
|
def __init__(self):
|
|
super(Net, self).__init__()
|
|
self.batch_size = 1
|
|
self.reshape = P.Reshape()
|
|
self.relu = ReLU()
|
|
weight1 = Tensor(np.ones([10, 16]).astype(np.float32) * 0.01)
|
|
weight2 = Tensor(np.ones([10, 10]).astype(np.float32) * 0.02)
|
|
weight3 = Tensor(np.ones([10, 10]).astype(np.float32) * 0.03)
|
|
bias1 = Tensor(np.zeros(10).astype(np.float32))
|
|
bias2 = Tensor(np.ones(10).astype(np.float32))
|
|
bias3 = Tensor(np.ones(10).astype(np.float32))
|
|
self.fc1 = Dense(16, 10, weight_init=weight1, bias_init=bias1)
|
|
self.fc2 = Dense(10, 10, weight_init=weight2, bias_init=bias2)
|
|
self.fc3 = Dense(10, 10, weight_init=weight3, bias_init=bias3)
|
|
|
|
def construct(self, input_x):
|
|
output = self.reshape(input_x, (self.batch_size, -1))
|
|
output = self.fc1(output)
|
|
output = self.relu(output)
|
|
output = self.fc2(output)
|
|
output = self.relu(output)
|
|
output = self.fc3(output)
|
|
return output
|
|
|
|
|
|
def dynamic_weight_decay_cmp(net1, net2, optimizer1, optimizer2):
|
|
epoch = 3
|
|
criterion = nn.SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
|
|
net_with_criterion1 = WithLossCell(net1, criterion)
|
|
net_with_criterion2 = WithLossCell(net2, criterion)
|
|
train_network1 = TrainOneStepCell(net_with_criterion1, optimizer1)
|
|
train_network2 = TrainOneStepCell(net_with_criterion2, optimizer2)
|
|
train_network1.set_train()
|
|
train_network2.set_train()
|
|
|
|
for _ in range(epoch):
|
|
data = Tensor(np.arange(0, 16).reshape(1, 1, 4, 4).astype(np.float32) * 0.01)
|
|
label = Tensor(np.array([0]).astype(np.int32))
|
|
loss1 = train_network1(data, label)
|
|
loss2 = train_network2(data, label)
|
|
assert abs(loss1.asnumpy() - loss2.asnumpy()) < 1.e-8
|
|
|
|
data = Tensor(np.arange(0, 16).reshape(1, 1, 4, 4).astype(np.float32) * 0.01)
|
|
label = Tensor(np.array([0]).astype(np.int32))
|
|
loss1 = net_with_criterion1(data, label)
|
|
loss2 = net_with_criterion2(data, label)
|
|
assert abs(loss1.asnumpy() - loss2.asnumpy()) > 1.e-8
|