p60975123
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Delete unused interface in graph_kernels.py

This commit is contained in:
looop5 2020-12-09 14:51:46 +08:00
parent 2799b6d35f
commit 4d8205cd93
8 changed files with 293 additions and 1245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
mindspore/nn/graph_kernels/__init__.py → mindspore/nn/_graph_kernels/__init__.py
View File

@ -18,13 +18,6 @@ GraphKernel.
GraphKernel provides a unified style to express graph and kernel for user.
It breaks the boundary between graph and kernel and provides more opportunities to do compile optimization.
"""
from .graph_kernels import MaximumGrad, MinimumGrad, AbsGrad, ApplyMomentum, BiasAdd, EqualCount, \
ReduceMean, ReLU, SoftmaxCrossEntropyWithLogits, LayerNorm, LayerNormXBackprop, \
LayerNormBetaGammaBackprop, LogSoftmax, Tanh, TanhGrad, Gelu, Softmax, BiasAddGrad, \
LambUpdateWithLR, LambNextMV
from .graph_kernels import LambUpdateWithLR, LambNextMV
__all__ = ['MaximumGrad', 'MinimumGrad', 'AbsGrad', 'ApplyMomentum', 'BiasAdd', 'EqualCount',
'ReduceMean', 'ReLU', 'SoftmaxCrossEntropyWithLogits', 'LayerNorm',
'LayerNormXBackprop', 'LayerNormBetaGammaBackprop', 'LogSoftmax', 'Tanh', 'TanhGrad',
'Gelu', 'Softmax', 'BiasAddGrad', 'LambUpdateWithLR', 'LambNextMV'
]
__all__ = ['LambUpdateWithLR', 'LambNextMV']

259
mindspore/nn/_graph_kernels/graph_kernels.py Normal file
View File

@ -0,0 +1,259 @@
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
"""
Graph kernels. They are composites of basic primitives and can be compiled into
a fused kernel automatically when context.set_context(enable_graph_kernel=True).
"""
from ...ops import operations as P
from ...ops.primitive import PrimitiveWithInfer, prim_attr_register
from ...ops.composite import multitype_ops as C
from ..cell import GraphKernel
class InplaceAssign(PrimitiveWithInfer):
"""
Inplace assign `Parameter` with a value.
This primitive can only be used in graph kernel.
Inputs:
- **variable** (Parameter) - The `Parameter`.
- **value** (Tensor) - The value to be assigned.
- **depend** (Tensor) - The dependent tensor to keep this op connected in graph.
Outputs:
Tensor, has the same type as original `variable`.
Examples:
>>> class MyClass(GraphKernel):
... def __init__(self):
... super(MyClass, self).__init__()
... self.mul = P.Mul()
... self.fake_output_assign = InplaceAssign()
... self.fake_output_assign.add_prim_attr("fake_output", True)
...
... def construct(self, i0, i1):
... mul_res = self.mul(i0, i1)
... # mul_res is a fake output and parameter i0 will be updated.
... mul_res = self.fake_output_assign(i0, mul_res, mul_res)
... return mul_res
"""
@prim_attr_register
def __init__(self):
super(InplaceAssign, self).__init__("InplaceAssign")
self.init_prim_io_names(inputs=['x', 'y', 'z'], outputs=['output'])
def infer_shape(self, x, y, z):
return z
def infer_dtype(self, x, y, z):
return z
def get_bprop(self):
def bprop(x, y, z, out, dout):
return (x, C.zeros_like(y), dout)
return bprop
class LambUpdateWithLR(GraphKernel):
r"""
Part of Lamb optimizer.
.. math::
s_1 = select(i_1 \gt y_g, select(i_0 \gt y_g, \frac{i_1}{i_2}, se), se)
i_5 = i_5 - max(min(s_1, y_m), y_g) \times i_3 \times i_4
Inputs:
- **input0** (Tensor) - The first tensor to be computed.
- **input1** (Tensor) - The second tensor to be computed.
- **input2** (Tensor) - The third tensor to be computed.
- **input3** (Tensor) - The fourth tensor to be computed.
- **input4** (Tensor) - The fifth tensor to be computed.
- **input5** (Tensor) - The sixth tensor to be computed. It will be updated by result.
- **greater_y** (Tensor) - The seventh tensor to be computed.
- **select_e** (Tensor) - The eighth tensor to be computed.
- **minimum_y** (Tensor) - The ninth tensor to be computed.
Outputs:
A fake output tensor.
Examples:
>>> import numpy as np
>>> import mindspore.context as context
>>> from mindspore.common import dtype as mstype
>>> from mindspore.common.tensor import Tensor
>>> from mindspore.common.parameter import Parameter
>>> from mindspore.nn.cell import Cell
>>> class Net(Cell):
... def __init__(self, i5):
... super(Net, self).__init__()
... self.i5 = Parameter(i5, name='i5')
... self.lamb_update = LambUpdateWithLR()
...
... def construct(self, i0, i1, i2, i3, i4, i6, i7, i8):
... return self.lamb_update(i0, i1, i2, i3, i4, self.i5, i6, i7, i8)
>>> shape = [1, 16]
>>> oshape = [1]
>>> i0 = Tensor(np.random.normal(0, 1, oshape).astype(np.float32))
>>> i1 = Tensor(np.random.normal(0, 1, oshape).astype(np.float32))
>>> i2 = Tensor(np.random.normal(0, 1, oshape).astype(np.float32))
>>> i3 = Tensor(np.random.normal(0, 1, oshape).astype(np.float32))
>>> i4 = Tensor(np.random.normal(0, 1, shape).astype(np.float32))
>>> i5 = Tensor(np.random.normal(0, 1, shape).astype(np.float32))
>>> i6 = Tensor(np.random.normal(0, 1, oshape).astype(np.float32))
>>> i7 = Tensor(np.random.normal(0, 1, oshape).astype(np.float32))
>>> i8 = Tensor(np.random.normal(0, 1, oshape).astype(np.float32))
>>> context.set_context(mode=context.GRAPH_MODE, enable_graph_kernel=True)
>>> net = Net(i5)
>>> _ = net(i0, i1, i2, i3, i4, i6, i7, i8)
>>> output = (net.i5)
"""
def __init__(self):
super(LambUpdateWithLR, self).__init__()
self.greater = P.Greater()
self.select = P.Select()
self.div = P.RealDiv()
self.min = P.Minimum()
self.max = P.Maximum()
self.mul = P.Mul()
self.sub = P.Sub()
self.fake_output_assign = InplaceAssign()
self.fake_output_assign.add_prim_attr("fake_output", True)
def construct(self, input0, input1, input2, input3, input4, input5, greater_y, select_e, minimum_y):
greater0 = self.greater(input0, greater_y)
greater1 = self.greater(input1, greater_y)
real_div0 = self.div(input1, input2)
select0 = self.select(greater0, real_div0, select_e)
select1 = self.select(greater1, select0, select_e)
min0 = self.min(select1, minimum_y)
max0 = self.max(min0, greater_y)
mul0 = self.mul(max0, input3)
mul1 = self.mul(mul0, input4)
sub0 = self.sub(input5, mul1)
sub0 = self.fake_output_assign(input5, sub0, sub0)
return sub0
class LambNextMV(GraphKernel):
r"""
Part of Lamb optimizer.
.. math::
rd_0 = \frac{i_8 \times i_5 + i_9 \times i_4}{i6}
rd_1 = \frac{x_0 \times i_2 + x_1 \times i_1}{i3}
y_2 = \frac{rd_0}{\sqrt{rd_1 + x3}} + x_2 \times i_7
y_3 = \frac{rd_0}{\sqrt{rd_1} + x3}
i5 = i_8 \times i_5 + i_9 \times i_4
i2 = x_0 \times i_2 + x_1 \times i_1
Inputs:
- **inputs1** (Tensor) - The first input tensor to be computed.
- **inputs2** (Tensor) - The second input tensor to be computed. It will be updated by result.
- **inputs3** (Tensor) - The third input tensor to be computed.
- **inputs4** (Tensor) - The fourth input tensor to be computed.
- **inputs5** (Tensor) - The fifth input tensor to be computed. It will be updated by result.
- **inputs6** (Tensor) - The sixth input tensor to be computed.
- **inputs7** (Tensor) - The seventh input tensor to be computed.
- **inputs8** (Tensor) - The eighth input tensor to be computed.
- **inputs9** (Tensor) - The ninth input tensor to be computed.
- **inputsx0** (Tensor) - The tenth input tensor to be computed.
- **inputsx1** (Tensor) - The eleventh input tensor to be computed.
- **inputsx2** (Tensor) - The twelfth input tensor to be computed.
- **inputsx3** (Tensor) - The thirteenth input tensor to be computed.
Outputs:
Tuple of 2 Tensors.
- **add3** (Tensor) - the shape is the same as the one after broadcasting, and the data type is
the one with higher precision or higher digits among the inputs.
- **realdiv4** (Tensor) - the shape is the same as the one after broadcasting, and the data type is
the one with higher precision or higher digits among the inputs.
Examples:
>>> import numpy as np
>>> import mindspore.context as context
>>> from mindspore.common import dtype as mstype
>>> from mindspore.common.tensor import Tensor
>>> from mindspore.common.parameter import Parameter
>>> from mindspore.nn.cell import Cell
>>> class Net(Cell):
... def __init__(self, i1, i4):
... super(Net, self).__init__()
... self.i1 = Parameter(i1, name='i1')
... self.i4 = Parameter(i4, name='i4')
... self.lamb_next = LambNextMV()
...
... def construct(self, i0, i2, i3, i5, i6, i7, i8, i9, i10, i11, i12):
... i0_ = i0 + i2
... return self.lamb_next(i0_, self.i1, i2, i3, self.i4, i5, i6, i7, i8, i9, i10, i11, i12)
>>> shape = [1, 16]
>>> i0 = Tensor(np.abs(np.random.normal(0, 1, shape)).astype(np.float32))
>>> i1 = Tensor(np.abs(np.random.normal(0, 1, shape)).astype(np.float32))
>>> i2 = Tensor(np.abs(np.random.normal(0, 1, shape)).astype(np.float32))
>>> i3 = Tensor(np.random.normal(0, 1, shape).astype(np.float32))
>>> i4 = Tensor(np.random.normal(0, 1, shape).astype(np.float32))
>>> i5 = Tensor(np.abs(np.random.normal(0, 1, shape)).astype(np.float32))
>>> i6 = Tensor(np.random.normal(0, 1, shape).astype(np.float32))
>>> i7 = Tensor(np.random.normal(0, 1, shape).astype(np.float32))
>>> i8 = Tensor(np.random.normal(0, 1, shape).astype(np.float32))
>>> i9 = Tensor(np.abs(np.random.normal(0, 1, shape)).astype(np.float32))
>>> i10 = Tensor(np.abs(np.random.normal(0, 1, shape)).astype(np.float32))
>>> i11 = Tensor(np.random.normal(0, 1, shape).astype(np.float32))
>>> i12 = Tensor(np.ones(shape).astype(np.float32) * 1e-6)
>>> context.set_context(mode=context.GRAPH_MODE, enable_graph_kernel=True)
>>> net = Net(i1, i4)
>>> (o0, o1) = net(i0, i2, i3, i5, i6, i7, i8, i9, i10, i11, i12)
>>> output = (o0, net.i4, net.i1, o1)
"""
def __init__(self):
super(LambNextMV, self).__init__()
self.mul = P.Mul()
self.add = P.TensorAdd()
self.div = P.RealDiv()
self.sqrt = P.Sqrt()
self.rsqrt = P.Rsqrt()
self.fake_output_assign_1 = InplaceAssign()
self.fake_output_assign_1.add_prim_attr("fake_output", False)
self.fake_output_assign_2 = InplaceAssign()
self.fake_output_assign_2.add_prim_attr("fake_output", False)
def construct(self, input1, input2, input3, input4, input5, input6, input7,
input8, input9, inputx0, inputx1, inputx2, inputx3):
mul3 = self.mul(inputx1, input1)
mul2 = self.mul(inputx0, input2)
add1 = self.add(mul2, mul3)
realdiv1 = self.div(add1, input3)
add2 = self.add(realdiv1, inputx3)
sqrt0 = self.rsqrt(add2)
sqrt1 = self.sqrt(realdiv1)
add4 = self.add(sqrt1, inputx3)
mul1 = self.mul(input9, input4)
mul0 = self.mul(input8, input5)
add0 = self.add(mul0, mul1)
realdiv0 = self.div(add0, input6)
realdiv2 = self.mul(realdiv0, sqrt0)
realdiv4 = self.div(realdiv0, add4)
mul4 = self.mul(inputx2, input7)
add3 = self.add(realdiv2, mul4)
add3 = self.fake_output_assign_1(input5, add0, add3)
add3 = self.fake_output_assign_2(input2, add1, add3)
return add3, realdiv4

1208
mindspore/nn/graph_kernels/graph_kernels.py
View File

File diff suppressed because it is too large Load Diff

2
mindspore/nn/optim/lamb.py
View File

@ -26,7 +26,7 @@ from mindspore._checkparam import Validator as validator
from mindspore._checkparam import Rel
from .optimizer import Optimizer
from .. import layer
from .. import graph_kernels as G
from .. import _graph_kernels as G
num_one = Tensor(np.ones([1]), mstype.float32)

2
mindspore/ops/_selected_grad_ops.py
View File

@ -17,7 +17,7 @@
from mindspore.ops.op_selector import new_ops_selector
op_selector = new_ops_selector(
"mindspore.ops.operations._grad_ops", "mindspore.nn.graph_kernels")
"mindspore.ops.operations._grad_ops", "mindspore.nn._graph_kernels")
@op_selector

6
mindspore/ops/_selected_ops.py
View File

@ -17,11 +17,11 @@
from mindspore.ops.op_selector import new_ops_selector
op_selector = new_ops_selector(
"mindspore.ops.operations", "mindspore.nn.graph_kernels")
"mindspore.ops.operations", "mindspore.nn._graph_kernels")
opt_selector = new_ops_selector(
"mindspore.nn.optim", "mindspore.nn.graph_kernels")
"mindspore.nn.optim", "mindspore.nn._graph_kernels")
nn_selector = new_ops_selector(
"mindspore.nn", "mindspore.nn.graph_kernels")
"mindspore.nn", "mindspore.nn._graph_kernels")
@nn_selector

48
tests/st/ops/graph_kernel/test_fuse.py
View File

@ -19,7 +19,7 @@ import mindspore.context as context
from mindspore import Tensor
from mindspore.nn import Cell
import mindspore.ops.operations as P
from mindspore.nn.graph_kernels import ReLU
from mindspore.ops.operations import _grad_ops as G
class Net(Cell):
@ -28,41 +28,45 @@ class Net(Cell):
self.add = P.TensorAdd()
self.sub = P.Sub()
self.mul = P.Mul()
self.relu = ReLU()
self.sqrt_grad = G.SqrtGrad()
def construct(self, x, y):
def construct(self, x, y, z):
sub_res = self.sub(x, y)
mul_res = self.mul(sub_res, x)
relu_res = self.relu(mul_res)
square_res = P.Square()(relu_res)
add_res = self.add(relu_res, square_res)
sqrt_grad_res = self.sqrt_grad(mul_res, z)
square_res = P.Square()(sqrt_grad_res)
add_res = self.add(sqrt_grad_res, square_res)
add1_res = self.add(add_res, add_res)
return self.add(add1_res, add1_res)
def test_basic():
input_x = np.random.normal(0, 1, [2, 3, 4, 3]).astype(np.float32)
input_y = np.random.normal(0, 1, [2, 3, 4, 3]).astype(np.float32)
sub_res = input_x - input_y
mul_res = sub_res * input_x
relu_res = np.maximum(mul_res, 0)
square_res = np.square(relu_res)
add_res = relu_res + square_res
add1_res = add_res + add_res
expect = add1_res + add1_res
def get_output(i0, i1, i2, enable_graph_kernel=False):
if enable_graph_kernel:
context.set_context(enable_graph_kernel=True)
net = Net()
result = net(Tensor(input_x), Tensor(input_y))
output = net(i0, i1, i2)
return output
res = np.allclose(expect, result.asnumpy(), rtol=1.e-4, atol=1.e-7, equal_nan=True)
assert res
def test_basic():
i0 = Tensor(np.random.normal(0, 1, [2, 3, 4, 3]).astype(np.float32))
i1 = Tensor(np.random.normal(0, 1, [2, 3, 4, 3]).astype(np.float32))
i2 = Tensor(np.random.normal(0, 1, [2, 3, 4, 3]).astype(np.float32))
expect = get_output(i0, i1, i2, False)
output = get_output(i0, i1, i2, True)
expect_np = expect.asnumpy().copy()
output_np = output.asnumpy().copy()
assert np.allclose(expect_np, output_np, 1.e-4, 1.e-7)
@pytest.mark.level0
@pytest.mark.platform_x86_gpu_training
@pytest.mark.env_onecard
def test_basic_gpu():
context.set_context(mode=context.GRAPH_MODE, enable_graph_kernel=True, device_target="GPU")
context.set_context(mode=context.GRAPH_MODE, device_target="GPU")
test_basic()
@ -71,5 +75,5 @@ def test_basic_gpu():
@pytest.mark.platform_x86_ascend_training
@pytest.mark.env_onecard
def test_basic_ascend():
context.set_context(mode=context.GRAPH_MODE, enable_graph_kernel=True, device_target="Ascend")
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
test_basic()

2
tests/st/ops/graph_kernel/test_lamb.py
View File

@ -18,7 +18,7 @@ import numpy as np
import mindspore.context as context
from mindspore import Tensor, Parameter
from mindspore.nn import Cell
from mindspore.nn.graph_kernels import LambUpdateWithLR, LambNextMV
from mindspore.nn._graph_kernels import LambUpdateWithLR, LambNextMV
class LambNet(Cell):