From 4a5d115a6651d0a22436035609099948bb3d3d72 Mon Sep 17 00:00:00 2001 From: Yi Huaijie Date: Sat, 29 Aug 2020 16:47:43 +0800 Subject: [PATCH] add get_seed() and set_seed() --- mindspore/common/__init__.py | 4 +- mindspore/common/initializer.py | 11 +++- mindspore/common/seed.py | 54 +++++++++++++++++++ .../nn/probability/distribution/bernoulli.py | 4 +- .../probability/distribution/categorical.py | 6 +-- .../probability/distribution/distribution.py | 7 ++- .../probability/distribution/exponential.py | 4 +- .../nn/probability/distribution/geometric.py | 4 +- .../nn/probability/distribution/normal.py | 4 +- .../distribution/transformed_distribution.py | 2 +- .../nn/probability/distribution/uniform.py | 4 +- mindspore/ops/composite/__init__.py | 3 +- mindspore/ops/composite/random_ops.py | 29 +++------- .../test_compoite_random_ops/test_gamma.py | 3 +- .../test_compoite_random_ops/test_normal.py | 3 +- .../test_compoite_random_ops/test_poisson.py | 3 +- .../test_compoite_random_ops/test_uniform.py | 3 +- .../parallel/test_initializer_weight_slice.py | 38 +++++++++++++ 18 files changed, 139 insertions(+), 47 deletions(-) create mode 100644 mindspore/common/seed.py diff --git a/mindspore/common/__init__.py b/mindspore/common/__init__.py index e3f8396c4d..2ede99a603 100644 --- a/mindspore/common/__init__.py +++ b/mindspore/common/__init__.py @@ -18,12 +18,14 @@ from .api import ms_function from .dtype import * from .parameter import Parameter, ParameterTuple from .tensor import MetaTensor, Tensor, RowTensor, SparseTensor +from .seed import set_seed, get_seed __all__ = [ "MetaTensor", "Tensor", "RowTensor", "SparseTensor", # tensor 'ms_function', # api 'Parameter', 'ParameterTuple', # parameter - "dtype" + "dtype", + "set_seed", "get_seed" # random seed ] __all__.extend(dtype.__all__) diff --git a/mindspore/common/initializer.py b/mindspore/common/initializer.py index 546d1e99b1..cf430c9a9c 100644 --- a/mindspore/common/initializer.py +++ b/mindspore/common/initializer.py @@ -23,6 +23,7 @@ from mindspore import log as logger from . import dtype as mstype from .tensor import Tensor +from .seed import get_seed from .._c_expression import random_normal _INITIALIZER_ALIAS = dict() @@ -71,7 +72,7 @@ class Initializer: Args: slice_index (int): Slice index of a parameter's slices. - Used when initialize a slice of a parameter, it guarantee that + Used when initialize a slice of the parameter, it guarantee that devices use the same slice can generate the same tensor. shape (list[int]): Shape of the slice, used when initialize a slice of the parameter. """ @@ -86,11 +87,17 @@ class Initializer: logger.error(msg) raise ValueError(msg) - if slice_index is not None: + global_seed = get_seed() + need_set_seed = ((slice_index is not None) and (global_seed is None)) + seed_saved = np.random.get_state()[1][0] + if need_set_seed: np.random.seed(slice_index) self.__call__(arr) + if need_set_seed: + np.random.seed(seed_saved) return Tensor(arr, dtype=self.dtype) + def _register(*aliases): """Return the alias register.""" def alias_reg(cls): diff --git a/mindspore/common/seed.py b/mindspore/common/seed.py new file mode 100644 index 0000000000..cb32d2ce9a --- /dev/null +++ b/mindspore/common/seed.py @@ -0,0 +1,54 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Provide random seed api.""" +import numpy as np + +# set global RNG seed +_GLOBAL_SEED = None + +def set_seed(seed): + """ + Set global random seed. + + Note: + The global seed is used by numpy.random, mindspore.common.Initializer, mindspore.ops.composite.random_ops and + mindspore.nn.probability.distribution. + If global seed is not set, these packages will use their own default seed independently, numpy.random and + mindspore.common.Initializer will choose a random seed, mindspore.ops.composite.random_ops and + mindspore.nn.probability.distribution will use zero. + Seed set by numpy.random.seed() only used by numpy.random, while seed set by this API will also used by + numpy.random, so just set all seed by this API is recommended. + + Args: + seed (int): The seed to be set. + + Raises: + ValueError: If seed is invalid (< 0). + TypeError: If seed isn't a int. + """ + if not isinstance(seed, int): + raise TypeError("The seed must be type of int.") + if seed < 0: + raise ValueError("The seed must be greater or equal to 0.") + np.random.seed(seed) + global _GLOBAL_SEED + _GLOBAL_SEED = seed + + +def get_seed(): + """ + Get global random seed. + """ + return _GLOBAL_SEED diff --git a/mindspore/nn/probability/distribution/bernoulli.py b/mindspore/nn/probability/distribution/bernoulli.py index 7fce4b7802..b3591cd360 100644 --- a/mindspore/nn/probability/distribution/bernoulli.py +++ b/mindspore/nn/probability/distribution/bernoulli.py @@ -27,7 +27,7 @@ class Bernoulli(Distribution): Args: probs (float, list, numpy.ndarray, Tensor, Parameter): probability of 1 as outcome. - seed (int): seed to use in sampling. Default: 0. + seed (int): seed to use in sampling. Global seed is used if it is None. Default: None. dtype (mindspore.dtype): type of the distribution. Default: mstype.int32. name (str): name of the distribution. Default: Bernoulli. @@ -91,7 +91,7 @@ class Bernoulli(Distribution): def __init__(self, probs=None, - seed=0, + seed=None, dtype=mstype.int32, name="Bernoulli"): """ diff --git a/mindspore/nn/probability/distribution/categorical.py b/mindspore/nn/probability/distribution/categorical.py index 88541db0e3..9e809b461e 100644 --- a/mindspore/nn/probability/distribution/categorical.py +++ b/mindspore/nn/probability/distribution/categorical.py @@ -27,7 +27,7 @@ class Categorical(Distribution): Args: probs (Tensor, list, numpy.ndarray, Parameter, float): event probabilities. logits (Tensor, list, numpy.ndarray, Parameter, float): event log-odds. - seed (int): seed to use in sampling. Default: 0. + seed (int): seed to use in sampling. Global seed is used if it is None. Default: None. dtype (mstype.int32): type of the distribution. Default: mstype.int32. name (str): name of the distribution. Default: Categorical. @@ -67,7 +67,7 @@ class Categorical(Distribution): def __init__(self, probs=None, logits=None, - seed=0, + seed=None, dtype=mstype.int32, name="Categorical"): param = dict(locals()) @@ -83,7 +83,7 @@ class Categorical(Distribution): self.reshape = P.Reshape() self.div = P.RealDiv() self.size = P.Size() - self.mutinomial = P.Multinomial(seed=seed) + self.mutinomial = P.Multinomial(seed=self.seed) self.cast = P.Cast() self.expandim = P.ExpandDims() self.gather = P.GatherNd() diff --git a/mindspore/nn/probability/distribution/distribution.py b/mindspore/nn/probability/distribution/distribution.py index 943b022057..ccf66fa432 100644 --- a/mindspore/nn/probability/distribution/distribution.py +++ b/mindspore/nn/probability/distribution/distribution.py @@ -17,6 +17,7 @@ from mindspore import context from mindspore.nn.cell import Cell from mindspore._checkparam import Validator as validator from mindspore._checkparam import Rel +from mindspore.common import get_seed from ._utils.utils import calc_broadcast_shape_from_param, check_scalar_from_param, cast_type_for_device from ._utils.utils import CheckTuple, CheckTensor @@ -26,7 +27,7 @@ class Distribution(Cell): Base class for all mathematical distributions. Args: - seed (int): random seed used in sampling. + seed (int): random seed used in sampling. Global seed is used if it is None. Default: None. dtype (mindspore.dtype): the type of the event samples. Default: subclass dtype. name (str): Python str name prefixed to Ops created by this class. Default: subclass name. param (dict): parameters used to initialize the distribution. @@ -56,6 +57,10 @@ class Distribution(Cell): Constructor of distribution class. """ super(Distribution, self).__init__() + if seed is None: + seed = get_seed() + if seed is None: + seed = 0 validator.check_value_type('name', name, [str], type(self).__name__) validator.check_integer('seed', seed, 0, Rel.GE, name) diff --git a/mindspore/nn/probability/distribution/exponential.py b/mindspore/nn/probability/distribution/exponential.py index 3ecc8250bb..7262e2779c 100644 --- a/mindspore/nn/probability/distribution/exponential.py +++ b/mindspore/nn/probability/distribution/exponential.py @@ -28,7 +28,7 @@ class Exponential(Distribution): Args: rate (float, list, numpy.ndarray, Tensor, Parameter): inverse scale. - seed (int): seed to use in sampling. Default: 0. + seed (int): seed to use in sampling. Global seed is used if it is None. Default: None. dtype (mindspore.dtype): type of the distribution. Default: mstype.float32. name (str): name of the distribution. Default: Exponential. @@ -92,7 +92,7 @@ class Exponential(Distribution): def __init__(self, rate=None, - seed=0, + seed=None, dtype=mstype.float32, name="Exponential"): """ diff --git a/mindspore/nn/probability/distribution/geometric.py b/mindspore/nn/probability/distribution/geometric.py index f3d3eb015f..47beff931b 100644 --- a/mindspore/nn/probability/distribution/geometric.py +++ b/mindspore/nn/probability/distribution/geometric.py @@ -30,7 +30,7 @@ class Geometric(Distribution): Args: probs (float, list, numpy.ndarray, Tensor, Parameter): probability of success. - seed (int): seed to use in sampling. Default: 0. + seed (int): seed to use in sampling. Global seed is used if it is None. Default: None. dtype (mindspore.dtype): type of the distribution. Default: mstype.int32. name (str): name of the distribution. Default: Geometric. @@ -94,7 +94,7 @@ class Geometric(Distribution): def __init__(self, probs=None, - seed=0, + seed=None, dtype=mstype.int32, name="Geometric"): """ diff --git a/mindspore/nn/probability/distribution/normal.py b/mindspore/nn/probability/distribution/normal.py index 014537560a..132eb53481 100644 --- a/mindspore/nn/probability/distribution/normal.py +++ b/mindspore/nn/probability/distribution/normal.py @@ -29,7 +29,7 @@ class Normal(Distribution): Args: mean (int, float, list, numpy.ndarray, Tensor, Parameter): mean of the Normal distribution. sd (int, float, list, numpy.ndarray, Tensor, Parameter): stddev of the Normal distribution. - seed (int): seed to use in sampling. Default: 0. + seed (int): seed to use in sampling. Global seed is used if it is None. Default: None. dtype (mindspore.dtype): type of the distribution. Default: mstype.float32. name (str): name of the distribution. Default: Normal. @@ -94,7 +94,7 @@ class Normal(Distribution): def __init__(self, mean=None, sd=None, - seed=0, + seed=None, dtype=mstype.float32, name="Normal"): """ diff --git a/mindspore/nn/probability/distribution/transformed_distribution.py b/mindspore/nn/probability/distribution/transformed_distribution.py index 5c2f91933c..9a9dca3671 100644 --- a/mindspore/nn/probability/distribution/transformed_distribution.py +++ b/mindspore/nn/probability/distribution/transformed_distribution.py @@ -62,7 +62,7 @@ class TransformedDistribution(Distribution): bijector, distribution, dtype, - seed=0, + seed=None, name="transformed_distribution"): """ Constructor of transformed_distribution class. diff --git a/mindspore/nn/probability/distribution/uniform.py b/mindspore/nn/probability/distribution/uniform.py index 160dc3ad2d..7a43f7e108 100644 --- a/mindspore/nn/probability/distribution/uniform.py +++ b/mindspore/nn/probability/distribution/uniform.py @@ -28,7 +28,7 @@ class Uniform(Distribution): Args: low (int, float, list, numpy.ndarray, Tensor, Parameter): lower bound of the distribution. high (int, float, list, numpy.ndarray, Tensor, Parameter): upper bound of the distribution. - seed (int): seed to use in sampling. Default: 0. + seed (int): seed to use in sampling. Global seed is used if it is None. Default: None. dtype (mindspore.dtype): type of the distribution. Default: mstype.float32. name (str): name of the distribution. Default: Uniform. @@ -93,7 +93,7 @@ class Uniform(Distribution): def __init__(self, low=None, high=None, - seed=0, + seed=None, dtype=mstype.float32, name="Uniform"): """ diff --git a/mindspore/ops/composite/__init__.py b/mindspore/ops/composite/__init__.py index 1df90830f8..5f38630ac6 100644 --- a/mindspore/ops/composite/__init__.py +++ b/mindspore/ops/composite/__init__.py @@ -26,7 +26,7 @@ from .clip_ops import clip_by_value from .multitype_ops.add_impl import hyper_add from .multitype_ops.ones_like_impl import ones_like from .multitype_ops.zeros_like_impl import zeros_like -from .random_ops import set_seed, normal, uniform, gamma, poisson, multinomial +from .random_ops import normal, uniform, gamma, poisson, multinomial __all__ = [ @@ -41,7 +41,6 @@ __all__ = [ 'zeros_like', 'ones_like', 'zip_operation', - 'set_seed', 'normal', 'uniform', 'gamma', diff --git a/mindspore/ops/composite/random_ops.py b/mindspore/ops/composite/random_ops.py index c4de6ab3cc..e760e9d328 100644 --- a/mindspore/ops/composite/random_ops.py +++ b/mindspore/ops/composite/random_ops.py @@ -20,34 +20,14 @@ from .. import functional as F from ..primitive import constexpr from .multitype_ops import _constexpr_utils as const_utils from ...common import dtype as mstype - -# set graph-level RNG seed -_GRAPH_SEED = 0 - -@constexpr -def set_seed(seed): - """ - Set the graph-level seed. - Graph-level seed is used as a global variable, that can be used in different ops in case op-level seed is not set. - If op-level seed is 0, use graph-level seed; if op-level seed is also 0, the system would generate a - random seed. - - Args: - seed(Int): the graph-level seed value that to be set. Must be non-negative. - - Examples: - >>> C.set_seed(10) - """ - const_utils.check_non_negative("seed", seed, "set_seed") - global _GRAPH_SEED - _GRAPH_SEED = seed +from ...common import get_seed as get_global_seed @constexpr def get_seed(): """ Get the graph-level seed. Graph-level seed is used as a global variable, that can be used in different ops in case op-level seed is not set. - If op-level seed is 0, use graph-level seed; if op-level seed is also 0, the system would generate a + If op-level seed is 0, use graph-level seed; if graph-level seed is also 0, the system would generate a random seed. Returns: @@ -56,7 +36,10 @@ def get_seed(): Examples: >>> C.get_seed() """ - return _GRAPH_SEED + global_seed = get_global_seed() + if global_seed is None: + return 0 + return global_seed def normal(shape, mean, stddev, seed=0): """ diff --git a/tests/st/ops/ascend/test_compoite_random_ops/test_gamma.py b/tests/st/ops/ascend/test_compoite_random_ops/test_gamma.py index e762aedc21..96c8ce3a6c 100644 --- a/tests/st/ops/ascend/test_compoite_random_ops/test_gamma.py +++ b/tests/st/ops/ascend/test_compoite_random_ops/test_gamma.py @@ -19,6 +19,7 @@ import mindspore.nn as nn from mindspore import Tensor from mindspore.common import dtype as mstype from mindspore.ops import composite as C +from mindspore.common import set_seed context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") @@ -30,7 +31,7 @@ class Net(nn.Cell): self.seed = seed def construct(self, alpha, beta): - C.set_seed(20) + set_seed(20) return C.gamma(self.shape, alpha, beta, self.seed) diff --git a/tests/st/ops/ascend/test_compoite_random_ops/test_normal.py b/tests/st/ops/ascend/test_compoite_random_ops/test_normal.py index 6c6e07b584..10cca4768d 100644 --- a/tests/st/ops/ascend/test_compoite_random_ops/test_normal.py +++ b/tests/st/ops/ascend/test_compoite_random_ops/test_normal.py @@ -19,6 +19,7 @@ import mindspore.nn as nn from mindspore import Tensor from mindspore.common import dtype as mstype from mindspore.ops import composite as C +from mindspore.common import set_seed context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") @@ -30,7 +31,7 @@ class Net(nn.Cell): self.seed = seed def construct(self, mean, stddev): - C.set_seed(20) + set_seed(20) return C.normal(self.shape, mean, stddev, self.seed) diff --git a/tests/st/ops/ascend/test_compoite_random_ops/test_poisson.py b/tests/st/ops/ascend/test_compoite_random_ops/test_poisson.py index caa0a1f642..df1e1f49ef 100644 --- a/tests/st/ops/ascend/test_compoite_random_ops/test_poisson.py +++ b/tests/st/ops/ascend/test_compoite_random_ops/test_poisson.py @@ -19,6 +19,7 @@ import mindspore.nn as nn from mindspore import Tensor from mindspore.common import dtype as mstype from mindspore.ops import composite as C +from mindspore.common import set_seed context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") @@ -30,7 +31,7 @@ class Net(nn.Cell): self.seed = seed def construct(self, mean): - C.set_seed(20) + set_seed(20) return C.poisson(self.shape, mean, self.seed) diff --git a/tests/st/ops/ascend/test_compoite_random_ops/test_uniform.py b/tests/st/ops/ascend/test_compoite_random_ops/test_uniform.py index 1e4f827b46..f2f72e0cee 100644 --- a/tests/st/ops/ascend/test_compoite_random_ops/test_uniform.py +++ b/tests/st/ops/ascend/test_compoite_random_ops/test_uniform.py @@ -19,6 +19,7 @@ import mindspore.nn as nn from mindspore import Tensor from mindspore.common import dtype as mstype from mindspore.ops import composite as C +from mindspore.common import set_seed context.set_context(mode=context.GRAPH_MODE, device_target="Ascend") @@ -30,7 +31,7 @@ class Net(nn.Cell): self.seed = seed def construct(self, minval, maxval): - C.set_seed(20) + set_seed(20) return C.uniform(self.shape, minval, maxval, self.seed) diff --git a/tests/ut/python/parallel/test_initializer_weight_slice.py b/tests/ut/python/parallel/test_initializer_weight_slice.py index 098a55c5a2..049bb65fda 100644 --- a/tests/ut/python/parallel/test_initializer_weight_slice.py +++ b/tests/ut/python/parallel/test_initializer_weight_slice.py @@ -21,6 +21,7 @@ from mindspore import Tensor, Parameter import mindspore as ms import mindspore.common.api as me from mindspore.common.initializer import initializer +from mindspore.common import set_seed from hccl_test.manage.api import Hccl class Net(nn.Cell): @@ -112,5 +113,42 @@ def test_wrong_order_set_parallel_mode_without_initializer(): net.set_auto_parallel() exe.compile(net, x, auto_parallel_mode=True, phase='train') +def test_check_initializer_weight_slice_seed(init_name="Uniform"): + def get_slice(rank): + set_seed(1) + hccl = Hccl() + rank_save = hccl.rank_id + hccl.rank_id = rank + context.reset_auto_parallel_context() + context.set_auto_parallel_context(device_num=8, global_rank=0) + context.set_auto_parallel_context(parallel_mode="semi_auto_parallel") + strategy1 = ((2, 1), (4, 1)) + strategy2 = ((2, 4),) + context.set_context(mode=context.GRAPH_MODE) + exe = me._executor + + x = Tensor(np.ones([32, 32]), dtype=ms.float32) + weight = initializer(init_name, [64, 32], ms.float32) + net = Net(strategy1, strategy2, weight) + net.set_auto_parallel() + exe.compile(net, x, auto_parallel_mode=True, phase='train') + hccl.rank_id = rank_save + return net.parameters_dict()['w1'].data.asnumpy() + + + slice0 = get_slice(0) + slice1 = get_slice(1) + slice4 = get_slice(4) + slice_shape = slice0.shape + + slice0 = slice0.flatten() + slice1 = slice1.flatten() + slice4 = slice4.flatten() + expect_slice_shape = (16, 32) + + assert expect_slice_shape == slice_shape + assert all(slice0 == slice4) + assert all(slice0 == slice1) + if __name__ == '__main__': test_initializer_weight_slice()