456 lines
15 KiB
Python
456 lines
15 KiB
Python
"""Utilities for input validation"""
|
|
# Authors: Olivier Grisel
|
|
# Gael Varoquaux
|
|
# Andreas Mueller
|
|
# Lars Buitinck
|
|
# Alexandre Gramfort
|
|
# Nicolas Tresegnie
|
|
# License: BSD 3 clause
|
|
|
|
import warnings
|
|
import numbers
|
|
|
|
import numpy as np
|
|
import scipy.sparse as sp
|
|
|
|
from ..externals import six
|
|
from inspect import getargspec
|
|
|
|
|
|
class DataConversionWarning(UserWarning):
|
|
"A warning on implicit data conversions happening in the code"
|
|
pass
|
|
|
|
warnings.simplefilter("always", DataConversionWarning)
|
|
|
|
|
|
class NonBLASDotWarning(UserWarning):
|
|
"A warning on implicit dispatch to numpy.dot"
|
|
pass
|
|
|
|
|
|
# Silenced by default to reduce verbosity. Turn on at runtime for
|
|
# performance profiling.
|
|
warnings.simplefilter('ignore', NonBLASDotWarning)
|
|
|
|
|
|
def _assert_all_finite(X):
|
|
"""Like assert_all_finite, but only for ndarray."""
|
|
X = np.asanyarray(X)
|
|
# First try an O(n) time, O(1) space solution for the common case that
|
|
# everything is finite; fall back to O(n) space np.isfinite to prevent
|
|
# false positives from overflow in sum method.
|
|
if (X.dtype.char in np.typecodes['AllFloat'] and not np.isfinite(X.sum())
|
|
and not np.isfinite(X).all()):
|
|
raise ValueError("Input contains NaN, infinity"
|
|
" or a value too large for %r." % X.dtype)
|
|
|
|
|
|
def assert_all_finite(X):
|
|
"""Throw a ValueError if X contains NaN or infinity.
|
|
|
|
Input MUST be an np.ndarray instance or a scipy.sparse matrix."""
|
|
_assert_all_finite(X.data if sp.issparse(X) else X)
|
|
|
|
|
|
def as_float_array(X, copy=True, force_all_finite=True):
|
|
"""Converts an array-like to an array of floats
|
|
|
|
The new dtype will be np.float32 or np.float64, depending on the original
|
|
type. The function can create a copy or modify the argument depending
|
|
on the argument copy.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}
|
|
|
|
copy : bool, optional
|
|
If True, a copy of X will be created. If False, a copy may still be
|
|
returned if X's dtype is not a floating point type.
|
|
|
|
Returns
|
|
-------
|
|
XT : {array, sparse matrix}
|
|
An array of type np.float
|
|
"""
|
|
if isinstance(X, np.matrix) or (not isinstance(X, np.ndarray)
|
|
and not sp.issparse(X)):
|
|
return check_array(X, ['csr', 'csc', 'coo'], dtype=np.float64,
|
|
copy=copy, force_all_finite=force_all_finite,
|
|
ensure_2d=False)
|
|
elif sp.issparse(X) and X.dtype in [np.float32, np.float64]:
|
|
return X.copy() if copy else X
|
|
elif X.dtype in [np.float32, np.float64]: # is numpy array
|
|
return X.copy('F' if X.flags['F_CONTIGUOUS'] else 'C') if copy else X
|
|
else:
|
|
return X.astype(np.float32 if X.dtype == np.int32 else np.float64)
|
|
|
|
|
|
def _is_arraylike(x):
|
|
"""Returns whether the input is array-like"""
|
|
return (hasattr(x, '__len__') or
|
|
hasattr(x, 'shape') or
|
|
hasattr(x, '__array__'))
|
|
|
|
|
|
def _num_samples(x):
|
|
"""Return number of samples in array-like x."""
|
|
if not hasattr(x, '__len__') and not hasattr(x, 'shape'):
|
|
if hasattr(x, '__array__'):
|
|
x = np.asarray(x)
|
|
else:
|
|
raise TypeError("Expected sequence or array-like, got %r" % x)
|
|
return x.shape[0] if hasattr(x, 'shape') else len(x)
|
|
|
|
|
|
def check_consistent_length(*arrays):
|
|
"""Check that all arrays have consistent first dimensions.
|
|
|
|
Checks whether all objects in arrays have the same shape or length.
|
|
|
|
Parameters
|
|
----------
|
|
arrays : list or tuple of input objects.
|
|
Objects that will be checked for consistent length.
|
|
"""
|
|
|
|
uniques = np.unique([_num_samples(X) for X in arrays if X is not None])
|
|
if len(uniques) > 1:
|
|
raise ValueError("Found arrays with inconsistent numbers of samples: %s"
|
|
% str(uniques))
|
|
|
|
|
|
def indexable(*iterables):
|
|
"""Make arrays indexable for cross-validation.
|
|
|
|
Checks consistent length, passes through None, and ensures that everything
|
|
can be indexed by converting sparse matrices to csr and converting
|
|
non-interable objects to arrays.
|
|
|
|
Parameters
|
|
----------
|
|
iterables : lists, dataframes, arrays, sparse matrices
|
|
List of objects to ensure sliceability.
|
|
"""
|
|
result = []
|
|
for X in iterables:
|
|
if sp.issparse(X):
|
|
result.append(X.tocsr())
|
|
elif hasattr(X, "__getitem__") or hasattr(X, "iloc"):
|
|
result.append(X)
|
|
elif X is None:
|
|
result.append(X)
|
|
else:
|
|
result.append(np.array(X))
|
|
check_consistent_length(*result)
|
|
return result
|
|
|
|
|
|
def _ensure_sparse_format(spmatrix, accept_sparse, dtype, order, copy,
|
|
force_all_finite):
|
|
"""Convert a sparse matrix to a given format.
|
|
|
|
Checks the sparse format of spmatrix and converts if necessary.
|
|
|
|
Parameters
|
|
----------
|
|
spmatrix : scipy sparse matrix
|
|
Input to validate and convert.
|
|
|
|
accept_sparse : string, list of string or None (default=None)
|
|
String[s] representing allowed sparse matrix formats ('csc',
|
|
'csr', 'coo', 'dok', 'bsr', 'lil', 'dia'). None means that sparse
|
|
matrix input will raise an error. If the input is sparse but not in
|
|
the allowed format, it will be converted to the first listed format.
|
|
|
|
dtype : string, type or None (default=none)
|
|
Data type of result. If None, the dtype of the input is preserved.
|
|
|
|
order : 'F', 'C' or None (default=None)
|
|
Whether an array will be forced to be fortran or c-style.
|
|
|
|
copy : boolean (default=False)
|
|
Whether a forced copy will be triggered. If copy=False, a copy might
|
|
be triggered by a conversion.
|
|
|
|
force_all_finite : boolean (default=True)
|
|
Whether to raise an error on np.inf and np.nan in X.
|
|
|
|
Returns
|
|
-------
|
|
spmatrix_converted : scipy sparse matrix.
|
|
Matrix that is ensured to have an allowed type.
|
|
"""
|
|
if accept_sparse is None:
|
|
raise TypeError('A sparse matrix was passed, but dense '
|
|
'data is required. Use X.toarray() to '
|
|
'convert to a dense numpy array.')
|
|
sparse_type = spmatrix.format
|
|
if dtype is None:
|
|
dtype = spmatrix.dtype
|
|
if sparse_type in accept_sparse:
|
|
# correct type
|
|
if dtype == spmatrix.dtype:
|
|
# correct dtype
|
|
if copy:
|
|
spmatrix = spmatrix.copy()
|
|
else:
|
|
# convert dtype
|
|
spmatrix = spmatrix.astype(dtype)
|
|
else:
|
|
# create new
|
|
spmatrix = spmatrix.asformat(accept_sparse[0]).astype(dtype)
|
|
if force_all_finite:
|
|
if not hasattr(spmatrix, "data"):
|
|
warnings.warn("Can't check %s sparse matrix for nan or inf."
|
|
% spmatrix.format)
|
|
else:
|
|
_assert_all_finite(spmatrix.data)
|
|
if hasattr(spmatrix, "data"):
|
|
spmatrix.data = np.array(spmatrix.data, copy=False, order=order)
|
|
return spmatrix
|
|
|
|
|
|
def check_array(array, accept_sparse=None, dtype=None, order=None, copy=False,
|
|
force_all_finite=True, ensure_2d=True, allow_nd=False):
|
|
"""Input validation on an array, list, sparse matrix or similar.
|
|
|
|
By default, the input is converted to an at least 2nd numpy array.
|
|
|
|
Parameters
|
|
----------
|
|
array : object
|
|
Input object to check / convert.
|
|
|
|
accept_sparse : string, list of string or None (default=None)
|
|
String[s] representing allowed sparse matrix formats, such as 'csc',
|
|
'csr', etc. None means that sparse matrix input will raise an error.
|
|
If the input is sparse but not in the allowed format, it will be
|
|
converted to the first listed format.
|
|
|
|
dtype : string, type or None (default=none)
|
|
Data type of result. If None, the dtype of the input is preserved.
|
|
|
|
order : 'F', 'C' or None (default=None)
|
|
Whether an array will be forced to be fortran or c-style.
|
|
|
|
copy : boolean (default=False)
|
|
Whether a forced copy will be triggered. If copy=False, a copy might
|
|
be triggered by a conversion.
|
|
|
|
force_all_finite : boolean (default=True)
|
|
Whether to raise an error on np.inf and np.nan in X.
|
|
|
|
ensure_2d : boolean (default=True)
|
|
Whether to make X at least 2d.
|
|
|
|
allow_nd : boolean (default=False)
|
|
Whether to allow X.ndim > 2.
|
|
|
|
Returns
|
|
-------
|
|
X_converted : object
|
|
The converted and validated X.
|
|
"""
|
|
if isinstance(accept_sparse, str):
|
|
accept_sparse = [accept_sparse]
|
|
|
|
if sp.issparse(array):
|
|
array = _ensure_sparse_format(array, accept_sparse, dtype, order,
|
|
copy, force_all_finite)
|
|
else:
|
|
if ensure_2d:
|
|
array = np.atleast_2d(array)
|
|
array = np.array(array, dtype=dtype, order=order, copy=copy)
|
|
if not allow_nd and array.ndim >= 3:
|
|
raise ValueError("Found array with dim %d. Expected <= 2" %
|
|
array.ndim)
|
|
if force_all_finite:
|
|
_assert_all_finite(array)
|
|
|
|
return array
|
|
|
|
|
|
def check_X_y(X, y, accept_sparse=None, dtype=None, order=None, copy=False,
|
|
force_all_finite=True, ensure_2d=True, allow_nd=False,
|
|
multi_output=False):
|
|
"""Input validation for standard estimators.
|
|
|
|
Checks X and y for consistent length, enforces X 2d and y 1d.
|
|
Standard input checks are only applied to y. For multi-label y,
|
|
set multi_ouput=True to allow 2d and sparse y.
|
|
|
|
Parameters
|
|
----------
|
|
X : nd-array, list or sparse matrix
|
|
Input data.
|
|
|
|
y : nd-array, list or sparse matrix
|
|
Labels.
|
|
|
|
accept_sparse : string, list of string or None (default=None)
|
|
String[s] representing allowed sparse matrix formats, such as 'csc',
|
|
'csr', etc. None means that sparse matrix input will raise an error.
|
|
If the input is sparse but not in the allowed format, it will be
|
|
converted to the first listed format.
|
|
|
|
dtype : string, type or None (default=none)
|
|
Data type of result. If None, the dtype of the input is preserved.
|
|
|
|
order : 'F', 'C' or None (default=None)
|
|
Whether an array will be forced to be fortran or c-style.
|
|
|
|
copy : boolean (default=False)
|
|
Whether a forced copy will be triggered. If copy=False, a copy might
|
|
be triggered by a conversion.
|
|
|
|
force_all_finite : boolean (default=True)
|
|
Whether to raise an error on np.inf and np.nan in X.
|
|
|
|
ensure_2d : boolean (default=True)
|
|
Whether to make X at least 2d.
|
|
|
|
allow_nd : boolean (default=False)
|
|
Whether to allow X.ndim > 2.
|
|
|
|
multi_output : boolean (default=False)
|
|
Whether to allow 2-d y (array or sparse matrix). If false, y will be
|
|
validated as a vector.
|
|
|
|
Returns
|
|
-------
|
|
X_converted : object
|
|
The converted and validated X.
|
|
"""
|
|
X = check_array(X, accept_sparse, dtype, order, copy, force_all_finite,
|
|
ensure_2d, allow_nd)
|
|
if multi_output:
|
|
y = check_array(y, 'csr', force_all_finite=True, ensure_2d=False)
|
|
else:
|
|
y = column_or_1d(y, warn=True)
|
|
_assert_all_finite(y)
|
|
|
|
check_consistent_length(X, y)
|
|
|
|
return X, y
|
|
|
|
|
|
def column_or_1d(y, warn=False):
|
|
""" Ravel column or 1d numpy array, else raises an error
|
|
|
|
Parameters
|
|
----------
|
|
y : array-like
|
|
|
|
Returns
|
|
-------
|
|
y : array
|
|
|
|
"""
|
|
shape = np.shape(y)
|
|
if len(shape) == 1:
|
|
return np.ravel(y)
|
|
if len(shape) == 2 and shape[1] == 1:
|
|
if warn:
|
|
warnings.warn("A column-vector y was passed when a 1d array was"
|
|
" expected. Please change the shape of y to "
|
|
"(n_samples, ), for example using ravel().",
|
|
DataConversionWarning, stacklevel=2)
|
|
return np.ravel(y)
|
|
|
|
raise ValueError("bad input shape {0}".format(shape))
|
|
|
|
|
|
def warn_if_not_float(X, estimator='This algorithm'):
|
|
"""Warning utility function to check that data type is floating point.
|
|
|
|
Returns True if a warning was raised (i.e. the input is not float) and
|
|
False otherwise, for easier input validation.
|
|
"""
|
|
if not isinstance(estimator, six.string_types):
|
|
estimator = estimator.__class__.__name__
|
|
if X.dtype.kind != 'f':
|
|
warnings.warn("%s assumes floating point values as input, "
|
|
"got %s" % (estimator, X.dtype))
|
|
return True
|
|
return False
|
|
|
|
|
|
def check_random_state(seed):
|
|
"""Turn seed into a np.random.RandomState instance
|
|
|
|
If seed is None, return the RandomState singleton used by np.random.
|
|
If seed is an int, return a new RandomState instance seeded with seed.
|
|
If seed is already a RandomState instance, return it.
|
|
Otherwise raise ValueError.
|
|
"""
|
|
if seed is None or seed is np.random:
|
|
return np.random.mtrand._rand
|
|
if isinstance(seed, (numbers.Integral, np.integer)):
|
|
return np.random.RandomState(seed)
|
|
if isinstance(seed, np.random.RandomState):
|
|
return seed
|
|
raise ValueError('%r cannot be used to seed a numpy.random.RandomState'
|
|
' instance' % seed)
|
|
|
|
def has_fit_parameter(estimator, parameter):
|
|
""" Checks whether the estimator's fit method supports the given parameter.
|
|
|
|
Example
|
|
-------
|
|
>>> from sklearn.svm import SVC
|
|
>>> has_fit_parameter(SVC(), "sample_weight")
|
|
True
|
|
|
|
"""
|
|
return parameter in getargspec(estimator.fit)[0]
|
|
|
|
|
|
def check_symmetric(array, tol=1E-10, raise_warning=True,
|
|
raise_exception=False):
|
|
"""
|
|
Check that the array is symmetric two-dimensional array or sparse matrix,
|
|
returning a symmetrized version and optionally raising a warning or
|
|
exception if the input is not symmetric.
|
|
|
|
Parameters
|
|
----------
|
|
array : nd-array or sparse matrix
|
|
Input object to check / convert
|
|
tol : float
|
|
Absolute tolerance for equivalence of arrays. Default = 1E-10.
|
|
raise_warning : boolean (default=True)
|
|
If True then raise a warning if conversion is required.
|
|
raise_exception : boolean (default=False)
|
|
If True then raise an exception if array is not symmetric.
|
|
|
|
Returns
|
|
-------
|
|
array_sym : object
|
|
Symmetrized version of the input array, i.e. the average of array
|
|
and array.transpose(). If sparse, then duplicate entries are first
|
|
summed and zeros are eliminated.
|
|
"""
|
|
if (array.ndim != 2) or (array.shape[0] != array.shape[1]):
|
|
raise ValueError("array must be 2-dimensional and symmetric")
|
|
|
|
if sp.issparse(array):
|
|
diff = (array - array.T).data
|
|
symmetric = np.all(abs(diff) < tol)
|
|
else:
|
|
symmetric = np.allclose(array, array.T, atol=tol)
|
|
|
|
if not symmetric:
|
|
if raise_exception:
|
|
raise ValueError("Array must be symmetric")
|
|
if raise_warning:
|
|
warnings.warn("Array is not symmetric, and will be converted "
|
|
"to symmetric by average with its transpose.")
|
|
if sp.issparse(array):
|
|
conversion = 'to' + array.format
|
|
array = getattr(0.5 * (array + array.T), conversion)()
|
|
else:
|
|
array = 0.5 * (array + array.T)
|
|
|
|
return array
|