384 lines
13 KiB
Python
384 lines
13 KiB
Python
# Authors: Gilles Louppe, Mathieu Blondel, Maheshakya Wijewardena
|
|
# License: BSD 3 clause
|
|
|
|
import numpy as np
|
|
import numbers
|
|
|
|
from ._base import SelectorMixin
|
|
from ._base import _get_feature_importances
|
|
from ..base import BaseEstimator, clone, MetaEstimatorMixin
|
|
from ..utils._tags import _safe_tags
|
|
from ..utils.validation import check_is_fitted
|
|
|
|
from ..exceptions import NotFittedError
|
|
from ..utils.metaestimators import available_if
|
|
from ..utils.validation import check_scalar
|
|
|
|
|
|
def _calculate_threshold(estimator, importances, threshold):
|
|
"""Interpret the threshold value"""
|
|
|
|
if threshold is None:
|
|
# determine default from estimator
|
|
est_name = estimator.__class__.__name__
|
|
if (
|
|
hasattr(estimator, "penalty") and estimator.penalty == "l1"
|
|
) or "Lasso" in est_name:
|
|
# the natural default threshold is 0 when l1 penalty was used
|
|
threshold = 1e-5
|
|
else:
|
|
threshold = "mean"
|
|
|
|
if isinstance(threshold, str):
|
|
if "*" in threshold:
|
|
scale, reference = threshold.split("*")
|
|
scale = float(scale.strip())
|
|
reference = reference.strip()
|
|
|
|
if reference == "median":
|
|
reference = np.median(importances)
|
|
elif reference == "mean":
|
|
reference = np.mean(importances)
|
|
else:
|
|
raise ValueError("Unknown reference: " + reference)
|
|
|
|
threshold = scale * reference
|
|
|
|
elif threshold == "median":
|
|
threshold = np.median(importances)
|
|
|
|
elif threshold == "mean":
|
|
threshold = np.mean(importances)
|
|
|
|
else:
|
|
raise ValueError(
|
|
"Expected threshold='mean' or threshold='median' got %s" % threshold
|
|
)
|
|
|
|
else:
|
|
threshold = float(threshold)
|
|
|
|
return threshold
|
|
|
|
|
|
def _estimator_has(attr):
|
|
"""Check if we can delegate a method to the underlying estimator.
|
|
|
|
First, we check the fitted estimator if available, otherwise we
|
|
check the unfitted estimator.
|
|
"""
|
|
return lambda self: (
|
|
hasattr(self.estimator_, attr)
|
|
if hasattr(self, "estimator_")
|
|
else hasattr(self.estimator, attr)
|
|
)
|
|
|
|
|
|
class SelectFromModel(MetaEstimatorMixin, SelectorMixin, BaseEstimator):
|
|
"""Meta-transformer for selecting features based on importance weights.
|
|
|
|
.. versionadded:: 0.17
|
|
|
|
Read more in the :ref:`User Guide <select_from_model>`.
|
|
|
|
Parameters
|
|
----------
|
|
estimator : object
|
|
The base estimator from which the transformer is built.
|
|
This can be both a fitted (if ``prefit`` is set to True)
|
|
or a non-fitted estimator. The estimator should have a
|
|
``feature_importances_`` or ``coef_`` attribute after fitting.
|
|
Otherwise, the ``importance_getter`` parameter should be used.
|
|
|
|
threshold : str or float, default=None
|
|
The threshold value to use for feature selection. Features whose
|
|
importance is greater or equal are kept while the others are
|
|
discarded. If "median" (resp. "mean"), then the ``threshold`` value is
|
|
the median (resp. the mean) of the feature importances. A scaling
|
|
factor (e.g., "1.25*mean") may also be used. If None and if the
|
|
estimator has a parameter penalty set to l1, either explicitly
|
|
or implicitly (e.g, Lasso), the threshold used is 1e-5.
|
|
Otherwise, "mean" is used by default.
|
|
|
|
prefit : bool, default=False
|
|
Whether a prefit model is expected to be passed into the constructor
|
|
directly or not. If True, ``transform`` must be called directly
|
|
and SelectFromModel cannot be used with ``cross_val_score``,
|
|
``GridSearchCV`` and similar utilities that clone the estimator.
|
|
Otherwise train the model using ``fit`` and then ``transform`` to do
|
|
feature selection.
|
|
|
|
norm_order : non-zero int, inf, -inf, default=1
|
|
Order of the norm used to filter the vectors of coefficients below
|
|
``threshold`` in the case where the ``coef_`` attribute of the
|
|
estimator is of dimension 2.
|
|
|
|
max_features : int, callable, default=None
|
|
The maximum number of features to select.
|
|
|
|
- If an integer, then it specifies the maximum number of features to
|
|
allow.
|
|
- If a callable, then it specifies how to calculate the maximum number of
|
|
features allowed by using the output of `max_feaures(X)`.
|
|
|
|
To only select based on ``max_features``, set ``threshold=-np.inf``.
|
|
|
|
.. versionadded:: 0.20
|
|
|
|
importance_getter : str or callable, default='auto'
|
|
If 'auto', uses the feature importance either through a ``coef_``
|
|
attribute or ``feature_importances_`` attribute of estimator.
|
|
|
|
Also accepts a string that specifies an attribute name/path
|
|
for extracting feature importance (implemented with `attrgetter`).
|
|
For example, give `regressor_.coef_` in case of
|
|
:class:`~sklearn.compose.TransformedTargetRegressor` or
|
|
`named_steps.clf.feature_importances_` in case of
|
|
:class:`~sklearn.pipeline.Pipeline` with its last step named `clf`.
|
|
|
|
If `callable`, overrides the default feature importance getter.
|
|
The callable is passed with the fitted estimator and it should
|
|
return importance for each feature.
|
|
|
|
.. versionadded:: 0.24
|
|
|
|
Attributes
|
|
----------
|
|
estimator_ : an estimator
|
|
The base estimator from which the transformer is built.
|
|
This is stored only when a non-fitted estimator is passed to the
|
|
``SelectFromModel``, i.e when prefit is False.
|
|
|
|
n_features_in_ : int
|
|
Number of features seen during :term:`fit`. Only defined if the
|
|
underlying estimator exposes such an attribute when fit.
|
|
|
|
.. versionadded:: 0.24
|
|
|
|
max_features_ : int
|
|
Maximum number of features calculated during :term:`fit`. Only defined
|
|
if the ``max_features`` is not `None`.
|
|
|
|
- If `max_features` is an int, then `max_features_ = max_features`.
|
|
- If `max_features` is a callable, then `max_features_ = max_features(X)`.
|
|
|
|
.. versionadded:: 1.1
|
|
|
|
feature_names_in_ : ndarray of shape (`n_features_in_`,)
|
|
Names of features seen during :term:`fit`. Defined only when `X`
|
|
has feature names that are all strings.
|
|
|
|
.. versionadded:: 1.0
|
|
|
|
threshold_ : float
|
|
The threshold value used for feature selection.
|
|
|
|
See Also
|
|
--------
|
|
RFE : Recursive feature elimination based on importance weights.
|
|
RFECV : Recursive feature elimination with built-in cross-validated
|
|
selection of the best number of features.
|
|
SequentialFeatureSelector : Sequential cross-validation based feature
|
|
selection. Does not rely on importance weights.
|
|
|
|
Notes
|
|
-----
|
|
Allows NaN/Inf in the input if the underlying estimator does as well.
|
|
|
|
Examples
|
|
--------
|
|
>>> from sklearn.feature_selection import SelectFromModel
|
|
>>> from sklearn.linear_model import LogisticRegression
|
|
>>> X = [[ 0.87, -1.34, 0.31 ],
|
|
... [-2.79, -0.02, -0.85 ],
|
|
... [-1.34, -0.48, -2.55 ],
|
|
... [ 1.92, 1.48, 0.65 ]]
|
|
>>> y = [0, 1, 0, 1]
|
|
>>> selector = SelectFromModel(estimator=LogisticRegression()).fit(X, y)
|
|
>>> selector.estimator_.coef_
|
|
array([[-0.3252302 , 0.83462377, 0.49750423]])
|
|
>>> selector.threshold_
|
|
0.55245...
|
|
>>> selector.get_support()
|
|
array([False, True, False])
|
|
>>> selector.transform(X)
|
|
array([[-1.34],
|
|
[-0.02],
|
|
[-0.48],
|
|
[ 1.48]])
|
|
|
|
Using a callable to create a selector that can use no more than half
|
|
of the input features.
|
|
|
|
>>> def half_callable(X):
|
|
... return round(len(X[0]) / 2)
|
|
>>> half_selector = SelectFromModel(estimator=LogisticRegression(),
|
|
... max_features=half_callable)
|
|
>>> _ = half_selector.fit(X, y)
|
|
>>> half_selector.max_features_
|
|
2
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
estimator,
|
|
*,
|
|
threshold=None,
|
|
prefit=False,
|
|
norm_order=1,
|
|
max_features=None,
|
|
importance_getter="auto",
|
|
):
|
|
self.estimator = estimator
|
|
self.threshold = threshold
|
|
self.prefit = prefit
|
|
self.importance_getter = importance_getter
|
|
self.norm_order = norm_order
|
|
self.max_features = max_features
|
|
|
|
def _get_support_mask(self):
|
|
# SelectFromModel can directly call on transform.
|
|
if self.prefit:
|
|
estimator = self.estimator
|
|
elif hasattr(self, "estimator_"):
|
|
estimator = self.estimator_
|
|
else:
|
|
raise ValueError(
|
|
"Either fit the model before transform or set"
|
|
' "prefit=True" while passing the fitted'
|
|
" estimator to the constructor."
|
|
)
|
|
scores = _get_feature_importances(
|
|
estimator=estimator,
|
|
getter=self.importance_getter,
|
|
transform_func="norm",
|
|
norm_order=self.norm_order,
|
|
)
|
|
threshold = _calculate_threshold(estimator, scores, self.threshold)
|
|
if self.max_features is not None:
|
|
mask = np.zeros_like(scores, dtype=bool)
|
|
candidate_indices = np.argsort(-scores, kind="mergesort")[
|
|
: self.max_features_
|
|
]
|
|
mask[candidate_indices] = True
|
|
else:
|
|
mask = np.ones_like(scores, dtype=bool)
|
|
mask[scores < threshold] = False
|
|
return mask
|
|
|
|
def fit(self, X, y=None, **fit_params):
|
|
"""Fit the SelectFromModel meta-transformer.
|
|
|
|
Parameters
|
|
----------
|
|
X : array-like of shape (n_samples, n_features)
|
|
The training input samples.
|
|
|
|
y : array-like of shape (n_samples,), default=None
|
|
The target values (integers that correspond to classes in
|
|
classification, real numbers in regression).
|
|
|
|
**fit_params : dict
|
|
Other estimator specific parameters.
|
|
|
|
Returns
|
|
-------
|
|
self : object
|
|
Fitted estimator.
|
|
"""
|
|
if self.max_features is not None:
|
|
if isinstance(self.max_features, numbers.Integral):
|
|
check_scalar(
|
|
self.max_features,
|
|
"max_features",
|
|
numbers.Integral,
|
|
min_val=0,
|
|
max_val=len(X[0]),
|
|
)
|
|
self.max_features_ = self.max_features
|
|
elif callable(self.max_features):
|
|
max_features = self.max_features(X)
|
|
check_scalar(
|
|
max_features,
|
|
"max_features(X)",
|
|
numbers.Integral,
|
|
min_val=0,
|
|
max_val=len(X[0]),
|
|
)
|
|
self.max_features_ = max_features
|
|
else:
|
|
raise TypeError(
|
|
"'max_features' must be either an int or a callable that takes"
|
|
f" 'X' as input. Got {self.max_features} instead."
|
|
)
|
|
|
|
if self.prefit:
|
|
raise NotFittedError("Since 'prefit=True', call transform directly")
|
|
self.estimator_ = clone(self.estimator)
|
|
self.estimator_.fit(X, y, **fit_params)
|
|
|
|
if hasattr(self.estimator_, "feature_names_in_"):
|
|
self.feature_names_in_ = self.estimator_.feature_names_in_
|
|
else:
|
|
self._check_feature_names(X, reset=True)
|
|
|
|
return self
|
|
|
|
@property
|
|
def threshold_(self):
|
|
"""Threshold value used for feature selection."""
|
|
scores = _get_feature_importances(
|
|
estimator=self.estimator_,
|
|
getter=self.importance_getter,
|
|
transform_func="norm",
|
|
norm_order=self.norm_order,
|
|
)
|
|
return _calculate_threshold(self.estimator, scores, self.threshold)
|
|
|
|
@available_if(_estimator_has("partial_fit"))
|
|
def partial_fit(self, X, y=None, **fit_params):
|
|
"""Fit the SelectFromModel meta-transformer only once.
|
|
|
|
Parameters
|
|
----------
|
|
X : array-like of shape (n_samples, n_features)
|
|
The training input samples.
|
|
|
|
y : array-like of shape (n_samples,), default=None
|
|
The target values (integers that correspond to classes in
|
|
classification, real numbers in regression).
|
|
|
|
**fit_params : dict
|
|
Other estimator specific parameters.
|
|
|
|
Returns
|
|
-------
|
|
self : object
|
|
Fitted estimator.
|
|
"""
|
|
if self.prefit:
|
|
raise NotFittedError("Since 'prefit=True', call transform directly")
|
|
if not hasattr(self, "estimator_"):
|
|
self.estimator_ = clone(self.estimator)
|
|
self.estimator_.partial_fit(X, y, **fit_params)
|
|
return self
|
|
|
|
@property
|
|
def n_features_in_(self):
|
|
"""Number of features seen during `fit`."""
|
|
# For consistency with other estimators we raise a AttributeError so
|
|
# that hasattr() fails if the estimator isn't fitted.
|
|
try:
|
|
check_is_fitted(self)
|
|
except NotFittedError as nfe:
|
|
raise AttributeError(
|
|
"{} object has no n_features_in_ attribute.".format(
|
|
self.__class__.__name__
|
|
)
|
|
) from nfe
|
|
|
|
return self.estimator_.n_features_in_
|
|
|
|
def _more_tags(self):
|
|
return {"allow_nan": _safe_tags(self.estimator, key="allow_nan")}
|