494 lines
17 KiB
Python
494 lines
17 KiB
Python
"""
|
|
Soft Voting/Majority Rule classifier and Voting regressor.
|
|
|
|
This module contains:
|
|
- A Soft Voting/Majority Rule classifier for classification estimators.
|
|
- A Voting regressor for regression estimators.
|
|
"""
|
|
|
|
# Authors: Sebastian Raschka <se.raschka@gmail.com>,
|
|
# Gilles Louppe <g.louppe@gmail.com>,
|
|
# Ramil Nugmanov <stsouko@live.ru>
|
|
# Mohamed Ali Jamaoui <m.ali.jamaoui@gmail.com>
|
|
#
|
|
# License: BSD 3 clause
|
|
|
|
import numpy as np
|
|
from abc import abstractmethod
|
|
|
|
from ..base import ClassifierMixin
|
|
from ..base import RegressorMixin
|
|
from ..base import TransformerMixin
|
|
from ..base import clone
|
|
from ..preprocessing import LabelEncoder
|
|
from ..utils._joblib import Parallel, delayed
|
|
from ..utils.validation import has_fit_parameter, check_is_fitted
|
|
from ..utils.metaestimators import _BaseComposition
|
|
from ..utils import Bunch
|
|
|
|
|
|
def _parallel_fit_estimator(estimator, X, y, sample_weight=None):
|
|
"""Private function used to fit an estimator within a job."""
|
|
if sample_weight is not None:
|
|
try:
|
|
estimator.fit(X, y, sample_weight=sample_weight)
|
|
except TypeError as exc:
|
|
if "unexpected keyword argument 'sample_weight'" in str(exc):
|
|
raise ValueError(
|
|
"Underlying estimator {} does not support sample weights."
|
|
.format(estimator.__class__.__name__)
|
|
) from exc
|
|
raise
|
|
else:
|
|
estimator.fit(X, y)
|
|
return estimator
|
|
|
|
|
|
class _BaseVoting(_BaseComposition, TransformerMixin):
|
|
"""Base class for voting.
|
|
|
|
Warning: This class should not be used directly. Use derived classes
|
|
instead.
|
|
"""
|
|
_required_parameters = ['estimators']
|
|
|
|
@property
|
|
def named_estimators(self):
|
|
return Bunch(**dict(self.estimators))
|
|
|
|
@property
|
|
def _weights_not_none(self):
|
|
"""Get the weights of not `None` estimators"""
|
|
if self.weights is None:
|
|
return None
|
|
return [w for est, w in zip(self.estimators, self.weights)
|
|
if est[1] not in (None, 'drop')]
|
|
|
|
def _predict(self, X):
|
|
"""Collect results from clf.predict calls. """
|
|
return np.asarray([clf.predict(X) for clf in self.estimators_]).T
|
|
|
|
@abstractmethod
|
|
def fit(self, X, y, sample_weight=None):
|
|
"""
|
|
common fit operations.
|
|
"""
|
|
if self.estimators is None or len(self.estimators) == 0:
|
|
raise AttributeError('Invalid `estimators` attribute, `estimators`'
|
|
' should be a list of (string, estimator)'
|
|
' tuples')
|
|
|
|
if (self.weights is not None and
|
|
len(self.weights) != len(self.estimators)):
|
|
raise ValueError('Number of `estimators` and weights must be equal'
|
|
'; got %d weights, %d estimators'
|
|
% (len(self.weights), len(self.estimators)))
|
|
|
|
names, clfs = zip(*self.estimators)
|
|
self._validate_names(names)
|
|
|
|
n_isnone = np.sum(
|
|
[clf in (None, 'drop') for _, clf in self.estimators]
|
|
)
|
|
if n_isnone == len(self.estimators):
|
|
raise ValueError(
|
|
'All estimators are None or "drop". At least one is required!'
|
|
)
|
|
|
|
self.estimators_ = Parallel(n_jobs=self.n_jobs)(
|
|
delayed(_parallel_fit_estimator)(clone(clf), X, y,
|
|
sample_weight=sample_weight)
|
|
for clf in clfs if clf not in (None, 'drop')
|
|
)
|
|
|
|
self.named_estimators_ = Bunch()
|
|
for k, e in zip(self.estimators, self.estimators_):
|
|
self.named_estimators_[k[0]] = e
|
|
return self
|
|
|
|
def set_params(self, **params):
|
|
""" Setting the parameters for the ensemble estimator
|
|
|
|
Valid parameter keys can be listed with get_params().
|
|
|
|
Parameters
|
|
----------
|
|
**params : keyword arguments
|
|
Specific parameters using e.g. set_params(parameter_name=new_value)
|
|
In addition, to setting the parameters of the ensemble estimator,
|
|
the individual estimators of the ensemble estimator can also be
|
|
set or replaced by setting them to None.
|
|
|
|
Examples
|
|
--------
|
|
# In this example, the RandomForestClassifier is removed
|
|
clf1 = LogisticRegression()
|
|
clf2 = RandomForestClassifier()
|
|
eclf = VotingClassifier(estimators=[('lr', clf1), ('rf', clf2)]
|
|
eclf.set_params(rf=None)
|
|
"""
|
|
return self._set_params('estimators', **params)
|
|
|
|
def get_params(self, deep=True):
|
|
""" Get the parameters of the ensemble estimator
|
|
|
|
Parameters
|
|
----------
|
|
deep : bool
|
|
Setting it to True gets the various estimators and the parameters
|
|
of the estimators as well
|
|
"""
|
|
return self._get_params('estimators', deep=deep)
|
|
|
|
|
|
class VotingClassifier(_BaseVoting, ClassifierMixin):
|
|
"""Soft Voting/Majority Rule classifier for unfitted estimators.
|
|
|
|
.. versionadded:: 0.17
|
|
|
|
Read more in the :ref:`User Guide <voting_classifier>`.
|
|
|
|
Parameters
|
|
----------
|
|
estimators : list of (string, estimator) tuples
|
|
Invoking the ``fit`` method on the ``VotingClassifier`` will fit clones
|
|
of those original estimators that will be stored in the class attribute
|
|
``self.estimators_``. An estimator can be set to ``None`` or ``'drop'``
|
|
using ``set_params``.
|
|
|
|
voting : str, {'hard', 'soft'} (default='hard')
|
|
If 'hard', uses predicted class labels for majority rule voting.
|
|
Else if 'soft', predicts the class label based on the argmax of
|
|
the sums of the predicted probabilities, which is recommended for
|
|
an ensemble of well-calibrated classifiers.
|
|
|
|
weights : array-like, shape (n_classifiers,), optional (default=`None`)
|
|
Sequence of weights (`float` or `int`) to weight the occurrences of
|
|
predicted class labels (`hard` voting) or class probabilities
|
|
before averaging (`soft` voting). Uses uniform weights if `None`.
|
|
|
|
n_jobs : int or None, optional (default=None)
|
|
The number of jobs to run in parallel for ``fit``.
|
|
``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
|
|
``-1`` means using all processors. See :term:`Glossary <n_jobs>`
|
|
for more details.
|
|
|
|
flatten_transform : bool, optional (default=True)
|
|
Affects shape of transform output only when voting='soft'
|
|
If voting='soft' and flatten_transform=True, transform method returns
|
|
matrix with shape (n_samples, n_classifiers * n_classes). If
|
|
flatten_transform=False, it returns
|
|
(n_classifiers, n_samples, n_classes).
|
|
|
|
Attributes
|
|
----------
|
|
estimators_ : list of classifiers
|
|
The collection of fitted sub-estimators as defined in ``estimators``
|
|
that are not `None`.
|
|
|
|
named_estimators_ : Bunch object, a dictionary with attribute access
|
|
Attribute to access any fitted sub-estimators by name.
|
|
|
|
.. versionadded:: 0.20
|
|
|
|
classes_ : array-like, shape (n_predictions,)
|
|
The classes labels.
|
|
|
|
Examples
|
|
--------
|
|
>>> import numpy as np
|
|
>>> from sklearn.linear_model import LogisticRegression
|
|
>>> from sklearn.naive_bayes import GaussianNB
|
|
>>> from sklearn.ensemble import RandomForestClassifier, VotingClassifier
|
|
>>> clf1 = LogisticRegression(multi_class='multinomial', random_state=1)
|
|
>>> clf2 = RandomForestClassifier(n_estimators=50, random_state=1)
|
|
>>> clf3 = GaussianNB()
|
|
>>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
|
|
>>> y = np.array([1, 1, 1, 2, 2, 2])
|
|
>>> eclf1 = VotingClassifier(estimators=[
|
|
... ('lr', clf1), ('rf', clf2), ('gnb', clf3)], voting='hard')
|
|
>>> eclf1 = eclf1.fit(X, y)
|
|
>>> print(eclf1.predict(X))
|
|
[1 1 1 2 2 2]
|
|
>>> np.array_equal(eclf1.named_estimators_.lr.predict(X),
|
|
... eclf1.named_estimators_['lr'].predict(X))
|
|
True
|
|
>>> eclf2 = VotingClassifier(estimators=[
|
|
... ('lr', clf1), ('rf', clf2), ('gnb', clf3)],
|
|
... voting='soft')
|
|
>>> eclf2 = eclf2.fit(X, y)
|
|
>>> print(eclf2.predict(X))
|
|
[1 1 1 2 2 2]
|
|
>>> eclf3 = VotingClassifier(estimators=[
|
|
... ('lr', clf1), ('rf', clf2), ('gnb', clf3)],
|
|
... voting='soft', weights=[2,1,1],
|
|
... flatten_transform=True)
|
|
>>> eclf3 = eclf3.fit(X, y)
|
|
>>> print(eclf3.predict(X))
|
|
[1 1 1 2 2 2]
|
|
>>> print(eclf3.transform(X).shape)
|
|
(6, 6)
|
|
|
|
See also
|
|
--------
|
|
VotingRegressor: Prediction voting regressor.
|
|
"""
|
|
|
|
def __init__(self, estimators, voting='hard', weights=None, n_jobs=None,
|
|
flatten_transform=True):
|
|
self.estimators = estimators
|
|
self.voting = voting
|
|
self.weights = weights
|
|
self.n_jobs = n_jobs
|
|
self.flatten_transform = flatten_transform
|
|
|
|
def fit(self, X, y, sample_weight=None):
|
|
""" Fit the estimators.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}, shape (n_samples, n_features)
|
|
Training vectors, where n_samples is the number of samples and
|
|
n_features is the number of features.
|
|
|
|
y : array-like, shape (n_samples,)
|
|
Target values.
|
|
|
|
sample_weight : array-like, shape (n_samples,) or None
|
|
Sample weights. If None, then samples are equally weighted.
|
|
Note that this is supported only if all underlying estimators
|
|
support sample weights.
|
|
|
|
Returns
|
|
-------
|
|
self : object
|
|
"""
|
|
if isinstance(y, np.ndarray) and len(y.shape) > 1 and y.shape[1] > 1:
|
|
raise NotImplementedError('Multilabel and multi-output'
|
|
' classification is not supported.')
|
|
|
|
if self.voting not in ('soft', 'hard'):
|
|
raise ValueError("Voting must be 'soft' or 'hard'; got (voting=%r)"
|
|
% self.voting)
|
|
|
|
self.le_ = LabelEncoder().fit(y)
|
|
self.classes_ = self.le_.classes_
|
|
transformed_y = self.le_.transform(y)
|
|
|
|
return super().fit(X, transformed_y, sample_weight)
|
|
|
|
def predict(self, X):
|
|
""" Predict class labels for X.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}, shape (n_samples, n_features)
|
|
The input samples.
|
|
|
|
Returns
|
|
-------
|
|
maj : array-like, shape (n_samples,)
|
|
Predicted class labels.
|
|
"""
|
|
|
|
check_is_fitted(self, 'estimators_')
|
|
if self.voting == 'soft':
|
|
maj = np.argmax(self.predict_proba(X), axis=1)
|
|
|
|
else: # 'hard' voting
|
|
predictions = self._predict(X)
|
|
maj = np.apply_along_axis(
|
|
lambda x: np.argmax(
|
|
np.bincount(x, weights=self._weights_not_none)),
|
|
axis=1, arr=predictions)
|
|
|
|
maj = self.le_.inverse_transform(maj)
|
|
|
|
return maj
|
|
|
|
def _collect_probas(self, X):
|
|
"""Collect results from clf.predict calls. """
|
|
return np.asarray([clf.predict_proba(X) for clf in self.estimators_])
|
|
|
|
def _predict_proba(self, X):
|
|
"""Predict class probabilities for X in 'soft' voting """
|
|
if self.voting == 'hard':
|
|
raise AttributeError("predict_proba is not available when"
|
|
" voting=%r" % self.voting)
|
|
check_is_fitted(self, 'estimators_')
|
|
avg = np.average(self._collect_probas(X), axis=0,
|
|
weights=self._weights_not_none)
|
|
return avg
|
|
|
|
@property
|
|
def predict_proba(self):
|
|
"""Compute probabilities of possible outcomes for samples in X.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}, shape (n_samples, n_features)
|
|
The input samples.
|
|
|
|
Returns
|
|
-------
|
|
avg : array-like, shape (n_samples, n_classes)
|
|
Weighted average probability for each class per sample.
|
|
"""
|
|
return self._predict_proba
|
|
|
|
def transform(self, X):
|
|
"""Return class labels or probabilities for X for each estimator.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}, shape (n_samples, n_features)
|
|
Training vectors, where n_samples is the number of samples and
|
|
n_features is the number of features.
|
|
|
|
Returns
|
|
-------
|
|
probabilities_or_labels
|
|
If `voting='soft'` and `flatten_transform=True`:
|
|
returns array-like of shape (n_classifiers, n_samples *
|
|
n_classes), being class probabilities calculated by each
|
|
classifier.
|
|
If `voting='soft' and `flatten_transform=False`:
|
|
array-like of shape (n_classifiers, n_samples, n_classes)
|
|
If `voting='hard'`:
|
|
array-like of shape (n_samples, n_classifiers), being
|
|
class labels predicted by each classifier.
|
|
"""
|
|
check_is_fitted(self, 'estimators_')
|
|
|
|
if self.voting == 'soft':
|
|
probas = self._collect_probas(X)
|
|
if not self.flatten_transform:
|
|
return probas
|
|
return np.hstack(probas)
|
|
|
|
else:
|
|
return self._predict(X)
|
|
|
|
|
|
class VotingRegressor(_BaseVoting, RegressorMixin):
|
|
"""Prediction voting regressor for unfitted estimators.
|
|
|
|
.. versionadded:: 0.21
|
|
|
|
A voting regressor is an ensemble meta-estimator that fits base
|
|
regressors each on the whole dataset. It, then, averages the individual
|
|
predictions to form a final prediction.
|
|
|
|
Read more in the :ref:`User Guide <voting_regressor>`.
|
|
|
|
Parameters
|
|
----------
|
|
estimators : list of (string, estimator) tuples
|
|
Invoking the ``fit`` method on the ``VotingRegressor`` will fit clones
|
|
of those original estimators that will be stored in the class attribute
|
|
``self.estimators_``. An estimator can be set to ``None`` or ``'drop'``
|
|
using ``set_params``.
|
|
|
|
weights : array-like, shape (n_regressors,), optional (default=`None`)
|
|
Sequence of weights (`float` or `int`) to weight the occurrences of
|
|
predicted values before averaging. Uses uniform weights if `None`.
|
|
|
|
n_jobs : int or None, optional (default=None)
|
|
The number of jobs to run in parallel for ``fit``.
|
|
``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
|
|
``-1`` means using all processors. See :term:`Glossary <n_jobs>`
|
|
for more details.
|
|
|
|
Attributes
|
|
----------
|
|
estimators_ : list of regressors
|
|
The collection of fitted sub-estimators as defined in ``estimators``
|
|
that are not `None`.
|
|
|
|
named_estimators_ : Bunch object, a dictionary with attribute access
|
|
Attribute to access any fitted sub-estimators by name.
|
|
|
|
Examples
|
|
--------
|
|
>>> import numpy as np
|
|
>>> from sklearn.linear_model import LinearRegression
|
|
>>> from sklearn.ensemble import RandomForestRegressor
|
|
>>> from sklearn.ensemble import VotingRegressor
|
|
>>> r1 = LinearRegression()
|
|
>>> r2 = RandomForestRegressor(n_estimators=10, random_state=1)
|
|
>>> X = np.array([[1, 1], [2, 4], [3, 9], [4, 16], [5, 25], [6, 36]])
|
|
>>> y = np.array([2, 6, 12, 20, 30, 42])
|
|
>>> er = VotingRegressor([('lr', r1), ('rf', r2)])
|
|
>>> print(er.fit(X, y).predict(X))
|
|
[ 3.3 5.7 11.8 19.7 28. 40.3]
|
|
|
|
See also
|
|
--------
|
|
VotingClassifier: Soft Voting/Majority Rule classifier.
|
|
"""
|
|
|
|
def __init__(self, estimators, weights=None, n_jobs=None):
|
|
self.estimators = estimators
|
|
self.weights = weights
|
|
self.n_jobs = n_jobs
|
|
|
|
def fit(self, X, y, sample_weight=None):
|
|
""" Fit the estimators.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}, shape (n_samples, n_features)
|
|
Training vectors, where n_samples is the number of samples and
|
|
n_features is the number of features.
|
|
|
|
y : array-like, shape (n_samples,)
|
|
Target values.
|
|
|
|
sample_weight : array-like, shape (n_samples,) or None
|
|
Sample weights. If None, then samples are equally weighted.
|
|
Note that this is supported only if all underlying estimators
|
|
support sample weights.
|
|
|
|
Returns
|
|
-------
|
|
self : object
|
|
"""
|
|
return super().fit(X, y, sample_weight)
|
|
|
|
def predict(self, X):
|
|
"""Predict regression target for X.
|
|
|
|
The predicted regression target of an input sample is computed as the
|
|
mean predicted regression targets of the estimators in the ensemble.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix} of shape (n_samples, n_features)
|
|
The input samples.
|
|
|
|
Returns
|
|
-------
|
|
y : array of shape (n_samples,)
|
|
The predicted values.
|
|
"""
|
|
check_is_fitted(self, "estimators_")
|
|
return np.average(self._predict(X), axis=1,
|
|
weights=self._weights_not_none)
|
|
|
|
def transform(self, X):
|
|
"""Return predictions for X for each estimator.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}, shape (n_samples, n_features)
|
|
The input samples.
|
|
|
|
Returns
|
|
-------
|
|
predictions
|
|
array-like of shape (n_samples, n_classifiers), being
|
|
values predicted by each regressor.
|
|
"""
|
|
check_is_fitted(self, 'estimators_')
|
|
return self._predict(X)
|