242 lines
8.5 KiB
Python
242 lines
8.5 KiB
Python
"""
|
|
Soft Voting/Majority Rule classifier.
|
|
|
|
This module contains a Soft Voting/Majority Rule classifier for
|
|
classification estimators.
|
|
|
|
"""
|
|
|
|
# Authors: Sebastian Raschka <se.raschka@gmail.com>,
|
|
# Gilles Louppe <g.louppe@gmail.com>
|
|
#
|
|
# Licence: BSD 3 clause
|
|
|
|
import numpy as np
|
|
|
|
from ..base import BaseEstimator
|
|
from ..base import ClassifierMixin
|
|
from ..base import TransformerMixin
|
|
from ..base import clone
|
|
from ..preprocessing import LabelEncoder
|
|
from ..externals import six
|
|
from ..exceptions import NotFittedError
|
|
from ..utils.validation import check_is_fitted
|
|
|
|
|
|
class VotingClassifier(BaseEstimator, ClassifierMixin, TransformerMixin):
|
|
"""Soft Voting/Majority Rule classifier for unfitted estimators.
|
|
|
|
.. versionadded:: 0.17
|
|
|
|
Read more in the :ref:`User Guide <voting_classifier>`.
|
|
|
|
Parameters
|
|
----------
|
|
estimators : list of (string, estimator) tuples
|
|
Invoking the ``fit`` method on the ``VotingClassifier`` will fit clones
|
|
of those original estimators that will be stored in the class attribute
|
|
`self.estimators_`.
|
|
|
|
voting : str, {'hard', 'soft'} (default='hard')
|
|
If 'hard', uses predicted class labels for majority rule voting.
|
|
Else if 'soft', predicts the class label based on the argmax of
|
|
the sums of the predicted probabilities, which is recommended for
|
|
an ensemble of well-calibrated classifiers.
|
|
|
|
weights : array-like, shape = [n_classifiers], optional (default=`None`)
|
|
Sequence of weights (`float` or `int`) to weight the occurrences of
|
|
predicted class labels (`hard` voting) or class probabilities
|
|
before averaging (`soft` voting). Uses uniform weights if `None`.
|
|
|
|
Attributes
|
|
----------
|
|
estimators_ : list of classifiers
|
|
The collection of fitted sub-estimators.
|
|
|
|
classes_ : array-like, shape = [n_predictions]
|
|
The classes labels.
|
|
|
|
Examples
|
|
--------
|
|
>>> import numpy as np
|
|
>>> from sklearn.linear_model import LogisticRegression
|
|
>>> from sklearn.naive_bayes import GaussianNB
|
|
>>> from sklearn.ensemble import RandomForestClassifier
|
|
>>> clf1 = LogisticRegression(random_state=1)
|
|
>>> clf2 = RandomForestClassifier(random_state=1)
|
|
>>> clf3 = GaussianNB()
|
|
>>> X = np.array([[-1, -1], [-2, -1], [-3, -2], [1, 1], [2, 1], [3, 2]])
|
|
>>> y = np.array([1, 1, 1, 2, 2, 2])
|
|
>>> eclf1 = VotingClassifier(estimators=[
|
|
... ('lr', clf1), ('rf', clf2), ('gnb', clf3)], voting='hard')
|
|
>>> eclf1 = eclf1.fit(X, y)
|
|
>>> print(eclf1.predict(X))
|
|
[1 1 1 2 2 2]
|
|
>>> eclf2 = VotingClassifier(estimators=[
|
|
... ('lr', clf1), ('rf', clf2), ('gnb', clf3)],
|
|
... voting='soft')
|
|
>>> eclf2 = eclf2.fit(X, y)
|
|
>>> print(eclf2.predict(X))
|
|
[1 1 1 2 2 2]
|
|
>>> eclf3 = VotingClassifier(estimators=[
|
|
... ('lr', clf1), ('rf', clf2), ('gnb', clf3)],
|
|
... voting='soft', weights=[2,1,1])
|
|
>>> eclf3 = eclf3.fit(X, y)
|
|
>>> print(eclf3.predict(X))
|
|
[1 1 1 2 2 2]
|
|
>>>
|
|
"""
|
|
|
|
def __init__(self, estimators, voting='hard', weights=None):
|
|
|
|
self.estimators = estimators
|
|
self.named_estimators = dict(estimators)
|
|
self.voting = voting
|
|
self.weights = weights
|
|
|
|
def fit(self, X, y):
|
|
""" Fit the estimators.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}, shape = [n_samples, n_features]
|
|
Training vectors, where n_samples is the number of samples and
|
|
n_features is the number of features.
|
|
|
|
y : array-like, shape = [n_samples]
|
|
Target values.
|
|
|
|
Returns
|
|
-------
|
|
self : object
|
|
"""
|
|
if isinstance(y, np.ndarray) and len(y.shape) > 1 and y.shape[1] > 1:
|
|
raise NotImplementedError('Multilabel and multi-output'
|
|
' classification is not supported.')
|
|
|
|
if self.voting not in ('soft', 'hard'):
|
|
raise ValueError("Voting must be 'soft' or 'hard'; got (voting=%r)"
|
|
% self.voting)
|
|
|
|
if self.estimators is None or len(self.estimators) == 0:
|
|
raise AttributeError('Invalid `estimators` attribute, `estimators`'
|
|
' should be a list of (string, estimator)'
|
|
' tuples')
|
|
|
|
if self.weights and len(self.weights) != len(self.estimators):
|
|
raise ValueError('Number of classifiers and weights must be equal'
|
|
'; got %d weights, %d estimators'
|
|
% (len(self.weights), len(self.estimators)))
|
|
|
|
self.le_ = LabelEncoder()
|
|
self.le_.fit(y)
|
|
self.classes_ = self.le_.classes_
|
|
self.estimators_ = []
|
|
|
|
for name, clf in self.estimators:
|
|
fitted_clf = clone(clf).fit(X, self.le_.transform(y))
|
|
self.estimators_.append(fitted_clf)
|
|
|
|
return self
|
|
|
|
def predict(self, X):
|
|
""" Predict class labels for X.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}, shape = [n_samples, n_features]
|
|
Training vectors, where n_samples is the number of samples and
|
|
n_features is the number of features.
|
|
|
|
Returns
|
|
----------
|
|
maj : array-like, shape = [n_samples]
|
|
Predicted class labels.
|
|
"""
|
|
|
|
check_is_fitted(self, 'estimators_')
|
|
if self.voting == 'soft':
|
|
maj = np.argmax(self.predict_proba(X), axis=1)
|
|
|
|
else: # 'hard' voting
|
|
predictions = self._predict(X)
|
|
maj = np.apply_along_axis(lambda x:
|
|
np.argmax(np.bincount(x,
|
|
weights=self.weights)),
|
|
axis=1,
|
|
arr=predictions.astype('int'))
|
|
|
|
maj = self.le_.inverse_transform(maj)
|
|
|
|
return maj
|
|
|
|
def _collect_probas(self, X):
|
|
"""Collect results from clf.predict calls. """
|
|
return np.asarray([clf.predict_proba(X) for clf in self.estimators_])
|
|
|
|
def _predict_proba(self, X):
|
|
"""Predict class probabilities for X in 'soft' voting """
|
|
if self.voting == 'hard':
|
|
raise AttributeError("predict_proba is not available when"
|
|
" voting=%r" % self.voting)
|
|
check_is_fitted(self, 'estimators_')
|
|
avg = np.average(self._collect_probas(X), axis=0, weights=self.weights)
|
|
return avg
|
|
|
|
@property
|
|
def predict_proba(self):
|
|
"""Compute probabilities of possible outcomes for samples in X.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}, shape = [n_samples, n_features]
|
|
Training vectors, where n_samples is the number of samples and
|
|
n_features is the number of features.
|
|
|
|
Returns
|
|
----------
|
|
avg : array-like, shape = [n_samples, n_classes]
|
|
Weighted average probability for each class per sample.
|
|
"""
|
|
return self._predict_proba
|
|
|
|
def transform(self, X):
|
|
"""Return class labels or probabilities for X for each estimator.
|
|
|
|
Parameters
|
|
----------
|
|
X : {array-like, sparse matrix}, shape = [n_samples, n_features]
|
|
Training vectors, where n_samples is the number of samples and
|
|
n_features is the number of features.
|
|
|
|
Returns
|
|
-------
|
|
If `voting='soft'`:
|
|
array-like = [n_classifiers, n_samples, n_classes]
|
|
Class probabilities calculated by each classifier.
|
|
If `voting='hard'`:
|
|
array-like = [n_classifiers, n_samples]
|
|
Class labels predicted by each classifier.
|
|
"""
|
|
check_is_fitted(self, 'estimators_')
|
|
if self.voting == 'soft':
|
|
return self._collect_probas(X)
|
|
else:
|
|
return self._predict(X)
|
|
|
|
def get_params(self, deep=True):
|
|
"""Return estimator parameter names for GridSearch support"""
|
|
if not deep:
|
|
return super(VotingClassifier, self).get_params(deep=False)
|
|
else:
|
|
out = super(VotingClassifier, self).get_params(deep=False)
|
|
out.update(self.named_estimators.copy())
|
|
for name, step in six.iteritems(self.named_estimators):
|
|
for key, value in six.iteritems(step.get_params(deep=True)):
|
|
out['%s__%s' % (name, key)] = value
|
|
return out
|
|
|
|
def _predict(self, X):
|
|
"""Collect results from clf.predict calls. """
|
|
return np.asarray([clf.predict(X) for clf in self.estimators_]).T
|