baohan
/
scikit-learn
mirror of https://github.com/scikit-learn/scikit-learn.git
0
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

FEA Add a new class RegressorChain similar to ClassifierChain (#9257)

This commit is contained in:
Kumar Ashutosh 2017-12-12 03:30:57 +05:30 committed by Joel Nothman
parent 26a3ed01e1
commit 87759c1924
6 changed files with 263 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
doc/modules/classes.rst
View File

@ -1064,6 +1064,7 @@ Model validation
multioutput.ClassifierChain
multioutput.MultiOutputRegressor
multioutput.MultiOutputClassifier
multioutput.RegressorChain
.. _naive_bayes_ref:

12
doc/modules/multiclass.rst
View File

@ -397,6 +397,8 @@ Below is an example of multioutput classification:
[0, 0, 2],
[2, 0, 0]])
.. classifierchain:
Classifier Chain
================
@ -425,3 +427,13 @@ averaged together.
Jesse Read, Bernhard Pfahringer, Geoff Holmes, Eibe Frank,
"Classifier Chains for Multi-label Classification", 2009.
.. regressorchain:
Regressor Chain
================
Regressor chains (see :class:`RegressorChain`) is analogous to
ClassifierChain as a way of combining a number of regressions
into a single multi-target model that is capable of exploiting
correlations among targets.

3
doc/whats_new/v0.20.rst
View File

@ -74,6 +74,9 @@ Model evaluation
``'balanced_accuracy'`` scorer for binary classification.
:issue:`8066` by :user:`xyguo` and :user:`Aman Dalmia <dalmia>`.
- Added :class:`multioutput.RegressorChain` for multi-target
regression. :issue:`9257` by :user:`Kumar Ashutosh <thechargedneutron>`.
Enhancements
............

239
sklearn/multioutput.py
View File

@ -28,7 +28,8 @@ from .utils.multiclass import check_classification_targets
from .externals.joblib import Parallel, delayed
from .externals import six
__all__ = ["MultiOutputRegressor", "MultiOutputClassifier", "ClassifierChain"]
__all__ = ["MultiOutputRegressor", "MultiOutputClassifier",
"ClassifierChain", "RegressorChain"]
def _fit_estimator(estimator, X, y, sample_weight=None):
@ -368,77 +369,14 @@ class MultiOutputClassifier(MultiOutputEstimator, ClassifierMixin):
return np.mean(np.all(y == y_pred, axis=1))
class ClassifierChain(BaseEstimator, ClassifierMixin, MetaEstimatorMixin):
"""A multi-label model that arranges binary classifiers into a chain.
Each model makes a prediction in the order specified by the chain using
all of the available features provided to the model plus the predictions
of models that are earlier in the chain.
Parameters
----------
base_estimator : estimator
The base estimator from which the classifier chain is built.
order : array-like, shape=[n_outputs] or 'random', optional
By default the order will be determined by the order of columns in
the label matrix Y.::
order = [0, 1, 2, ..., Y.shape[1] - 1]
The order of the chain can be explicitly set by providing a list of
integers. For example, for a chain of length 5.::
order = [1, 3, 2, 4, 0]
means that the first model in the chain will make predictions for
column 1 in the Y matrix, the second model will make predictions
for column 3, etc.
If order is 'random' a random ordering will be used.
cv : int, cross-validation generator or an iterable, optional (
default=None)
Determines whether to use cross validated predictions or true
labels for the results of previous estimators in the chain.
If cv is None the true labels are used when fitting. Otherwise
possible inputs for cv are:
* integer, to specify the number of folds in a (Stratified)KFold,
* An object to be used as a cross-validation generator.
* An iterable yielding train, test splits.
random_state : int, RandomState instance or None, optional (default=None)
If int, random_state is the seed used by the random number generator;
If RandomState instance, random_state is the random number generator;
If None, the random number generator is the RandomState instance used
by `np.random`.
The random number generator is used to generate random chain orders.
Attributes
----------
classes_ : list
A list of arrays of length ``len(estimators_)`` containing the
class labels for each estimator in the chain.
estimators_ : list
A list of clones of base_estimator.
order_ : list
The order of labels in the classifier chain.
References
----------
Jesse Read, Bernhard Pfahringer, Geoff Holmes, Eibe Frank, "Classifier
Chains for Multi-label Classification", 2009.
"""
class _BaseChain(six.with_metaclass(ABCMeta, BaseEstimator)):
def __init__(self, base_estimator, order=None, cv=None, random_state=None):
self.base_estimator = base_estimator
self.order = order
self.cv = cv
self.random_state = random_state
@abstractmethod
def fit(self, X, Y):
"""Fit the model to data matrix X and targets Y.
@ -470,8 +408,6 @@ class ClassifierChain(BaseEstimator, ClassifierMixin, MetaEstimatorMixin):
self.estimators_ = [clone(self.base_estimator)
for _ in range(Y.shape[1])]
self.classes_ = []
if self.cv is None:
Y_pred_chain = Y[:, self.order_]
if sp.issparse(X):
@ -503,7 +439,6 @@ class ClassifierChain(BaseEstimator, ClassifierMixin, MetaEstimatorMixin):
else:
X_aug[:, col_idx] = cv_result
self.classes_.append(estimator.classes_)
return self
def predict(self, X):
@ -539,6 +474,95 @@ class ClassifierChain(BaseEstimator, ClassifierMixin, MetaEstimatorMixin):
return Y_pred
class ClassifierChain(_BaseChain, ClassifierMixin, MetaEstimatorMixin):
"""A multi-label model that arranges binary classifiers into a chain.
Each model makes a prediction in the order specified by the chain using
all of the available features provided to the model plus the predictions
of models that are earlier in the chain.
Read more in the :ref:`User Guide <classifierchain>`.
Parameters
----------
base_estimator : estimator
The base estimator from which the classifier chain is built.
order : array-like, shape=[n_outputs] or 'random', optional
By default the order will be determined by the order of columns in
the label matrix Y.::
order = [0, 1, 2, ..., Y.shape[1] - 1]
The order of the chain can be explicitly set by providing a list of
integers. For example, for a chain of length 5.::
order = [1, 3, 2, 4, 0]
means that the first model in the chain will make predictions for
column 1 in the Y matrix, the second model will make predictions
for column 3, etc.
If order is 'random' a random ordering will be used.
cv : int, cross-validation generator or an iterable, optional \
(default=None)
Determines whether to use cross validated predictions or true
labels for the results of previous estimators in the chain.
If cv is None the true labels are used when fitting. Otherwise
possible inputs for cv are:
* integer, to specify the number of folds in a (Stratified)KFold,
* An object to be used as a cross-validation generator.
* An iterable yielding train, test splits.
random_state : int, RandomState instance or None, optional (default=None)
If int, random_state is the seed used by the random number generator;
If RandomState instance, random_state is the random number generator;
If None, the random number generator is the RandomState instance used
by `np.random`.
The random number generator is used to generate random chain orders.
Attributes
----------
classes_ : list
A list of arrays of length ``len(estimators_)`` containing the
class labels for each estimator in the chain.
estimators_ : list
A list of clones of base_estimator.
order_ : list
The order of labels in the classifier chain.
References
----------
Jesse Read, Bernhard Pfahringer, Geoff Holmes, Eibe Frank, "Classifier
Chains for Multi-label Classification", 2009.
"""
def fit(self, X, Y):
"""Fit the model to data matrix X and targets Y.
Parameters
----------
X : {array-like, sparse matrix}, shape (n_samples, n_features)
The input data.
Y : array-like, shape (n_samples, n_classes)
The target values.
Returns
-------
self : object
Returns self.
"""
super(ClassifierChain, self).fit(X, Y)
self.classes_ = []
for chain_idx, estimator in enumerate(self.estimators_):
self.classes_.append(estimator.classes_)
return self
@if_delegate_has_method('base_estimator')
def predict_proba(self, X):
"""Predict probability estimates.
@ -598,3 +622,80 @@ class ClassifierChain(BaseEstimator, ClassifierMixin, MetaEstimatorMixin):
Y_decision = Y_decision_chain[:, inv_order]
return Y_decision
class RegressorChain(_BaseChain, RegressorMixin, MetaEstimatorMixin):
"""A multi-label model that arranges regressions into a chain.
Each model makes a prediction in the order specified by the chain using
all of the available features provided to the model plus the predictions
of models that are earlier in the chain.
Read more in the :ref:`User Guide <regressorchain>`.
Parameters
----------
base_estimator : estimator
The base estimator from which the classifier chain is built.
order : array-like, shape=[n_outputs] or 'random', optional
By default the order will be determined by the order of columns in
the label matrix Y.::
order = [0, 1, 2, ..., Y.shape[1] - 1]
The order of the chain can be explicitly set by providing a list of
integers. For example, for a chain of length 5.::
order = [1, 3, 2, 4, 0]
means that the first model in the chain will make predictions for
column 1 in the Y matrix, the second model will make predictions
for column 3, etc.
If order is 'random' a random ordering will be used.
cv : int, cross-validation generator or an iterable, optional \
(default=None)
Determines whether to use cross validated predictions or true
labels for the results of previous estimators in the chain.
If cv is None the true labels are used when fitting. Otherwise
possible inputs for cv are:
* integer, to specify the number of folds in a (Stratified)KFold,
* An object to be used as a cross-validation generator.
* An iterable yielding train, test splits.
random_state : int, RandomState instance or None, optional (default=None)
If int, random_state is the seed used by the random number generator;
If RandomState instance, random_state is the random number generator;
If None, the random number generator is the RandomState instance used
by `np.random`.
The random number generator is used to generate random chain orders.
Attributes
----------
estimators_ : list
A list of clones of base_estimator.
order_ : list
The order of labels in the classifier chain.
"""
def fit(self, X, Y):
"""Fit the model to data matrix X and targets Y.
Parameters
----------
X : {array-like, sparse matrix}, shape (n_samples, n_features)
The input data.
Y : array-like, shape (n_samples, n_classes)
The target values.
Returns
-------
self : object
Returns self.
"""
super(RegressorChain, self).fit(X, Y)
return self

150
sklearn/tests/test_multioutput.py
View File

@ -21,11 +21,12 @@ from sklearn.exceptions import NotFittedError
from sklearn.externals.joblib import cpu_count
from sklearn.linear_model import Lasso
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import Ridge
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import SGDRegressor
from sklearn.metrics import jaccard_similarity_score
from sklearn.metrics import jaccard_similarity_score, mean_squared_error
from sklearn.multiclass import OneVsRestClassifier
from sklearn.multioutput import ClassifierChain
from sklearn.multioutput import ClassifierChain, RegressorChain
from sklearn.multioutput import MultiOutputClassifier
from sklearn.multioutput import MultiOutputRegressor
from sklearn.svm import LinearSVC
@ -366,25 +367,6 @@ def generate_multilabel_dataset_with_correlations():
return X, Y_multi
def test_classifier_chain_fit_and_predict_with_logistic_regression():
# Fit classifier chain and verify predict performance
X, Y = generate_multilabel_dataset_with_correlations()
classifier_chain = ClassifierChain(LogisticRegression())
classifier_chain.fit(X, Y)
Y_pred = classifier_chain.predict(X)
assert_equal(Y_pred.shape, Y.shape)
Y_prob = classifier_chain.predict_proba(X)
Y_binary = (Y_prob >= .5)
assert_array_equal(Y_binary, Y_pred)
assert_equal([c.coef_.size for c in classifier_chain.estimators_],
list(range(X.shape[1], X.shape[1] + Y.shape[1])))
assert isinstance(classifier_chain, ClassifierMixin)
def test_classifier_chain_fit_and_predict_with_linear_svc():
# Fit classifier chain and verify predict performance using LinearSVC
X, Y = generate_multilabel_dataset_with_correlations()
@ -417,60 +399,6 @@ def test_classifier_chain_fit_and_predict_with_sparse_data():
assert_array_equal(Y_pred_sparse, Y_pred_dense)
def test_classifier_chain_fit_and_predict_with_sparse_data_and_cv():
# Fit classifier chain with sparse data cross_val_predict
X, Y = generate_multilabel_dataset_with_correlations()
X_sparse = sp.csr_matrix(X)
classifier_chain = ClassifierChain(LogisticRegression(), cv=3)
classifier_chain.fit(X_sparse, Y)
Y_pred = classifier_chain.predict(X_sparse)
assert_equal(Y_pred.shape, Y.shape)
def test_classifier_chain_random_order():
# Fit classifier chain with random order
X, Y = generate_multilabel_dataset_with_correlations()
classifier_chain_random = ClassifierChain(LogisticRegression(),
order='random',
random_state=42)
classifier_chain_random.fit(X, Y)
Y_pred_random = classifier_chain_random.predict(X)
assert_not_equal(list(classifier_chain_random.order), list(range(4)))
assert_equal(len(classifier_chain_random.order_), 4)
assert_equal(len(set(classifier_chain_random.order_)), 4)
classifier_chain_fixed = \
ClassifierChain(LogisticRegression(),
order=classifier_chain_random.order_)
classifier_chain_fixed.fit(X, Y)
Y_pred_fixed = classifier_chain_fixed.predict(X)
# Randomly ordered chain should behave identically to a fixed order chain
# with the same order.
assert_array_equal(Y_pred_random, Y_pred_fixed)
def test_classifier_chain_crossval_fit_and_predict():
# Fit classifier chain with cross_val_predict and verify predict
# performance
X, Y = generate_multilabel_dataset_with_correlations()
classifier_chain_cv = ClassifierChain(LogisticRegression(), cv=3)
classifier_chain_cv.fit(X, Y)
classifier_chain = ClassifierChain(LogisticRegression())
classifier_chain.fit(X, Y)
Y_pred_cv = classifier_chain_cv.predict(X)
Y_pred = classifier_chain.predict(X)
assert_equal(Y_pred_cv.shape, Y.shape)
assert_greater(jaccard_similarity_score(Y, Y_pred_cv), 0.4)
assert_not_equal(jaccard_similarity_score(Y, Y_pred_cv),
jaccard_similarity_score(Y, Y_pred))
def test_classifier_chain_vs_independent_models():
# Verify that an ensemble of classifier chains (each of length
# N) can achieve a higher Jaccard similarity score than N independent
@ -491,3 +419,75 @@ def test_classifier_chain_vs_independent_models():
assert_greater(jaccard_similarity_score(Y_test, Y_pred_chain),
jaccard_similarity_score(Y_test, Y_pred_ovr))
def test_base_chain_fit_and_predict():
# Fit base chain and verify predict performance
X, Y = generate_multilabel_dataset_with_correlations()
chains = [RegressorChain(Ridge()),
ClassifierChain(LogisticRegression())]
for chain in chains:
chain.fit(X, Y)
Y_pred = chain.predict(X)
assert_equal(Y_pred.shape, Y.shape)
assert_equal([c.coef_.size for c in chain.estimators_],
list(range(X.shape[1], X.shape[1] + Y.shape[1])))
Y_prob = chains[1].predict_proba(X)
Y_binary = (Y_prob >= .5)
assert_array_equal(Y_binary, Y_pred)
assert isinstance(chains[1], ClassifierMixin)
def test_base_chain_fit_and_predict_with_sparse_data_and_cv():
# Fit base chain with sparse data cross_val_predict
X, Y = generate_multilabel_dataset_with_correlations()
X_sparse = sp.csr_matrix(X)
base_chains = [ClassifierChain(LogisticRegression(), cv=3),
RegressorChain(Ridge(), cv=3)]
for chain in base_chains:
chain.fit(X_sparse, Y)
Y_pred = chain.predict(X_sparse)
assert_equal(Y_pred.shape, Y.shape)
def test_base_chain_random_order():
# Fit base chain with random order
X, Y = generate_multilabel_dataset_with_correlations()
for chain in [ClassifierChain(LogisticRegression()),
RegressorChain(Ridge())]:
chain_random = clone(chain).set_params(order='random', random_state=42)
chain_random.fit(X, Y)
chain_fixed = clone(chain).set_params(order=chain_random.order_)
chain_fixed.fit(X, Y)
assert_array_equal(chain_fixed.order_, chain_random.order_)
assert_not_equal(list(chain_random.order), list(range(4)))
assert_equal(len(chain_random.order_), 4)
assert_equal(len(set(chain_random.order_)), 4)
# Randomly ordered chain should behave identically to a fixed order
# chain with the same order.
for est1, est2 in zip(chain_random.estimators_,
chain_fixed.estimators_):
assert_array_almost_equal(est1.coef_, est2.coef_)
def test_base_chain_crossval_fit_and_predict():
# Fit chain with cross_val_predict and verify predict
# performance
X, Y = generate_multilabel_dataset_with_correlations()
for chain in [ClassifierChain(LogisticRegression()),
RegressorChain(Ridge())]:
chain.fit(X, Y)
chain_cv = clone(chain).set_params(cv=3)
chain_cv.fit(X, Y)
Y_pred_cv = chain_cv.predict(X)
Y_pred = chain.predict(X)
assert Y_pred_cv.shape == Y_pred.shape
assert not np.all(Y_pred == Y_pred_cv)
if isinstance(chain, ClassifierChain):
assert jaccard_similarity_score(Y, Y_pred_cv) > .4
else:
assert mean_squared_error(Y, Y_pred_cv) < .25

3
sklearn/utils/testing.py
View File

@ -511,7 +511,8 @@ def uninstall_mldata_mock():
META_ESTIMATORS = ["OneVsOneClassifier", "MultiOutputEstimator",
"MultiOutputRegressor", "MultiOutputClassifier",
"OutputCodeClassifier", "OneVsRestClassifier",
"RFE", "RFECV", "BaseEnsemble", "ClassifierChain"]
"RFE", "RFECV", "BaseEnsemble", "ClassifierChain",
"RegressorChain"]
# estimators that there is no way to default-construct sensibly
OTHER = ["Pipeline", "FeatureUnion", "GridSearchCV", "RandomizedSearchCV",
"SelectFromModel"]