From 87759c19243ea8e3773d494c04331f6817fc55ff Mon Sep 17 00:00:00 2001 From: Kumar Ashutosh Date: Tue, 12 Dec 2017 03:30:57 +0530 Subject: [PATCH] FEA Add a new class RegressorChain similar to ClassifierChain (#9257) --- doc/modules/classes.rst | 1 + doc/modules/multiclass.rst | 12 ++ doc/whats_new/v0.20.rst | 3 + sklearn/multioutput.py | 239 +++++++++++++++++++++--------- sklearn/tests/test_multioutput.py | 150 +++++++++---------- sklearn/utils/testing.py | 3 +- 6 files changed, 263 insertions(+), 145 deletions(-) diff --git a/doc/modules/classes.rst b/doc/modules/classes.rst index feb050e20ad..2792ba84846 100644 --- a/doc/modules/classes.rst +++ b/doc/modules/classes.rst @@ -1064,6 +1064,7 @@ Model validation multioutput.ClassifierChain multioutput.MultiOutputRegressor multioutput.MultiOutputClassifier + multioutput.RegressorChain .. _naive_bayes_ref: diff --git a/doc/modules/multiclass.rst b/doc/modules/multiclass.rst index 93e4c1a6c36..924cff8e137 100644 --- a/doc/modules/multiclass.rst +++ b/doc/modules/multiclass.rst @@ -397,6 +397,8 @@ Below is an example of multioutput classification: [0, 0, 2], [2, 0, 0]]) +.. classifierchain: + Classifier Chain ================ @@ -425,3 +427,13 @@ averaged together. Jesse Read, Bernhard Pfahringer, Geoff Holmes, Eibe Frank, "Classifier Chains for Multi-label Classification", 2009. + +.. regressorchain: + +Regressor Chain +================ + +Regressor chains (see :class:`RegressorChain`) is analogous to +ClassifierChain as a way of combining a number of regressions +into a single multi-target model that is capable of exploiting +correlations among targets. \ No newline at end of file diff --git a/doc/whats_new/v0.20.rst b/doc/whats_new/v0.20.rst index 12c34d5f972..be0b6a0d85d 100644 --- a/doc/whats_new/v0.20.rst +++ b/doc/whats_new/v0.20.rst @@ -74,6 +74,9 @@ Model evaluation ``'balanced_accuracy'`` scorer for binary classification. :issue:`8066` by :user:`xyguo` and :user:`Aman Dalmia `. +- Added :class:`multioutput.RegressorChain` for multi-target + regression. :issue:`9257` by :user:`Kumar Ashutosh `. + Enhancements ............ diff --git a/sklearn/multioutput.py b/sklearn/multioutput.py index 5b4389fd0f3..036940b1f63 100644 --- a/sklearn/multioutput.py +++ b/sklearn/multioutput.py @@ -28,7 +28,8 @@ from .utils.multiclass import check_classification_targets from .externals.joblib import Parallel, delayed from .externals import six -__all__ = ["MultiOutputRegressor", "MultiOutputClassifier", "ClassifierChain"] +__all__ = ["MultiOutputRegressor", "MultiOutputClassifier", + "ClassifierChain", "RegressorChain"] def _fit_estimator(estimator, X, y, sample_weight=None): @@ -368,77 +369,14 @@ class MultiOutputClassifier(MultiOutputEstimator, ClassifierMixin): return np.mean(np.all(y == y_pred, axis=1)) -class ClassifierChain(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): - """A multi-label model that arranges binary classifiers into a chain. - - Each model makes a prediction in the order specified by the chain using - all of the available features provided to the model plus the predictions - of models that are earlier in the chain. - - Parameters - ---------- - base_estimator : estimator - The base estimator from which the classifier chain is built. - - order : array-like, shape=[n_outputs] or 'random', optional - By default the order will be determined by the order of columns in - the label matrix Y.:: - - order = [0, 1, 2, ..., Y.shape[1] - 1] - - The order of the chain can be explicitly set by providing a list of - integers. For example, for a chain of length 5.:: - - order = [1, 3, 2, 4, 0] - - means that the first model in the chain will make predictions for - column 1 in the Y matrix, the second model will make predictions - for column 3, etc. - - If order is 'random' a random ordering will be used. - - cv : int, cross-validation generator or an iterable, optional ( - default=None) - Determines whether to use cross validated predictions or true - labels for the results of previous estimators in the chain. - If cv is None the true labels are used when fitting. Otherwise - possible inputs for cv are: - * integer, to specify the number of folds in a (Stratified)KFold, - * An object to be used as a cross-validation generator. - * An iterable yielding train, test splits. - - random_state : int, RandomState instance or None, optional (default=None) - If int, random_state is the seed used by the random number generator; - If RandomState instance, random_state is the random number generator; - If None, the random number generator is the RandomState instance used - by `np.random`. - - The random number generator is used to generate random chain orders. - - Attributes - ---------- - classes_ : list - A list of arrays of length ``len(estimators_)`` containing the - class labels for each estimator in the chain. - - estimators_ : list - A list of clones of base_estimator. - - order_ : list - The order of labels in the classifier chain. - - References - ---------- - Jesse Read, Bernhard Pfahringer, Geoff Holmes, Eibe Frank, "Classifier - Chains for Multi-label Classification", 2009. - - """ +class _BaseChain(six.with_metaclass(ABCMeta, BaseEstimator)): def __init__(self, base_estimator, order=None, cv=None, random_state=None): self.base_estimator = base_estimator self.order = order self.cv = cv self.random_state = random_state + @abstractmethod def fit(self, X, Y): """Fit the model to data matrix X and targets Y. @@ -470,8 +408,6 @@ class ClassifierChain(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): self.estimators_ = [clone(self.base_estimator) for _ in range(Y.shape[1])] - self.classes_ = [] - if self.cv is None: Y_pred_chain = Y[:, self.order_] if sp.issparse(X): @@ -503,7 +439,6 @@ class ClassifierChain(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): else: X_aug[:, col_idx] = cv_result - self.classes_.append(estimator.classes_) return self def predict(self, X): @@ -539,6 +474,95 @@ class ClassifierChain(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): return Y_pred + +class ClassifierChain(_BaseChain, ClassifierMixin, MetaEstimatorMixin): + """A multi-label model that arranges binary classifiers into a chain. + + Each model makes a prediction in the order specified by the chain using + all of the available features provided to the model plus the predictions + of models that are earlier in the chain. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + base_estimator : estimator + The base estimator from which the classifier chain is built. + + order : array-like, shape=[n_outputs] or 'random', optional + By default the order will be determined by the order of columns in + the label matrix Y.:: + + order = [0, 1, 2, ..., Y.shape[1] - 1] + + The order of the chain can be explicitly set by providing a list of + integers. For example, for a chain of length 5.:: + + order = [1, 3, 2, 4, 0] + + means that the first model in the chain will make predictions for + column 1 in the Y matrix, the second model will make predictions + for column 3, etc. + + If order is 'random' a random ordering will be used. + + cv : int, cross-validation generator or an iterable, optional \ + (default=None) + Determines whether to use cross validated predictions or true + labels for the results of previous estimators in the chain. + If cv is None the true labels are used when fitting. Otherwise + possible inputs for cv are: + * integer, to specify the number of folds in a (Stratified)KFold, + * An object to be used as a cross-validation generator. + * An iterable yielding train, test splits. + + random_state : int, RandomState instance or None, optional (default=None) + If int, random_state is the seed used by the random number generator; + If RandomState instance, random_state is the random number generator; + If None, the random number generator is the RandomState instance used + by `np.random`. + + The random number generator is used to generate random chain orders. + + Attributes + ---------- + classes_ : list + A list of arrays of length ``len(estimators_)`` containing the + class labels for each estimator in the chain. + + estimators_ : list + A list of clones of base_estimator. + + order_ : list + The order of labels in the classifier chain. + + References + ---------- + Jesse Read, Bernhard Pfahringer, Geoff Holmes, Eibe Frank, "Classifier + Chains for Multi-label Classification", 2009. + + """ + + def fit(self, X, Y): + """Fit the model to data matrix X and targets Y. + Parameters + ---------- + X : {array-like, sparse matrix}, shape (n_samples, n_features) + The input data. + Y : array-like, shape (n_samples, n_classes) + The target values. + + Returns + ------- + self : object + Returns self. + """ + super(ClassifierChain, self).fit(X, Y) + self.classes_ = [] + for chain_idx, estimator in enumerate(self.estimators_): + self.classes_.append(estimator.classes_) + return self + @if_delegate_has_method('base_estimator') def predict_proba(self, X): """Predict probability estimates. @@ -598,3 +622,80 @@ class ClassifierChain(BaseEstimator, ClassifierMixin, MetaEstimatorMixin): Y_decision = Y_decision_chain[:, inv_order] return Y_decision + + +class RegressorChain(_BaseChain, RegressorMixin, MetaEstimatorMixin): + """A multi-label model that arranges regressions into a chain. + + Each model makes a prediction in the order specified by the chain using + all of the available features provided to the model plus the predictions + of models that are earlier in the chain. + + Read more in the :ref:`User Guide `. + + Parameters + ---------- + base_estimator : estimator + The base estimator from which the classifier chain is built. + + order : array-like, shape=[n_outputs] or 'random', optional + By default the order will be determined by the order of columns in + the label matrix Y.:: + + order = [0, 1, 2, ..., Y.shape[1] - 1] + + The order of the chain can be explicitly set by providing a list of + integers. For example, for a chain of length 5.:: + + order = [1, 3, 2, 4, 0] + + means that the first model in the chain will make predictions for + column 1 in the Y matrix, the second model will make predictions + for column 3, etc. + + If order is 'random' a random ordering will be used. + + cv : int, cross-validation generator or an iterable, optional \ + (default=None) + Determines whether to use cross validated predictions or true + labels for the results of previous estimators in the chain. + If cv is None the true labels are used when fitting. Otherwise + possible inputs for cv are: + * integer, to specify the number of folds in a (Stratified)KFold, + * An object to be used as a cross-validation generator. + * An iterable yielding train, test splits. + + random_state : int, RandomState instance or None, optional (default=None) + If int, random_state is the seed used by the random number generator; + If RandomState instance, random_state is the random number generator; + If None, the random number generator is the RandomState instance used + by `np.random`. + + The random number generator is used to generate random chain orders. + + Attributes + ---------- + estimators_ : list + A list of clones of base_estimator. + + order_ : list + The order of labels in the classifier chain. + + """ + def fit(self, X, Y): + """Fit the model to data matrix X and targets Y. + + Parameters + ---------- + X : {array-like, sparse matrix}, shape (n_samples, n_features) + The input data. + Y : array-like, shape (n_samples, n_classes) + The target values. + + Returns + ------- + self : object + Returns self. + """ + super(RegressorChain, self).fit(X, Y) + return self diff --git a/sklearn/tests/test_multioutput.py b/sklearn/tests/test_multioutput.py index 26981d20fc6..23529c650ec 100644 --- a/sklearn/tests/test_multioutput.py +++ b/sklearn/tests/test_multioutput.py @@ -21,11 +21,12 @@ from sklearn.exceptions import NotFittedError from sklearn.externals.joblib import cpu_count from sklearn.linear_model import Lasso from sklearn.linear_model import LogisticRegression +from sklearn.linear_model import Ridge from sklearn.linear_model import SGDClassifier from sklearn.linear_model import SGDRegressor -from sklearn.metrics import jaccard_similarity_score +from sklearn.metrics import jaccard_similarity_score, mean_squared_error from sklearn.multiclass import OneVsRestClassifier -from sklearn.multioutput import ClassifierChain +from sklearn.multioutput import ClassifierChain, RegressorChain from sklearn.multioutput import MultiOutputClassifier from sklearn.multioutput import MultiOutputRegressor from sklearn.svm import LinearSVC @@ -366,25 +367,6 @@ def generate_multilabel_dataset_with_correlations(): return X, Y_multi -def test_classifier_chain_fit_and_predict_with_logistic_regression(): - # Fit classifier chain and verify predict performance - X, Y = generate_multilabel_dataset_with_correlations() - classifier_chain = ClassifierChain(LogisticRegression()) - classifier_chain.fit(X, Y) - - Y_pred = classifier_chain.predict(X) - assert_equal(Y_pred.shape, Y.shape) - - Y_prob = classifier_chain.predict_proba(X) - Y_binary = (Y_prob >= .5) - assert_array_equal(Y_binary, Y_pred) - - assert_equal([c.coef_.size for c in classifier_chain.estimators_], - list(range(X.shape[1], X.shape[1] + Y.shape[1]))) - - assert isinstance(classifier_chain, ClassifierMixin) - - def test_classifier_chain_fit_and_predict_with_linear_svc(): # Fit classifier chain and verify predict performance using LinearSVC X, Y = generate_multilabel_dataset_with_correlations() @@ -417,60 +399,6 @@ def test_classifier_chain_fit_and_predict_with_sparse_data(): assert_array_equal(Y_pred_sparse, Y_pred_dense) -def test_classifier_chain_fit_and_predict_with_sparse_data_and_cv(): - # Fit classifier chain with sparse data cross_val_predict - X, Y = generate_multilabel_dataset_with_correlations() - X_sparse = sp.csr_matrix(X) - classifier_chain = ClassifierChain(LogisticRegression(), cv=3) - classifier_chain.fit(X_sparse, Y) - Y_pred = classifier_chain.predict(X_sparse) - assert_equal(Y_pred.shape, Y.shape) - - -def test_classifier_chain_random_order(): - # Fit classifier chain with random order - X, Y = generate_multilabel_dataset_with_correlations() - classifier_chain_random = ClassifierChain(LogisticRegression(), - order='random', - random_state=42) - classifier_chain_random.fit(X, Y) - Y_pred_random = classifier_chain_random.predict(X) - - assert_not_equal(list(classifier_chain_random.order), list(range(4))) - assert_equal(len(classifier_chain_random.order_), 4) - assert_equal(len(set(classifier_chain_random.order_)), 4) - - classifier_chain_fixed = \ - ClassifierChain(LogisticRegression(), - order=classifier_chain_random.order_) - classifier_chain_fixed.fit(X, Y) - Y_pred_fixed = classifier_chain_fixed.predict(X) - - # Randomly ordered chain should behave identically to a fixed order chain - # with the same order. - assert_array_equal(Y_pred_random, Y_pred_fixed) - - -def test_classifier_chain_crossval_fit_and_predict(): - # Fit classifier chain with cross_val_predict and verify predict - # performance - X, Y = generate_multilabel_dataset_with_correlations() - classifier_chain_cv = ClassifierChain(LogisticRegression(), cv=3) - classifier_chain_cv.fit(X, Y) - - classifier_chain = ClassifierChain(LogisticRegression()) - classifier_chain.fit(X, Y) - - Y_pred_cv = classifier_chain_cv.predict(X) - Y_pred = classifier_chain.predict(X) - - assert_equal(Y_pred_cv.shape, Y.shape) - assert_greater(jaccard_similarity_score(Y, Y_pred_cv), 0.4) - - assert_not_equal(jaccard_similarity_score(Y, Y_pred_cv), - jaccard_similarity_score(Y, Y_pred)) - - def test_classifier_chain_vs_independent_models(): # Verify that an ensemble of classifier chains (each of length # N) can achieve a higher Jaccard similarity score than N independent @@ -491,3 +419,75 @@ def test_classifier_chain_vs_independent_models(): assert_greater(jaccard_similarity_score(Y_test, Y_pred_chain), jaccard_similarity_score(Y_test, Y_pred_ovr)) + + +def test_base_chain_fit_and_predict(): + # Fit base chain and verify predict performance + X, Y = generate_multilabel_dataset_with_correlations() + chains = [RegressorChain(Ridge()), + ClassifierChain(LogisticRegression())] + for chain in chains: + chain.fit(X, Y) + Y_pred = chain.predict(X) + assert_equal(Y_pred.shape, Y.shape) + assert_equal([c.coef_.size for c in chain.estimators_], + list(range(X.shape[1], X.shape[1] + Y.shape[1]))) + + Y_prob = chains[1].predict_proba(X) + Y_binary = (Y_prob >= .5) + assert_array_equal(Y_binary, Y_pred) + + assert isinstance(chains[1], ClassifierMixin) + + +def test_base_chain_fit_and_predict_with_sparse_data_and_cv(): + # Fit base chain with sparse data cross_val_predict + X, Y = generate_multilabel_dataset_with_correlations() + X_sparse = sp.csr_matrix(X) + base_chains = [ClassifierChain(LogisticRegression(), cv=3), + RegressorChain(Ridge(), cv=3)] + for chain in base_chains: + chain.fit(X_sparse, Y) + Y_pred = chain.predict(X_sparse) + assert_equal(Y_pred.shape, Y.shape) + + +def test_base_chain_random_order(): + # Fit base chain with random order + X, Y = generate_multilabel_dataset_with_correlations() + for chain in [ClassifierChain(LogisticRegression()), + RegressorChain(Ridge())]: + chain_random = clone(chain).set_params(order='random', random_state=42) + chain_random.fit(X, Y) + chain_fixed = clone(chain).set_params(order=chain_random.order_) + chain_fixed.fit(X, Y) + assert_array_equal(chain_fixed.order_, chain_random.order_) + assert_not_equal(list(chain_random.order), list(range(4))) + assert_equal(len(chain_random.order_), 4) + assert_equal(len(set(chain_random.order_)), 4) + # Randomly ordered chain should behave identically to a fixed order + # chain with the same order. + for est1, est2 in zip(chain_random.estimators_, + chain_fixed.estimators_): + assert_array_almost_equal(est1.coef_, est2.coef_) + + +def test_base_chain_crossval_fit_and_predict(): + # Fit chain with cross_val_predict and verify predict + # performance + X, Y = generate_multilabel_dataset_with_correlations() + + for chain in [ClassifierChain(LogisticRegression()), + RegressorChain(Ridge())]: + chain.fit(X, Y) + chain_cv = clone(chain).set_params(cv=3) + chain_cv.fit(X, Y) + Y_pred_cv = chain_cv.predict(X) + Y_pred = chain.predict(X) + + assert Y_pred_cv.shape == Y_pred.shape + assert not np.all(Y_pred == Y_pred_cv) + if isinstance(chain, ClassifierChain): + assert jaccard_similarity_score(Y, Y_pred_cv) > .4 + else: + assert mean_squared_error(Y, Y_pred_cv) < .25 diff --git a/sklearn/utils/testing.py b/sklearn/utils/testing.py index 90994b71b78..e9cf9083c3e 100644 --- a/sklearn/utils/testing.py +++ b/sklearn/utils/testing.py @@ -511,7 +511,8 @@ def uninstall_mldata_mock(): META_ESTIMATORS = ["OneVsOneClassifier", "MultiOutputEstimator", "MultiOutputRegressor", "MultiOutputClassifier", "OutputCodeClassifier", "OneVsRestClassifier", - "RFE", "RFECV", "BaseEnsemble", "ClassifierChain"] + "RFE", "RFECV", "BaseEnsemble", "ClassifierChain", + "RegressorChain"] # estimators that there is no way to default-construct sensibly OTHER = ["Pipeline", "FeatureUnion", "GridSearchCV", "RandomizedSearchCV", "SelectFromModel"]