416 lines
16 KiB
Python
416 lines
16 KiB
Python
import pickle
|
|
import tempfile
|
|
import shutil
|
|
import os
|
|
import numbers
|
|
|
|
import numpy as np
|
|
|
|
from sklearn.utils.testing import assert_almost_equal
|
|
from sklearn.utils.testing import assert_array_equal
|
|
from sklearn.utils.testing import assert_raises
|
|
from sklearn.utils.testing import assert_raises_regexp
|
|
from sklearn.utils.testing import assert_true
|
|
from sklearn.utils.testing import ignore_warnings
|
|
from sklearn.utils.testing import assert_not_equal
|
|
|
|
from sklearn.base import BaseEstimator
|
|
from sklearn.metrics import (f1_score, r2_score, roc_auc_score, fbeta_score,
|
|
log_loss, precision_score, recall_score)
|
|
from sklearn.metrics.cluster import adjusted_rand_score
|
|
from sklearn.metrics.scorer import (check_scoring, _PredictScorer,
|
|
_passthrough_scorer)
|
|
from sklearn.metrics import make_scorer, get_scorer, SCORERS
|
|
from sklearn.svm import LinearSVC
|
|
from sklearn.pipeline import make_pipeline
|
|
from sklearn.cluster import KMeans
|
|
from sklearn.dummy import DummyRegressor
|
|
from sklearn.linear_model import Ridge, LogisticRegression
|
|
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
|
|
from sklearn.datasets import make_blobs
|
|
from sklearn.datasets import make_classification
|
|
from sklearn.datasets import make_multilabel_classification
|
|
from sklearn.datasets import load_diabetes
|
|
from sklearn.model_selection import train_test_split, cross_val_score
|
|
from sklearn.model_selection import GridSearchCV
|
|
from sklearn.multiclass import OneVsRestClassifier
|
|
from sklearn.externals import joblib
|
|
|
|
|
|
REGRESSION_SCORERS = ['r2', 'mean_absolute_error', 'mean_squared_error',
|
|
'median_absolute_error']
|
|
|
|
CLF_SCORERS = ['accuracy', 'f1', 'f1_weighted', 'f1_macro', 'f1_micro',
|
|
'roc_auc', 'average_precision', 'precision',
|
|
'precision_weighted', 'precision_macro', 'precision_micro',
|
|
'recall', 'recall_weighted', 'recall_macro', 'recall_micro',
|
|
'log_loss',
|
|
'adjusted_rand_score' # not really, but works
|
|
]
|
|
|
|
MULTILABEL_ONLY_SCORERS = ['precision_samples', 'recall_samples', 'f1_samples']
|
|
|
|
|
|
def _make_estimators(X_train, y_train, y_ml_train):
|
|
# Make estimators that make sense to test various scoring methods
|
|
sensible_regr = DummyRegressor(strategy='median')
|
|
sensible_regr.fit(X_train, y_train)
|
|
sensible_clf = DecisionTreeClassifier(random_state=0)
|
|
sensible_clf.fit(X_train, y_train)
|
|
sensible_ml_clf = DecisionTreeClassifier(random_state=0)
|
|
sensible_ml_clf.fit(X_train, y_ml_train)
|
|
return dict(
|
|
[(name, sensible_regr) for name in REGRESSION_SCORERS] +
|
|
[(name, sensible_clf) for name in CLF_SCORERS] +
|
|
[(name, sensible_ml_clf) for name in MULTILABEL_ONLY_SCORERS]
|
|
)
|
|
|
|
|
|
X_mm, y_mm, y_ml_mm = None, None, None
|
|
ESTIMATORS = None
|
|
TEMP_FOLDER = None
|
|
|
|
|
|
def setup_module():
|
|
# Create some memory mapped data
|
|
global X_mm, y_mm, y_ml_mm, TEMP_FOLDER, ESTIMATORS
|
|
TEMP_FOLDER = tempfile.mkdtemp(prefix='sklearn_test_score_objects_')
|
|
X, y = make_classification(n_samples=30, n_features=5, random_state=0)
|
|
_, y_ml = make_multilabel_classification(n_samples=X.shape[0],
|
|
random_state=0)
|
|
filename = os.path.join(TEMP_FOLDER, 'test_data.pkl')
|
|
joblib.dump((X, y, y_ml), filename)
|
|
X_mm, y_mm, y_ml_mm = joblib.load(filename, mmap_mode='r')
|
|
ESTIMATORS = _make_estimators(X_mm, y_mm, y_ml_mm)
|
|
|
|
|
|
def teardown_module():
|
|
global X_mm, y_mm, y_ml_mm, TEMP_FOLDER, ESTIMATORS
|
|
# GC closes the mmap file descriptors
|
|
X_mm, y_mm, y_ml_mm, ESTIMATORS = None, None, None, None
|
|
shutil.rmtree(TEMP_FOLDER)
|
|
|
|
|
|
class EstimatorWithoutFit(object):
|
|
"""Dummy estimator to test check_scoring"""
|
|
pass
|
|
|
|
|
|
class EstimatorWithFit(BaseEstimator):
|
|
"""Dummy estimator to test check_scoring"""
|
|
def fit(self, X, y):
|
|
return self
|
|
|
|
|
|
class EstimatorWithFitAndScore(object):
|
|
"""Dummy estimator to test check_scoring"""
|
|
def fit(self, X, y):
|
|
return self
|
|
|
|
def score(self, X, y):
|
|
return 1.0
|
|
|
|
|
|
class EstimatorWithFitAndPredict(object):
|
|
"""Dummy estimator to test check_scoring"""
|
|
def fit(self, X, y):
|
|
self.y = y
|
|
return self
|
|
|
|
def predict(self, X):
|
|
return self.y
|
|
|
|
|
|
class DummyScorer(object):
|
|
"""Dummy scorer that always returns 1."""
|
|
def __call__(self, est, X, y):
|
|
return 1
|
|
|
|
|
|
def test_all_scorers_repr():
|
|
# Test that all scorers have a working repr
|
|
for name, scorer in SCORERS.items():
|
|
repr(scorer)
|
|
|
|
|
|
def test_check_scoring():
|
|
# Test all branches of check_scoring
|
|
estimator = EstimatorWithoutFit()
|
|
pattern = (r"estimator should be an estimator implementing 'fit' method,"
|
|
r" .* was passed")
|
|
assert_raises_regexp(TypeError, pattern, check_scoring, estimator)
|
|
|
|
estimator = EstimatorWithFitAndScore()
|
|
estimator.fit([[1]], [1])
|
|
scorer = check_scoring(estimator)
|
|
assert_true(scorer is _passthrough_scorer)
|
|
assert_almost_equal(scorer(estimator, [[1]], [1]), 1.0)
|
|
|
|
estimator = EstimatorWithFitAndPredict()
|
|
estimator.fit([[1]], [1])
|
|
pattern = (r"If no scoring is specified, the estimator passed should have"
|
|
r" a 'score' method\. The estimator .* does not\.")
|
|
assert_raises_regexp(TypeError, pattern, check_scoring, estimator)
|
|
|
|
scorer = check_scoring(estimator, "accuracy")
|
|
assert_almost_equal(scorer(estimator, [[1]], [1]), 1.0)
|
|
|
|
estimator = EstimatorWithFit()
|
|
scorer = check_scoring(estimator, "accuracy")
|
|
assert_true(isinstance(scorer, _PredictScorer))
|
|
|
|
estimator = EstimatorWithFit()
|
|
scorer = check_scoring(estimator, allow_none=True)
|
|
assert_true(scorer is None)
|
|
|
|
|
|
def test_check_scoring_gridsearchcv():
|
|
# test that check_scoring works on GridSearchCV and pipeline.
|
|
# slightly redundant non-regression test.
|
|
|
|
grid = GridSearchCV(LinearSVC(), param_grid={'C': [.1, 1]})
|
|
scorer = check_scoring(grid, "f1")
|
|
assert_true(isinstance(scorer, _PredictScorer))
|
|
|
|
pipe = make_pipeline(LinearSVC())
|
|
scorer = check_scoring(pipe, "f1")
|
|
assert_true(isinstance(scorer, _PredictScorer))
|
|
|
|
# check that cross_val_score definitely calls the scorer
|
|
# and doesn't make any assumptions about the estimator apart from having a
|
|
# fit.
|
|
scores = cross_val_score(EstimatorWithFit(), [[1], [2], [3]], [1, 0, 1],
|
|
scoring=DummyScorer())
|
|
assert_array_equal(scores, 1)
|
|
|
|
|
|
def test_make_scorer():
|
|
# Sanity check on the make_scorer factory function.
|
|
f = lambda *args: 0
|
|
assert_raises(ValueError, make_scorer, f, needs_threshold=True,
|
|
needs_proba=True)
|
|
|
|
|
|
def test_classification_scores():
|
|
# Test classification scorers.
|
|
X, y = make_blobs(random_state=0, centers=2)
|
|
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
|
|
clf = LinearSVC(random_state=0)
|
|
clf.fit(X_train, y_train)
|
|
|
|
for prefix, metric in [('f1', f1_score), ('precision', precision_score),
|
|
('recall', recall_score)]:
|
|
|
|
score1 = get_scorer('%s_weighted' % prefix)(clf, X_test, y_test)
|
|
score2 = metric(y_test, clf.predict(X_test), pos_label=None,
|
|
average='weighted')
|
|
assert_almost_equal(score1, score2)
|
|
|
|
score1 = get_scorer('%s_macro' % prefix)(clf, X_test, y_test)
|
|
score2 = metric(y_test, clf.predict(X_test), pos_label=None,
|
|
average='macro')
|
|
assert_almost_equal(score1, score2)
|
|
|
|
score1 = get_scorer('%s_micro' % prefix)(clf, X_test, y_test)
|
|
score2 = metric(y_test, clf.predict(X_test), pos_label=None,
|
|
average='micro')
|
|
assert_almost_equal(score1, score2)
|
|
|
|
score1 = get_scorer('%s' % prefix)(clf, X_test, y_test)
|
|
score2 = metric(y_test, clf.predict(X_test), pos_label=1)
|
|
assert_almost_equal(score1, score2)
|
|
|
|
# test fbeta score that takes an argument
|
|
scorer = make_scorer(fbeta_score, beta=2)
|
|
score1 = scorer(clf, X_test, y_test)
|
|
score2 = fbeta_score(y_test, clf.predict(X_test), beta=2)
|
|
assert_almost_equal(score1, score2)
|
|
|
|
# test that custom scorer can be pickled
|
|
unpickled_scorer = pickle.loads(pickle.dumps(scorer))
|
|
score3 = unpickled_scorer(clf, X_test, y_test)
|
|
assert_almost_equal(score1, score3)
|
|
|
|
# smoke test the repr:
|
|
repr(fbeta_score)
|
|
|
|
|
|
def test_regression_scorers():
|
|
# Test regression scorers.
|
|
diabetes = load_diabetes()
|
|
X, y = diabetes.data, diabetes.target
|
|
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
|
|
clf = Ridge()
|
|
clf.fit(X_train, y_train)
|
|
score1 = get_scorer('r2')(clf, X_test, y_test)
|
|
score2 = r2_score(y_test, clf.predict(X_test))
|
|
assert_almost_equal(score1, score2)
|
|
|
|
|
|
def test_thresholded_scorers():
|
|
# Test scorers that take thresholds.
|
|
X, y = make_blobs(random_state=0, centers=2)
|
|
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
|
|
clf = LogisticRegression(random_state=0)
|
|
clf.fit(X_train, y_train)
|
|
score1 = get_scorer('roc_auc')(clf, X_test, y_test)
|
|
score2 = roc_auc_score(y_test, clf.decision_function(X_test))
|
|
score3 = roc_auc_score(y_test, clf.predict_proba(X_test)[:, 1])
|
|
assert_almost_equal(score1, score2)
|
|
assert_almost_equal(score1, score3)
|
|
|
|
logscore = get_scorer('log_loss')(clf, X_test, y_test)
|
|
logloss = log_loss(y_test, clf.predict_proba(X_test))
|
|
assert_almost_equal(-logscore, logloss)
|
|
|
|
# same for an estimator without decision_function
|
|
clf = DecisionTreeClassifier()
|
|
clf.fit(X_train, y_train)
|
|
score1 = get_scorer('roc_auc')(clf, X_test, y_test)
|
|
score2 = roc_auc_score(y_test, clf.predict_proba(X_test)[:, 1])
|
|
assert_almost_equal(score1, score2)
|
|
|
|
# test with a regressor (no decision_function)
|
|
reg = DecisionTreeRegressor()
|
|
reg.fit(X_train, y_train)
|
|
score1 = get_scorer('roc_auc')(reg, X_test, y_test)
|
|
score2 = roc_auc_score(y_test, reg.predict(X_test))
|
|
assert_almost_equal(score1, score2)
|
|
|
|
# Test that an exception is raised on more than two classes
|
|
X, y = make_blobs(random_state=0, centers=3)
|
|
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
|
|
clf.fit(X_train, y_train)
|
|
assert_raises(ValueError, get_scorer('roc_auc'), clf, X_test, y_test)
|
|
|
|
|
|
def test_thresholded_scorers_multilabel_indicator_data():
|
|
# Test that the scorer work with multilabel-indicator format
|
|
# for multilabel and multi-output multi-class classifier
|
|
X, y = make_multilabel_classification(allow_unlabeled=False,
|
|
random_state=0)
|
|
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
|
|
|
|
# Multi-output multi-class predict_proba
|
|
clf = DecisionTreeClassifier()
|
|
clf.fit(X_train, y_train)
|
|
y_proba = clf.predict_proba(X_test)
|
|
score1 = get_scorer('roc_auc')(clf, X_test, y_test)
|
|
score2 = roc_auc_score(y_test, np.vstack(p[:, -1] for p in y_proba).T)
|
|
assert_almost_equal(score1, score2)
|
|
|
|
# Multi-output multi-class decision_function
|
|
# TODO Is there any yet?
|
|
clf = DecisionTreeClassifier()
|
|
clf.fit(X_train, y_train)
|
|
clf._predict_proba = clf.predict_proba
|
|
clf.predict_proba = None
|
|
clf.decision_function = lambda X: [p[:, 1] for p in clf._predict_proba(X)]
|
|
|
|
y_proba = clf.decision_function(X_test)
|
|
score1 = get_scorer('roc_auc')(clf, X_test, y_test)
|
|
score2 = roc_auc_score(y_test, np.vstack(p for p in y_proba).T)
|
|
assert_almost_equal(score1, score2)
|
|
|
|
# Multilabel predict_proba
|
|
clf = OneVsRestClassifier(DecisionTreeClassifier())
|
|
clf.fit(X_train, y_train)
|
|
score1 = get_scorer('roc_auc')(clf, X_test, y_test)
|
|
score2 = roc_auc_score(y_test, clf.predict_proba(X_test))
|
|
assert_almost_equal(score1, score2)
|
|
|
|
# Multilabel decision function
|
|
clf = OneVsRestClassifier(LinearSVC(random_state=0))
|
|
clf.fit(X_train, y_train)
|
|
score1 = get_scorer('roc_auc')(clf, X_test, y_test)
|
|
score2 = roc_auc_score(y_test, clf.decision_function(X_test))
|
|
assert_almost_equal(score1, score2)
|
|
|
|
|
|
def test_unsupervised_scorers():
|
|
# Test clustering scorers against gold standard labeling.
|
|
# We don't have any real unsupervised Scorers yet.
|
|
X, y = make_blobs(random_state=0, centers=2)
|
|
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)
|
|
km = KMeans(n_clusters=3)
|
|
km.fit(X_train)
|
|
score1 = get_scorer('adjusted_rand_score')(km, X_test, y_test)
|
|
score2 = adjusted_rand_score(y_test, km.predict(X_test))
|
|
assert_almost_equal(score1, score2)
|
|
|
|
|
|
@ignore_warnings
|
|
def test_raises_on_score_list():
|
|
# Test that when a list of scores is returned, we raise proper errors.
|
|
X, y = make_blobs(random_state=0)
|
|
f1_scorer_no_average = make_scorer(f1_score, average=None)
|
|
clf = DecisionTreeClassifier()
|
|
assert_raises(ValueError, cross_val_score, clf, X, y,
|
|
scoring=f1_scorer_no_average)
|
|
grid_search = GridSearchCV(clf, scoring=f1_scorer_no_average,
|
|
param_grid={'max_depth': [1, 2]})
|
|
assert_raises(ValueError, grid_search.fit, X, y)
|
|
|
|
|
|
@ignore_warnings
|
|
def test_scorer_sample_weight():
|
|
# Test that scorers support sample_weight or raise sensible errors
|
|
|
|
# Unlike the metrics invariance test, in the scorer case it's harder
|
|
# to ensure that, on the classifier output, weighted and unweighted
|
|
# scores really should be unequal.
|
|
X, y = make_classification(random_state=0)
|
|
_, y_ml = make_multilabel_classification(n_samples=X.shape[0],
|
|
random_state=0)
|
|
split = train_test_split(X, y, y_ml, random_state=0)
|
|
X_train, X_test, y_train, y_test, y_ml_train, y_ml_test = split
|
|
|
|
sample_weight = np.ones_like(y_test)
|
|
sample_weight[:10] = 0
|
|
|
|
# get sensible estimators for each metric
|
|
estimator = _make_estimators(X_train, y_train, y_ml_train)
|
|
|
|
for name, scorer in SCORERS.items():
|
|
if name in MULTILABEL_ONLY_SCORERS:
|
|
target = y_ml_test
|
|
else:
|
|
target = y_test
|
|
try:
|
|
weighted = scorer(estimator[name], X_test, target,
|
|
sample_weight=sample_weight)
|
|
ignored = scorer(estimator[name], X_test[10:], target[10:])
|
|
unweighted = scorer(estimator[name], X_test, target)
|
|
assert_not_equal(weighted, unweighted,
|
|
msg="scorer {0} behaves identically when "
|
|
"called with sample weights: {1} vs "
|
|
"{2}".format(name, weighted, unweighted))
|
|
assert_almost_equal(weighted, ignored,
|
|
err_msg="scorer {0} behaves differently when "
|
|
"ignoring samples and setting sample_weight to"
|
|
" 0: {1} vs {2}".format(name, weighted,
|
|
ignored))
|
|
|
|
except TypeError as e:
|
|
assert_true("sample_weight" in str(e),
|
|
"scorer {0} raises unhelpful exception when called "
|
|
"with sample weights: {1}".format(name, str(e)))
|
|
|
|
|
|
@ignore_warnings # UndefinedMetricWarning for P / R scores
|
|
def check_scorer_memmap(scorer_name):
|
|
scorer, estimator = SCORERS[scorer_name], ESTIMATORS[scorer_name]
|
|
if scorer_name in MULTILABEL_ONLY_SCORERS:
|
|
score = scorer(estimator, X_mm, y_ml_mm)
|
|
else:
|
|
score = scorer(estimator, X_mm, y_mm)
|
|
assert isinstance(score, numbers.Number), scorer_name
|
|
|
|
|
|
def test_scorer_memmap_input():
|
|
# Non-regression test for #6147: some score functions would
|
|
# return singleton memmap when computed on memmap data instead of scalar
|
|
# float values.
|
|
for name in SCORERS.keys():
|
|
yield check_scorer_memmap, name
|