106 lines
3.6 KiB
Python
106 lines
3.6 KiB
Python
"""
|
|
=============================================================
|
|
Receiver Operating Characteristic (ROC) with cross validation
|
|
=============================================================
|
|
|
|
Example of Receiver Operating Characteristic (ROC) metric to evaluate
|
|
classifier output quality using cross-validation.
|
|
|
|
ROC curves typically feature true positive rate on the Y axis, and false
|
|
positive rate on the X axis. This means that the top left corner of the plot is
|
|
the "ideal" point - a false positive rate of zero, and a true positive rate of
|
|
one. This is not very realistic, but it does mean that a larger area under the
|
|
curve (AUC) is usually better.
|
|
|
|
The "steepness" of ROC curves is also important, since it is ideal to maximize
|
|
the true positive rate while minimizing the false positive rate.
|
|
|
|
This example shows the ROC response of different datasets, created from K-fold
|
|
cross-validation. Taking all of these curves, it is possible to calculate the
|
|
mean area under curve, and see the variance of the curve when the
|
|
training set is split into different subsets. This roughly shows how the
|
|
classifier output is affected by changes in the training data, and how
|
|
different the splits generated by K-fold cross-validation are from one another.
|
|
|
|
.. note::
|
|
|
|
See also :func:`sklearn.metrics.roc_auc_score`,
|
|
:func:`sklearn.model_selection.cross_val_score`,
|
|
:ref:`sphx_glr_auto_examples_model_selection_plot_roc.py`,
|
|
|
|
"""
|
|
print(__doc__)
|
|
|
|
import numpy as np
|
|
from scipy import interp
|
|
import matplotlib.pyplot as plt
|
|
from itertools import cycle
|
|
|
|
from sklearn import svm, datasets
|
|
from sklearn.metrics import roc_curve, auc
|
|
from sklearn.model_selection import StratifiedKFold
|
|
|
|
# #############################################################################
|
|
# Data IO and generation
|
|
|
|
# Import some data to play with
|
|
iris = datasets.load_iris()
|
|
X = iris.data
|
|
y = iris.target
|
|
X, y = X[y != 2], y[y != 2]
|
|
n_samples, n_features = X.shape
|
|
|
|
# Add noisy features
|
|
random_state = np.random.RandomState(0)
|
|
X = np.c_[X, random_state.randn(n_samples, 200 * n_features)]
|
|
|
|
# #############################################################################
|
|
# Classification and ROC analysis
|
|
|
|
# Run classifier with cross-validation and plot ROC curves
|
|
cv = StratifiedKFold(n_splits=6)
|
|
classifier = svm.SVC(kernel='linear', probability=True,
|
|
random_state=random_state)
|
|
|
|
tprs = []
|
|
aucs = []
|
|
mean_fpr = np.linspace(0, 1, 100)
|
|
|
|
i = 0
|
|
for train, test in cv.split(X, y):
|
|
probas_ = classifier.fit(X[train], y[train]).predict_proba(X[test])
|
|
# Compute ROC curve and area the curve
|
|
fpr, tpr, thresholds = roc_curve(y[test], probas_[:, 1])
|
|
tprs.append(interp(mean_fpr, fpr, tpr))
|
|
tprs[-1][0] = 0.0
|
|
roc_auc = auc(fpr, tpr)
|
|
aucs.append(roc_auc)
|
|
plt.plot(fpr, tpr, lw=1, alpha=0.3,
|
|
label='ROC fold %d (AUC = %0.2f)' % (i, roc_auc))
|
|
|
|
i += 1
|
|
plt.plot([0, 1], [0, 1], linestyle='--', lw=2, color='r',
|
|
label='Chance', alpha=.8)
|
|
|
|
mean_tpr = np.mean(tprs, axis=0)
|
|
mean_tpr[-1] = 1.0
|
|
mean_auc = auc(mean_fpr, mean_tpr)
|
|
std_auc = np.std(aucs)
|
|
plt.plot(mean_fpr, mean_tpr, color='b',
|
|
label=r'Mean ROC (AUC = %0.2f $\pm$ %0.2f)' % (mean_auc, std_auc),
|
|
lw=2, alpha=.8)
|
|
|
|
std_tpr = np.std(tprs, axis=0)
|
|
tprs_upper = np.minimum(mean_tpr + std_tpr, 1)
|
|
tprs_lower = np.maximum(mean_tpr - std_tpr, 0)
|
|
plt.fill_between(mean_fpr, tprs_lower, tprs_upper, color='grey', alpha=.2,
|
|
label=r'$\pm$ 1 std. dev.')
|
|
|
|
plt.xlim([-0.05, 1.05])
|
|
plt.ylim([-0.05, 1.05])
|
|
plt.xlabel('False Positive Rate')
|
|
plt.ylabel('True Positive Rate')
|
|
plt.title('Receiver operating characteristic example')
|
|
plt.legend(loc="lower right")
|
|
plt.show()
|