245 lines
8.3 KiB
Python
245 lines
8.3 KiB
Python
"""Author: Arthur Mensch
|
|
|
|
Benchmarks of sklearn SAGA vs lightning SAGA vs Liblinear. Shows the gain
|
|
in using multinomial logistic regression in term of learning time.
|
|
"""
|
|
import json
|
|
import time
|
|
from os.path import expanduser
|
|
|
|
from joblib import delayed, Parallel, Memory
|
|
import matplotlib.pyplot as plt
|
|
import numpy as np
|
|
|
|
from sklearn.datasets import fetch_rcv1, load_iris, load_digits, \
|
|
fetch_20newsgroups_vectorized
|
|
from sklearn.linear_model import LogisticRegression
|
|
from sklearn.metrics import log_loss
|
|
from sklearn.model_selection import train_test_split
|
|
from sklearn.preprocessing import LabelBinarizer, LabelEncoder
|
|
from sklearn.utils.extmath import safe_sparse_dot, softmax
|
|
|
|
|
|
def fit_single(solver, X, y, penalty='l2', single_target=True, C=1,
|
|
max_iter=10, skip_slow=False):
|
|
if skip_slow and solver == 'lightning' and penalty == 'l1':
|
|
print('skip_slowping l1 logistic regression with solver lightning.')
|
|
return
|
|
|
|
print('Solving %s logistic regression with penalty %s, solver %s.'
|
|
% ('binary' if single_target else 'multinomial',
|
|
penalty, solver))
|
|
|
|
if solver == 'lightning':
|
|
from lightning.classification import SAGAClassifier
|
|
|
|
if single_target or solver not in ['sag', 'saga']:
|
|
multi_class = 'ovr'
|
|
else:
|
|
multi_class = 'multinomial'
|
|
|
|
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42,
|
|
stratify=y)
|
|
n_samples = X_train.shape[0]
|
|
n_classes = np.unique(y_train).shape[0]
|
|
test_scores = [1]
|
|
train_scores = [1]
|
|
accuracies = [1 / n_classes]
|
|
times = [0]
|
|
|
|
if penalty == 'l2':
|
|
alpha = 1. / (C * n_samples)
|
|
beta = 0
|
|
lightning_penalty = None
|
|
else:
|
|
alpha = 0.
|
|
beta = 1. / (C * n_samples)
|
|
lightning_penalty = 'l1'
|
|
|
|
for this_max_iter in range(1, max_iter + 1, 2):
|
|
print('[%s, %s, %s] Max iter: %s' %
|
|
('binary' if single_target else 'multinomial',
|
|
penalty, solver, this_max_iter))
|
|
if solver == 'lightning':
|
|
lr = SAGAClassifier(loss='log', alpha=alpha, beta=beta,
|
|
penalty=lightning_penalty,
|
|
tol=-1, max_iter=this_max_iter)
|
|
else:
|
|
lr = LogisticRegression(solver=solver,
|
|
multi_class=multi_class,
|
|
C=C,
|
|
penalty=penalty,
|
|
fit_intercept=False, tol=1e-24,
|
|
max_iter=this_max_iter,
|
|
random_state=42,
|
|
)
|
|
t0 = time.clock()
|
|
lr.fit(X_train, y_train)
|
|
train_time = time.clock() - t0
|
|
|
|
scores = []
|
|
for (X, y) in [(X_train, y_train), (X_test, y_test)]:
|
|
try:
|
|
y_pred = lr.predict_proba(X)
|
|
except NotImplementedError:
|
|
# Lightning predict_proba is not implemented for n_classes > 2
|
|
y_pred = _predict_proba(lr, X)
|
|
score = log_loss(y, y_pred, normalize=False) / n_samples
|
|
score += (0.5 * alpha * np.sum(lr.coef_ ** 2) +
|
|
beta * np.sum(np.abs(lr.coef_)))
|
|
scores.append(score)
|
|
train_score, test_score = tuple(scores)
|
|
|
|
y_pred = lr.predict(X_test)
|
|
accuracy = np.sum(y_pred == y_test) / y_test.shape[0]
|
|
test_scores.append(test_score)
|
|
train_scores.append(train_score)
|
|
accuracies.append(accuracy)
|
|
times.append(train_time)
|
|
return lr, times, train_scores, test_scores, accuracies
|
|
|
|
|
|
def _predict_proba(lr, X):
|
|
pred = safe_sparse_dot(X, lr.coef_.T)
|
|
if hasattr(lr, "intercept_"):
|
|
pred += lr.intercept_
|
|
return softmax(pred)
|
|
|
|
|
|
def exp(solvers, penalties, single_target, n_samples=30000, max_iter=20,
|
|
dataset='rcv1', n_jobs=1, skip_slow=False):
|
|
mem = Memory(cachedir=expanduser('~/cache'), verbose=0)
|
|
|
|
if dataset == 'rcv1':
|
|
rcv1 = fetch_rcv1()
|
|
|
|
lbin = LabelBinarizer()
|
|
lbin.fit(rcv1.target_names)
|
|
|
|
X = rcv1.data
|
|
y = rcv1.target
|
|
y = lbin.inverse_transform(y)
|
|
le = LabelEncoder()
|
|
y = le.fit_transform(y)
|
|
if single_target:
|
|
y_n = y.copy()
|
|
y_n[y > 16] = 1
|
|
y_n[y <= 16] = 0
|
|
y = y_n
|
|
|
|
elif dataset == 'digits':
|
|
digits = load_digits()
|
|
X, y = digits.data, digits.target
|
|
if single_target:
|
|
y_n = y.copy()
|
|
y_n[y < 5] = 1
|
|
y_n[y >= 5] = 0
|
|
y = y_n
|
|
elif dataset == 'iris':
|
|
iris = load_iris()
|
|
X, y = iris.data, iris.target
|
|
elif dataset == '20newspaper':
|
|
ng = fetch_20newsgroups_vectorized()
|
|
X = ng.data
|
|
y = ng.target
|
|
if single_target:
|
|
y_n = y.copy()
|
|
y_n[y > 4] = 1
|
|
y_n[y <= 16] = 0
|
|
y = y_n
|
|
|
|
X = X[:n_samples]
|
|
y = y[:n_samples]
|
|
|
|
cached_fit = mem.cache(fit_single)
|
|
out = Parallel(n_jobs=n_jobs, mmap_mode=None)(
|
|
delayed(cached_fit)(solver, X, y,
|
|
penalty=penalty, single_target=single_target,
|
|
C=1, max_iter=max_iter, skip_slow=skip_slow)
|
|
for solver in solvers
|
|
for penalty in penalties)
|
|
|
|
res = []
|
|
idx = 0
|
|
for solver in solvers:
|
|
for penalty in penalties:
|
|
if not (skip_slow and solver == 'lightning' and penalty == 'l1'):
|
|
lr, times, train_scores, test_scores, accuracies = out[idx]
|
|
this_res = dict(solver=solver, penalty=penalty,
|
|
single_target=single_target,
|
|
times=times, train_scores=train_scores,
|
|
test_scores=test_scores,
|
|
accuracies=accuracies)
|
|
res.append(this_res)
|
|
idx += 1
|
|
|
|
with open('bench_saga.json', 'w+') as f:
|
|
json.dump(res, f)
|
|
|
|
|
|
def plot():
|
|
import pandas as pd
|
|
with open('bench_saga.json', 'r') as f:
|
|
f = json.load(f)
|
|
res = pd.DataFrame(f)
|
|
res.set_index(['single_target', 'penalty'], inplace=True)
|
|
|
|
grouped = res.groupby(level=['single_target', 'penalty'])
|
|
|
|
colors = {'saga': 'blue', 'liblinear': 'orange', 'lightning': 'green'}
|
|
|
|
for idx, group in grouped:
|
|
single_target, penalty = idx
|
|
fig = plt.figure(figsize=(12, 4))
|
|
ax = fig.add_subplot(131)
|
|
|
|
train_scores = group['train_scores'].values
|
|
ref = np.min(np.concatenate(train_scores)) * 0.999
|
|
|
|
for scores, times, solver in zip(group['train_scores'], group['times'],
|
|
group['solver']):
|
|
scores = scores / ref - 1
|
|
ax.plot(times, scores, label=solver, color=colors[solver])
|
|
ax.set_xlabel('Time (s)')
|
|
ax.set_ylabel('Training objective (relative to min)')
|
|
ax.set_yscale('log')
|
|
|
|
ax = fig.add_subplot(132)
|
|
|
|
test_scores = group['test_scores'].values
|
|
ref = np.min(np.concatenate(test_scores)) * 0.999
|
|
|
|
for scores, times, solver in zip(group['test_scores'], group['times'],
|
|
group['solver']):
|
|
scores = scores / ref - 1
|
|
ax.plot(times, scores, label=solver, color=colors[solver])
|
|
ax.set_xlabel('Time (s)')
|
|
ax.set_ylabel('Test objective (relative to min)')
|
|
ax.set_yscale('log')
|
|
|
|
ax = fig.add_subplot(133)
|
|
|
|
for accuracy, times, solver in zip(group['accuracies'], group['times'],
|
|
group['solver']):
|
|
ax.plot(times, accuracy, label=solver, color=colors[solver])
|
|
ax.set_xlabel('Time (s)')
|
|
ax.set_ylabel('Test accuracy')
|
|
ax.legend()
|
|
name = 'single_target' if single_target else 'multi_target'
|
|
name += '_%s' % penalty
|
|
plt.suptitle(name)
|
|
name += '.png'
|
|
fig.tight_layout()
|
|
fig.subplots_adjust(top=0.9)
|
|
plt.savefig(name)
|
|
plt.close(fig)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
solvers = ['saga', 'liblinear', 'lightning']
|
|
penalties = ['l1', 'l2']
|
|
single_target = True
|
|
exp(solvers, penalties, single_target, n_samples=None, n_jobs=1,
|
|
dataset='20newspaper', max_iter=20)
|
|
plot()
|