168 lines
5.0 KiB
Python
168 lines
5.0 KiB
Python
import numpy as np
|
|
import scipy.sparse as sp
|
|
from joblib import Memory
|
|
from pathlib import Path
|
|
|
|
from sklearn.decomposition import TruncatedSVD
|
|
from sklearn.datasets import (
|
|
make_blobs,
|
|
fetch_20newsgroups,
|
|
fetch_openml,
|
|
load_digits,
|
|
make_regression,
|
|
make_classification,
|
|
fetch_olivetti_faces,
|
|
)
|
|
from sklearn.preprocessing import MaxAbsScaler, StandardScaler
|
|
from sklearn.feature_extraction.text import TfidfVectorizer
|
|
from sklearn.model_selection import train_test_split
|
|
|
|
# memory location for caching datasets
|
|
M = Memory(location=str(Path(__file__).resolve().parent / "cache"))
|
|
|
|
|
|
@M.cache
|
|
def _blobs_dataset(n_samples=500000, n_features=3, n_clusters=100, dtype=np.float32):
|
|
X, _ = make_blobs(
|
|
n_samples=n_samples, n_features=n_features, centers=n_clusters, random_state=0
|
|
)
|
|
X = X.astype(dtype, copy=False)
|
|
|
|
X, X_val = train_test_split(X, test_size=0.1, random_state=0)
|
|
return X, X_val, None, None
|
|
|
|
|
|
@M.cache
|
|
def _20newsgroups_highdim_dataset(n_samples=None, ngrams=(1, 1), dtype=np.float32):
|
|
newsgroups = fetch_20newsgroups(random_state=0)
|
|
vectorizer = TfidfVectorizer(ngram_range=ngrams, dtype=dtype)
|
|
X = vectorizer.fit_transform(newsgroups.data[:n_samples])
|
|
y = newsgroups.target[:n_samples]
|
|
|
|
X, X_val, y, y_val = train_test_split(X, y, test_size=0.1, random_state=0)
|
|
return X, X_val, y, y_val
|
|
|
|
|
|
@M.cache
|
|
def _20newsgroups_lowdim_dataset(n_components=100, ngrams=(1, 1), dtype=np.float32):
|
|
newsgroups = fetch_20newsgroups()
|
|
vectorizer = TfidfVectorizer(ngram_range=ngrams)
|
|
X = vectorizer.fit_transform(newsgroups.data)
|
|
X = X.astype(dtype, copy=False)
|
|
svd = TruncatedSVD(n_components=n_components)
|
|
X = svd.fit_transform(X)
|
|
y = newsgroups.target
|
|
|
|
X, X_val, y, y_val = train_test_split(X, y, test_size=0.1, random_state=0)
|
|
return X, X_val, y, y_val
|
|
|
|
|
|
@M.cache
|
|
def _mnist_dataset(dtype=np.float32):
|
|
X, y = fetch_openml("mnist_784", version=1, return_X_y=True, as_frame=False)
|
|
X = X.astype(dtype, copy=False)
|
|
X = MaxAbsScaler().fit_transform(X)
|
|
|
|
X, X_val, y, y_val = train_test_split(X, y, test_size=0.1, random_state=0)
|
|
return X, X_val, y, y_val
|
|
|
|
|
|
@M.cache
|
|
def _digits_dataset(n_samples=None, dtype=np.float32):
|
|
X, y = load_digits(return_X_y=True)
|
|
X = X.astype(dtype, copy=False)
|
|
X = MaxAbsScaler().fit_transform(X)
|
|
X = X[:n_samples]
|
|
y = y[:n_samples]
|
|
|
|
X, X_val, y, y_val = train_test_split(X, y, test_size=0.1, random_state=0)
|
|
return X, X_val, y, y_val
|
|
|
|
|
|
@M.cache
|
|
def _synth_regression_dataset(n_samples=100000, n_features=100, dtype=np.float32):
|
|
X, y = make_regression(
|
|
n_samples=n_samples,
|
|
n_features=n_features,
|
|
n_informative=n_features // 10,
|
|
noise=50,
|
|
random_state=0,
|
|
)
|
|
X = X.astype(dtype, copy=False)
|
|
X = StandardScaler().fit_transform(X)
|
|
|
|
X, X_val, y, y_val = train_test_split(X, y, test_size=0.1, random_state=0)
|
|
return X, X_val, y, y_val
|
|
|
|
|
|
@M.cache
|
|
def _synth_regression_sparse_dataset(
|
|
n_samples=10000, n_features=10000, density=0.01, dtype=np.float32
|
|
):
|
|
X = sp.random(
|
|
m=n_samples, n=n_features, density=density, format="csr", random_state=0
|
|
)
|
|
X.data = np.random.RandomState(0).randn(X.getnnz())
|
|
X = X.astype(dtype, copy=False)
|
|
coefs = sp.random(m=n_features, n=1, density=0.5, random_state=0)
|
|
coefs.data = np.random.RandomState(0).randn(coefs.getnnz())
|
|
y = X.dot(coefs.toarray()).reshape(-1)
|
|
y += 0.2 * y.std() * np.random.randn(n_samples)
|
|
|
|
X, X_val, y, y_val = train_test_split(X, y, test_size=0.1, random_state=0)
|
|
return X, X_val, y, y_val
|
|
|
|
|
|
@M.cache
|
|
def _synth_classification_dataset(
|
|
n_samples=1000, n_features=10000, n_classes=2, dtype=np.float32
|
|
):
|
|
X, y = make_classification(
|
|
n_samples=n_samples,
|
|
n_features=n_features,
|
|
n_classes=n_classes,
|
|
random_state=0,
|
|
n_informative=n_features,
|
|
n_redundant=0,
|
|
)
|
|
X = X.astype(dtype, copy=False)
|
|
X = StandardScaler().fit_transform(X)
|
|
|
|
X, X_val, y, y_val = train_test_split(X, y, test_size=0.1, random_state=0)
|
|
return X, X_val, y, y_val
|
|
|
|
|
|
@M.cache
|
|
def _olivetti_faces_dataset():
|
|
dataset = fetch_olivetti_faces(shuffle=True, random_state=42)
|
|
faces = dataset.data
|
|
n_samples, n_features = faces.shape
|
|
faces_centered = faces - faces.mean(axis=0)
|
|
# local centering
|
|
faces_centered -= faces_centered.mean(axis=1).reshape(n_samples, -1)
|
|
X = faces_centered
|
|
|
|
X, X_val = train_test_split(X, test_size=0.1, random_state=0)
|
|
return X, X_val, None, None
|
|
|
|
|
|
@M.cache
|
|
def _random_dataset(
|
|
n_samples=1000, n_features=1000, representation="dense", dtype=np.float32
|
|
):
|
|
if representation == "dense":
|
|
X = np.random.RandomState(0).random_sample((n_samples, n_features))
|
|
X = X.astype(dtype, copy=False)
|
|
else:
|
|
X = sp.random(
|
|
n_samples,
|
|
n_features,
|
|
density=0.05,
|
|
format="csr",
|
|
dtype=dtype,
|
|
random_state=0,
|
|
)
|
|
|
|
X, X_val = train_test_split(X, test_size=0.1, random_state=0)
|
|
return X, X_val, None, None
|