87 lines
2.7 KiB
Python
87 lines
2.7 KiB
Python
""" This module implements a fast and memory-efficient (no memory copying)
|
|
loader for the svmlight / libsvm sparse dataset format. """
|
|
|
|
# Authors: Mathieu Blondel <mathieu@mblondel.org>
|
|
# Lars Buitinck <L.J.Buitinck@uva.nl>
|
|
# License: Simple BSD.
|
|
|
|
import os.path
|
|
|
|
import scipy.sparse as sp
|
|
|
|
from _svmlight_format import _load_svmlight_file
|
|
|
|
|
|
def load_svmlight_file(file_path, other_file_path=None,
|
|
n_features=None, buffer_mb=40):
|
|
"""Load datasets in the svmlight / libsvm format directly into
|
|
scipy sparse CSR matrices.
|
|
|
|
Parameters
|
|
----------
|
|
file_path: str
|
|
Path to a file to load.
|
|
|
|
other_file_path: str or None
|
|
Path to another file to load. scikit-learn will make sure that the
|
|
number of features in the returned matrix is the same as for
|
|
file_path.
|
|
|
|
n_features: int or None
|
|
The number of features to use. If None, it will be inferred.
|
|
|
|
buffer_mb: int (default: 40)
|
|
The size of the buffer used while loading the dataset in mega-bytes.
|
|
|
|
Returns
|
|
-------
|
|
(X, y)
|
|
|
|
where X is a scipy.sparse matrix of shape (n_samples, n_features),
|
|
y is a ndarray of shape (n_samples,),
|
|
|
|
or, if other_file_path is not None,
|
|
|
|
(X1, y1, X2, y2)
|
|
|
|
where X1 and X2 are scipy.sparse matrices of shape
|
|
(n_samples1, n_features) and
|
|
(n_samples2, n_features),
|
|
y1 and y2 are ndarrays of shape (n_samples1,) and (n_samples2,).
|
|
|
|
Note
|
|
----
|
|
When fitting a model to a matrix X_train and evaluating it against a matrix
|
|
X_test, it is essential that X_train and X_test have the same number of
|
|
features (X_train.shape[1] == X_test.shape[1]). This may not be the case if
|
|
you load them with load_svmlight_format separately. To address this
|
|
problem, we recommend to use load_svmlight_format(train_file, test_file)
|
|
or load_svmlight_format(test_file, n_features=X_train.shape[1]).
|
|
"""
|
|
data, indices, indptr, labels = _load_svmlight_file(file_path, buffer_mb)
|
|
|
|
if n_features is not None:
|
|
shape = (indptr.shape[0] - 1, n_features)
|
|
else:
|
|
shape = None # inferred
|
|
|
|
X_train = sp.csr_matrix((data, indices, indptr), shape)
|
|
|
|
ret = [X_train, labels]
|
|
|
|
if other_file_path is not None:
|
|
tup = _load_svmlight_file(other_file_path, buffer_mb)
|
|
data, indices, indptr, labels = tup
|
|
|
|
if n_features is None:
|
|
n_features = X_train.shape[1]
|
|
|
|
shape = (indptr.shape[0] - 1, n_features)
|
|
|
|
X_test = sp.csr_matrix((data, indices, indptr), shape)
|
|
|
|
ret.append(X_test)
|
|
ret.append(labels)
|
|
|
|
return tuple(ret)
|