106 lines
3.1 KiB
Cython
106 lines
3.1 KiB
Cython
# Optimized inner loop of load_svmlight_file.
|
|
#
|
|
# Authors: Mathieu Blondel <mathieu@mblondel.org>
|
|
# Lars Buitinck <L.J.Buitinck@uva.nl>
|
|
# Olivier Grisel <olivier.grisel@ensta.org>
|
|
# License: BSD 3 clause
|
|
|
|
import array
|
|
from cpython cimport array
|
|
cimport cython
|
|
from libc.string cimport strchr
|
|
|
|
cimport numpy as np
|
|
import numpy as np
|
|
import scipy.sparse as sp
|
|
|
|
from ..externals.six import b
|
|
|
|
np.import_array()
|
|
|
|
|
|
cdef bytes COMMA = u','.encode('ascii')
|
|
cdef bytes COLON = u':'.encode('ascii')
|
|
|
|
|
|
@cython.boundscheck(False)
|
|
@cython.wraparound(False)
|
|
def _load_svmlight_file(f, dtype, bint multilabel, bint zero_based,
|
|
bint query_id):
|
|
cdef array.array data, indices, indptr, query
|
|
cdef bytes line
|
|
cdef char *hash_ptr, *line_cstr
|
|
cdef int idx, prev_idx
|
|
cdef Py_ssize_t i
|
|
cdef bytes qid_prefix = b('qid')
|
|
cdef Py_ssize_t n_features
|
|
|
|
# Special-case float32 but use float64 for everything else;
|
|
# the Python code will do further conversions.
|
|
if dtype == np.float32:
|
|
data = array.array("f")
|
|
else:
|
|
dtype = np.float64
|
|
data = array.array("d")
|
|
indices = array.array("i")
|
|
indptr = array.array("i", [0])
|
|
query = array.array("i")
|
|
|
|
if multilabel:
|
|
labels = []
|
|
else:
|
|
labels = array.array("d")
|
|
|
|
for line in f:
|
|
# skip comments
|
|
line_cstr = line
|
|
hash_ptr = strchr(line_cstr, '#')
|
|
if hash_ptr != NULL:
|
|
line = line[:hash_ptr - line_cstr]
|
|
|
|
line_parts = line.split()
|
|
if len(line_parts) == 0:
|
|
continue
|
|
|
|
target, features = line_parts[0], line_parts[1:]
|
|
if multilabel:
|
|
target = [float(y) for y in target.split(COMMA)]
|
|
target.sort()
|
|
labels.append(tuple(target))
|
|
else:
|
|
array.resize_smart(labels, len(labels) + 1)
|
|
labels[len(labels) - 1] = float(target)
|
|
|
|
prev_idx = -1
|
|
n_features = len(features)
|
|
if n_features and line_parts[1].startswith(qid_prefix):
|
|
_, value = line_parts[1].split(COLON, 1)
|
|
if query_id:
|
|
array.resize_smart(query, len(query) + 1)
|
|
query[len(query) - 1] = int(value)
|
|
line_parts.pop(1)
|
|
n_features -= 1
|
|
|
|
for i in xrange(1, n_features + 1):
|
|
idx_s, value = line_parts[i].split(COLON, 1)
|
|
idx = int(idx_s)
|
|
if idx < 0 or not zero_based and idx == 0:
|
|
raise ValueError(
|
|
"Invalid index %d in SVMlight/LibSVM data file." % idx)
|
|
if idx <= prev_idx:
|
|
raise ValueError("Feature indices in SVMlight/LibSVM data "
|
|
"file should be sorted and unique.")
|
|
|
|
array.resize_smart(indices, len(indices) + 1)
|
|
indices[len(indices) - 1] = idx
|
|
|
|
array.resize_smart(data, len(data) + 1)
|
|
data[len(data) - 1] = float(value)
|
|
|
|
prev_idx = idx
|
|
|
|
array.resize_smart(indptr, len(indptr) + 1)
|
|
indptr[len(indptr) - 1] = len(data)
|
|
|
|
return (dtype, data, indices, indptr, labels, query)
|