179 lines
6.9 KiB
Cython
179 lines
6.9 KiB
Cython
# encoding: utf-8
|
|
# cython: cdivision=True
|
|
# cython: boundscheck=False
|
|
# cython: wraparound=False
|
|
#
|
|
# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>
|
|
#
|
|
# License: BSD Style.
|
|
|
|
import numpy as np
|
|
|
|
cimport numpy as np
|
|
cimport cython
|
|
|
|
np.import_array()
|
|
|
|
|
|
cdef class SequentialDataset:
|
|
"""Base class for datasets with sequential data access. """
|
|
|
|
cdef void next(self, DOUBLE **x_data_ptr, INTEGER **x_ind_ptr,
|
|
int *nnz, DOUBLE *y, DOUBLE *sample_weight):
|
|
"""Get the next example ``x`` from the dataset.
|
|
|
|
Parameters
|
|
----------
|
|
x_data_ptr : np.float64**
|
|
A pointer to the double array which holds the feature
|
|
values of the next example.
|
|
x_ind_ptr : np.int32**
|
|
A pointer to the int32 array which holds the feature
|
|
indices of the next example.
|
|
nnz : int*
|
|
A pointer to an int holding the number of non-zero
|
|
values of the next example.
|
|
y : np.float64*
|
|
The target value of the next example.
|
|
sample_weight : np.float64*
|
|
The weight of the next example.
|
|
"""
|
|
raise NotImplementedError()
|
|
|
|
cdef void shuffle(self, seed):
|
|
"""Permutes the ordering of examples. """
|
|
raise NotImplementedError()
|
|
|
|
|
|
cdef class ArrayDataset(SequentialDataset):
|
|
"""Dataset backed by a two-dimensional numpy array.
|
|
|
|
The dtype of the numpy array is expected to be ``np.float64``
|
|
and C-style memory layout.
|
|
"""
|
|
|
|
def __cinit__(self, np.ndarray[DOUBLE, ndim=2, mode='c'] X,
|
|
np.ndarray[DOUBLE, ndim=1, mode='c'] Y,
|
|
np.ndarray[DOUBLE, ndim=1, mode='c'] sample_weights):
|
|
"""A ``SequentialDataset`` backed by a two-dimensional numpy array.
|
|
|
|
Paramters
|
|
---------
|
|
X : ndarray, dtype=np.float64, ndim=2, mode='c'
|
|
The samples; a two-dimensional c-continuous numpy array of
|
|
dtype np.float64.
|
|
Y : ndarray, dtype=np.float64, ndim=1, mode='c'
|
|
The target values; a one-dimensional c-continuous numpy array of
|
|
dtype np.float64.
|
|
sample_weights : ndarray, dtype=np.float64, ndim=1, mode='c'
|
|
The weight of each sample; a one-dimensional c-continuous numpy
|
|
array of dtype np.float64.
|
|
"""
|
|
self.n_samples = X.shape[0]
|
|
self.n_features = X.shape[1]
|
|
cdef np.ndarray[INTEGER, ndim=1,
|
|
mode='c'] feature_indices = np.arange(0, self.n_features,
|
|
dtype=np.int32)
|
|
self.feature_indices = feature_indices
|
|
self.feature_indices_ptr = <INTEGER *> feature_indices.data
|
|
self.current_index = -1
|
|
self.stride = X.strides[0] / X.itemsize
|
|
self.X_data_ptr = <DOUBLE *>X.data
|
|
self.Y_data_ptr = <DOUBLE *>Y.data
|
|
self.sample_weight_data = <DOUBLE *>sample_weights.data
|
|
|
|
# Use index array for fast shuffling
|
|
cdef np.ndarray[INTEGER, ndim=1,
|
|
mode='c'] index = np.arange(0, self.n_samples,
|
|
dtype=np.int32)
|
|
self.index = index
|
|
self.index_data_ptr = <INTEGER *> index.data
|
|
|
|
cdef void next(self, DOUBLE **x_data_ptr, INTEGER **x_ind_ptr,
|
|
int *nnz, DOUBLE *y, DOUBLE *sample_weight):
|
|
cdef int current_index = self.current_index
|
|
if current_index >= (self.n_samples - 1):
|
|
current_index = -1
|
|
|
|
current_index += 1
|
|
cdef int sample_idx = self.index_data_ptr[current_index]
|
|
cdef int offset = sample_idx * self.stride
|
|
|
|
y[0] = self.Y_data_ptr[sample_idx]
|
|
x_data_ptr[0] = self.X_data_ptr + offset
|
|
x_ind_ptr[0] = self.feature_indices_ptr
|
|
nnz[0] = self.n_features
|
|
sample_weight[0] = self.sample_weight_data[sample_idx]
|
|
|
|
self.current_index = current_index
|
|
|
|
cdef void shuffle(self, seed):
|
|
np.random.RandomState(seed).shuffle(self.index)
|
|
|
|
|
|
cdef class CSRDataset(SequentialDataset):
|
|
"""A ``SequentialDataset`` backed by a scipy sparse CSR matrix. """
|
|
|
|
def __cinit__(self, np.ndarray[DOUBLE, ndim=1, mode='c'] X_data,
|
|
np.ndarray[INTEGER, ndim=1, mode='c'] X_indptr,
|
|
np.ndarray[INTEGER, ndim=1, mode='c'] X_indices,
|
|
np.ndarray[DOUBLE, ndim=1, mode='c'] Y,
|
|
np.ndarray[DOUBLE, ndim=1, mode='c'] sample_weight):
|
|
"""Dataset backed by a scipy sparse CSR matrix.
|
|
|
|
The feature indices of ``x`` are given by x_ind_ptr[0:nnz].
|
|
The corresponding feature values are given by
|
|
x_data_ptr[0:nnz].
|
|
|
|
Parameters
|
|
----------
|
|
X_data : ndarray, dtype=np.float64, ndim=1, mode='c'
|
|
The data array of the CSR matrix; a one-dimensional c-continuous
|
|
numpy array of dtype np.float64.
|
|
X_indptr : ndarray, dtype=np.int32, ndim=1, mode='c'
|
|
The index pointer array of the CSR matrix; a one-dimensional
|
|
c-continuous numpy array of dtype np.int32.
|
|
X_indices : ndarray, dtype=np.int32, ndim=1, mode='c'
|
|
The column indices array of the CSR matrix; a one-dimensional
|
|
c-continuous numpy array of dtype np.int32.
|
|
Y : ndarray, dtype=np.float64, ndim=1, mode='c'
|
|
The target values; a one-dimensional c-continuous numpy array of
|
|
dtype np.float64.
|
|
sample_weights : ndarray, dtype=np.float64, ndim=1, mode='c'
|
|
The weight of each sample; a one-dimensional c-continuous numpy
|
|
array of dtype np.float64.
|
|
"""
|
|
self.n_samples = Y.shape[0]
|
|
self.current_index = -1
|
|
self.X_data_ptr = <DOUBLE *>X_data.data
|
|
self.X_indptr_ptr = <INTEGER *>X_indptr.data
|
|
self.X_indices_ptr = <INTEGER *>X_indices.data
|
|
self.Y_data_ptr = <DOUBLE *>Y.data
|
|
self.sample_weight_data = <DOUBLE *> sample_weight.data
|
|
# Use index array for fast shuffling
|
|
cdef np.ndarray[INTEGER, ndim=1,
|
|
mode='c'] index = np.arange(0, self.n_samples,
|
|
dtype=np.int32)
|
|
self.index = index
|
|
self.index_data_ptr = <INTEGER *> index.data
|
|
|
|
cdef void next(self, DOUBLE **x_data_ptr, INTEGER **x_ind_ptr,
|
|
int *nnz, DOUBLE *y, DOUBLE *sample_weight):
|
|
cdef int current_index = self.current_index
|
|
if current_index >= (self.n_samples - 1):
|
|
current_index = -1
|
|
|
|
current_index += 1
|
|
cdef int sample_idx = self.index_data_ptr[current_index]
|
|
cdef int offset = self.X_indptr_ptr[sample_idx]
|
|
y[0] = self.Y_data_ptr[sample_idx]
|
|
x_data_ptr[0] = self.X_data_ptr + offset
|
|
x_ind_ptr[0] = self.X_indices_ptr + offset
|
|
nnz[0] = self.X_indptr_ptr[sample_idx + 1] - offset
|
|
sample_weight[0] = self.sample_weight_data[sample_idx]
|
|
|
|
self.current_index = current_index
|
|
|
|
cdef void shuffle(self, seed):
|
|
np.random.RandomState(seed).shuffle(self.index)
|