271 lines
8.6 KiB
Python
271 lines
8.6 KiB
Python
"""
|
|
Base IO code for all datasets
|
|
"""
|
|
|
|
# Copyright (c) 2007 David Cournapeau <cournape@gmail.com>
|
|
# 2010 Fabian Pedregosa <fabian.pedregosa@inria.fr>
|
|
# 2010 Olivier Grisel <olivier.grisel@ensta.org>
|
|
# License: Simplified BSD
|
|
|
|
import csv
|
|
import shutil
|
|
from os import environ
|
|
from os.path import dirname
|
|
from os.path import join
|
|
from os.path import exists
|
|
from os.path import expanduser
|
|
from os.path import isdir
|
|
from os import listdir
|
|
from os import makedirs
|
|
|
|
import numpy as np
|
|
|
|
|
|
class Bunch(dict):
|
|
""" Container object for datasets: dictionnary-like object that
|
|
exposes its keys as attributes.
|
|
"""
|
|
|
|
def __init__(self, **kwargs):
|
|
dict.__init__(self, kwargs)
|
|
self.__dict__ = self
|
|
|
|
|
|
def get_data_home(data_home=None):
|
|
"""Return the path of the scikit-learn data dir
|
|
|
|
This folder is used by some large dataset loaders to avoid
|
|
downloading the data several times.
|
|
|
|
By default the data dir is set to a folder named 'scikit_learn_data'
|
|
in the user home folder.
|
|
|
|
Alternatively, it can be set by the 'SCIKIT_LEARN_DATA' environment
|
|
variable or programatically by giving an explit folder path. The
|
|
'~' symbol is expanded to the user home folder.
|
|
|
|
If the folder does not already exist, it is automatically created.
|
|
"""
|
|
if data_home is None:
|
|
data_home = environ.get('SCIKIT_LEARN_DATA',
|
|
join('~', 'scikit_learn_data'))
|
|
data_home = expanduser(data_home)
|
|
if not exists(data_home):
|
|
makedirs(data_home)
|
|
return data_home
|
|
|
|
|
|
def clear_data_home(data_home=None):
|
|
"""Delete all the content of the data home cache"""
|
|
data_home = get_data_home(data_home)
|
|
shutil.rmtree(data_home)
|
|
|
|
|
|
def load_filenames(container_path, description=None, categories=None,
|
|
shuffle=True, rng=42):
|
|
"""Load filenames with categories as subfolder names
|
|
|
|
Individual samples are assumed to be files stored a two levels folder
|
|
structure such as the following:
|
|
|
|
container_folder/
|
|
category_1_folder/
|
|
file_1.txt
|
|
file_2.txt
|
|
...
|
|
file_42.txt
|
|
category_2_folder/
|
|
file_43.txt
|
|
file_44.txt
|
|
...
|
|
|
|
The folder names are used has supervised signal label names. The indivial
|
|
file names are not important.
|
|
|
|
This function does not try to extract features into a numpy array or
|
|
scipy sparse matrix, nor does it try to load the files in memory.
|
|
|
|
To use utf-8 text files in a scikit-learn classification or clustering
|
|
algorithm you will first need to use the `scikits.learn.features.text`
|
|
module to build a feature extraction transformer that suits your
|
|
problem.
|
|
|
|
Similar feature extractors should be build for other kind of unstructured
|
|
data input such as images, audio, video, ...
|
|
|
|
Parameters
|
|
----------
|
|
|
|
container_path : string or unicode
|
|
the path to the main folder holding one subfolder per category
|
|
|
|
description: string or unicode
|
|
a paragraph describing the characteristic of the dataset, its source,
|
|
reference, ...
|
|
|
|
categories : None or collection of string or unicode
|
|
if None (default), load all the categories.
|
|
if not Non, list of category names to load (other categories ignored)
|
|
|
|
shuffle : True by default
|
|
whether or not to shuffle the data: might be important for models that
|
|
make the assumption that the samples are independent and identically
|
|
distributed (i.i.d.) such as stochastic gradient descent for instance.
|
|
|
|
rng : a numpy random number generator or a seed integer, 42 by default
|
|
used to shuffle the dataset
|
|
|
|
Returns
|
|
-------
|
|
|
|
data : Bunch
|
|
Dictionary-like object, the interesting attributes are:
|
|
'filenames', the files holding the raw to learn, 'target', the
|
|
classification labels (integer index), 'target_names',
|
|
the meaning of the labels, and 'DESCR', the full description of the
|
|
dataset.
|
|
|
|
"""
|
|
target = []
|
|
target_names = []
|
|
filenames = []
|
|
|
|
folders = [f for f in sorted(listdir(container_path))
|
|
if isdir(join(container_path, f))]
|
|
|
|
if categories is not None:
|
|
folders = [f for f in folders if f in categories]
|
|
|
|
for label, folder in enumerate(folders):
|
|
target_names.append(folder)
|
|
folder_path = join(container_path, folder)
|
|
documents = [join(folder_path, d)
|
|
for d in sorted(listdir(folder_path))]
|
|
target.extend(len(documents) * [label])
|
|
filenames.extend(documents)
|
|
|
|
# convert as array for fancy indexing
|
|
filenames = np.array(filenames)
|
|
target = np.array(target)
|
|
|
|
if shuffle:
|
|
if isinstance(rng, int):
|
|
rng = np.random.RandomState(rng)
|
|
indices = np.arange(filenames.shape[0])
|
|
rng.shuffle(indices)
|
|
filenames = filenames[indices]
|
|
target = target[indices]
|
|
|
|
return Bunch(filenames=filenames,
|
|
target_names=target_names,
|
|
target=target,
|
|
DESCR=description)
|
|
|
|
|
|
###############################################################################
|
|
|
|
def load_iris():
|
|
"""load the iris dataset and returns it.
|
|
|
|
Returns
|
|
-------
|
|
data : Bunch
|
|
Dictionnary-like object, the interesting attributes are:
|
|
'data', the data to learn, 'target', the classification labels,
|
|
'target_names', the meaning of the labels, and 'DESCR', the
|
|
full description of the dataset.
|
|
|
|
Example
|
|
-------
|
|
Let's say you are interested in the samples 10, 25, and 50, and want to
|
|
know their class name.
|
|
|
|
>>> from scikits.learn.datasets import load_iris
|
|
>>> data = load_iris()
|
|
>>> data.target[[10, 25, 50]]
|
|
array([0, 0, 1])
|
|
>>> list(data.target_names)
|
|
['setosa', 'versicolor', 'virginica']
|
|
|
|
"""
|
|
|
|
module_path = dirname(__file__)
|
|
data_file = csv.reader(open(join(module_path, 'data', 'iris.csv')))
|
|
fdescr = open(join(module_path, 'descr', 'iris.rst'))
|
|
temp = data_file.next()
|
|
n_samples = int(temp[0])
|
|
n_features = int(temp[1])
|
|
target_names = np.array(temp[2:])
|
|
data = np.empty((n_samples, n_features))
|
|
target = np.empty((n_samples,), dtype=np.int)
|
|
for i, ir in enumerate(data_file):
|
|
data[i] = np.asanyarray(ir[:-1], dtype=np.float)
|
|
target[i] = np.asanyarray(ir[-1], dtype=np.int)
|
|
return Bunch(data=data, target=target, target_names=target_names,
|
|
DESCR=fdescr.read())
|
|
|
|
|
|
def load_digits():
|
|
"""load the digits dataset and returns it.
|
|
|
|
Returns
|
|
-------
|
|
data : Bunch
|
|
Dictionnary-like object, the interesting attributes are:
|
|
'data', the data to learn, `images`, the images corresponding
|
|
to each sample, 'target', the classification labels for each
|
|
sample, 'target_names', the meaning of the labels, and 'DESCR',
|
|
the full description of the dataset.
|
|
|
|
Example
|
|
-------
|
|
To load the data and visualize the images::
|
|
|
|
import pylab as pl
|
|
digits = datasets.load_digits()
|
|
pl.gray()
|
|
# Visualize the first image:
|
|
pl.matshow(digits.raw_data[0])
|
|
|
|
"""
|
|
|
|
module_path = dirname(__file__)
|
|
data = np.loadtxt(join(module_path, 'data', 'digits.csv.gz'),
|
|
delimiter=',')
|
|
descr = open(join(module_path, 'descr', 'digits.rst')).read()
|
|
target = data[:, -1]
|
|
flat_data = data[:, :-1]
|
|
images = flat_data.view()
|
|
images.shape = (-1, 8, 8)
|
|
return Bunch(data=flat_data, target=target.astype(np.int),
|
|
target_names=np.arange(10),
|
|
images=images,
|
|
DESCR=descr)
|
|
|
|
|
|
def load_diabetes():
|
|
base_dir = join(dirname(__file__), 'data')
|
|
data = np.loadtxt(join(base_dir, 'diabetes_data.csv.gz'))
|
|
target = np.loadtxt(join(base_dir, 'diabetes_target.csv.gz'))
|
|
return Bunch(data=data, target=target)
|
|
|
|
|
|
def load_linnerud():
|
|
base_dir = join(dirname(__file__), 'data/')
|
|
# Read data
|
|
data_exercise = np.loadtxt(base_dir + 'linnerud_exercise.csv', skiprows=1)
|
|
data_physiological = np.loadtxt(base_dir + 'linnerud_physiological.csv',
|
|
skiprows=1)
|
|
# Read header
|
|
f = open(base_dir + 'linnerud_exercise.csv')
|
|
header_exercise = f.readline().split()
|
|
f.close()
|
|
f = open(base_dir + 'linnerud_physiological.csv')
|
|
header_physiological = f.readline().split()
|
|
f.close()
|
|
fdescr = open(dirname(__file__) + '/descr/linnerud.rst')
|
|
return Bunch(data_exercise=data_exercise, header_exercise=header_exercise,
|
|
data_physiological=data_physiological,
|
|
header_physiological=header_physiological,
|
|
DESCR=fdescr.read())
|