p60975123
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

switch input columns and operation 

change ImagefolderDV2 name

change ds.transforms.vision to ds.vision

change batch api to match map api more closely

compose op changes

test_pylint

remove compose op from vision, move to transform module, refactor map and batch to use column_order
This commit is contained in:
nhussain 2020-08-27 15:30:21 -04:00
parent 75045e3e2a
commit 3bac9d3713
156 changed files with 1339 additions and 1107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc
View File

@ -733,7 +733,7 @@ Status DEPipeline::ParseMapOp(const py::dict &args, std::shared_ptr<DatasetOp> *
(void)map_builder.SetInColNames(in_col_names);
} else if (key == "output_columns") {
(void)map_builder.SetOutColNames(ToStringVector(value));
} else if (key == "columns_order") {
} else if (key == "column_order") {
project_columns = ToStringVector(value);
} else if (key == "num_parallel_workers") {
num_workers = ToInt(value);

2
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc
View File

@ -113,7 +113,7 @@ Status ImageFolderOp::PrescanMasterEntry(const std::string &filedir) {
num_rows_ = image_label_pairs_.size();
if (num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API ImageFolderDatasetV2.Please check file path or dataset "
"There is no valid data matching the dataset API ImageFolderDataset. Please check file path or dataset "
"API validation first.");
}
// free memory of two queues used for pre-scan

2
mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h
View File

@ -111,7 +111,7 @@ constexpr char kWhitespaceTokenizerOp[] = "WhitespaceTokenizerOp";
constexpr char kWordpieceTokenizerOp[] = "WordpieceTokenizerOp";
constexpr char kRandomChoiceOp[] = "RandomChoiceOp";
constexpr char kRandomApplyOp[] = "RandomApplyOp";
constexpr char kComposeOp[] = "ComposeOp";
constexpr char kComposeOp[] = "Compose";
constexpr char kRandomSelectSubpolicyOp[] = "RandomSelectSubpolicyOp";
constexpr char kSentencepieceTokenizerOp[] = "SentencepieceTokenizerOp";

4
mindspore/dataset/__init__.py
View File

@ -19,7 +19,7 @@ can also create samplers with this module to sample data.
"""
from .core import config
from .engine.datasets import TFRecordDataset, ImageFolderDatasetV2, MnistDataset, MindDataset, NumpySlicesDataset, \
from .engine.datasets import TFRecordDataset, ImageFolderDataset, MnistDataset, MindDataset, NumpySlicesDataset, \
GeneratorDataset, ManifestDataset, Cifar10Dataset, Cifar100Dataset, VOCDataset, CocoDataset, CelebADataset, \
TextFileDataset, CLUEDataset, CSVDataset, Schema, Shuffle, zip, RandomDataset, PaddedDataset
from .engine.samplers import DistributedSampler, PKSampler, RandomSampler, SequentialSampler, SubsetRandomSampler, \
@ -28,7 +28,7 @@ from .engine.cache_client import DatasetCache
from .engine.serializer_deserializer import serialize, deserialize, show
from .engine.graphdata import GraphData
__all__ = ["config", "ImageFolderDatasetV2", "MnistDataset", "PaddedDataset",
__all__ = ["config", "ImageFolderDataset", "MnistDataset", "PaddedDataset",
"MindDataset", "GeneratorDataset", "TFRecordDataset",
"ManifestDataset", "Cifar10Dataset", "Cifar100Dataset", "CelebADataset", "NumpySlicesDataset", "VOCDataset",
"CocoDataset", "TextFileDataset", "CLUEDataset", "CSVDataset", "Schema", "DistributedSampler", "PKSampler",

31
mindspore/dataset/core/py_util_helpers.py Normal file
View File

@ -0,0 +1,31 @@
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
General py_transforms_utils functions.
"""
import numpy as np
def is_numpy(img):
"""
Check if the input image is Numpy format.
Args:
img: Image to be checked.
Returns:
Bool, True if input is Numpy image.
"""
return isinstance(img, np.ndarray)

2
mindspore/dataset/engine/__init__.py
View File

@ -28,7 +28,7 @@ from .serializer_deserializer import serialize, deserialize, show, compare
from .samplers import *
from ..core import config
__all__ = ["config", "zip", "ImageFolderDatasetV2", "MnistDataset",
__all__ = ["config", "zip", "ImageFolderDataset", "MnistDataset",
"MindDataset", "GeneratorDataset", "TFRecordDataset", "CLUEDataset", "CSVDataset",
"ManifestDataset", "Cifar10Dataset", "Cifar100Dataset", "CelebADataset",
"VOCDataset", "CocoDataset", "TextFileDataset", "Schema", "DistributedSampler",

138
mindspore/dataset/engine/datasets.py
View File

@ -41,7 +41,7 @@ from . import samplers
from .iterators import DictIterator, TupleIterator, DummyIterator, SaveOp, Iterator
from .validators import check_batch, check_shuffle, check_map, check_filter, check_repeat, check_skip, check_zip, \
check_rename, check_numpyslicesdataset, check_device_send, \
check_take, check_project, check_imagefolderdatasetv2, check_mnist_cifar_dataset, check_manifestdataset, \
check_take, check_project, check_imagefolderdataset, check_mnist_cifar_dataset, check_manifestdataset, \
check_tfrecorddataset, check_vocdataset, check_cocodataset, check_celebadataset, check_minddataset, \
check_generatordataset, check_sync_wait, check_zip_dataset, check_add_column, check_textfiledataset, check_concat, \
check_random_dataset, check_split, check_bucket_batch_by_length, check_cluedataset, check_save, check_csvdataset, \
@ -81,8 +81,8 @@ def zip(datasets):
>>>
>>> dataset_dir1 = "path/to/imagefolder_directory1"
>>> dataset_dir2 = "path/to/imagefolder_directory2"
>>> ds1 = ds.ImageFolderDatasetV2(dataset_dir1, num_parallel_workers=8)
>>> ds2 = ds.ImageFolderDatasetV2(dataset_dir2, num_parallel_workers=8)
>>> ds1 = ds.ImageFolderDataset(dataset_dir1, num_parallel_workers=8)
>>> ds2 = ds.ImageFolderDataset(dataset_dir2, num_parallel_workers=8)
>>>
>>> # creates a dataset which is the combination of ds1 and ds2
>>> data = ds.zip((ds1, ds2))
@ -246,7 +246,7 @@ class Dataset:
@check_batch
def batch(self, batch_size, drop_remainder=False, num_parallel_workers=None, per_batch_map=None,
input_columns=None, pad_info=None):
input_columns=None, output_columns=None, column_order=None, pad_info=None):
"""
Combine batch_size number of consecutive rows into batches.
@ -272,6 +272,18 @@ class Dataset:
The last parameter of the callable should always be a BatchInfo object.
input_columns (list[str], optional): List of names of the input columns. The size of the list should
match with signature of per_batch_map callable.
output_columns (list[str], optional): [Not currently implmented] List of names assigned to the columns
outputted by the last operation. This parameter is mandatory if len(input_columns) !=
len(output_columns). The size of this list must match the number of output
columns of the last operation. (default=None, output columns will have the same
name as the input columns, i.e., the columns will be replaced).
column_order (list[str], optional): [Not currently implmented] list of all the desired columns to
propagate to the child node. This list must be a subset of all the columns in the dataset after
all operations are applied. The order of the columns in each row propagated to the
child node follow the order they appear in this list. The parameter is mandatory
if the len(input_columns) != len(output_columns). (default=None, all columns
will be propagated to the child node, the order of the columns will remain the
same).
pad_info (dict, optional): Whether to perform padding on selected columns. pad_info={"col1":([224,224],0)}
would pad column with name "col1" to a tensor of size [224,224] and fill the missing with 0.
@ -286,7 +298,7 @@ class Dataset:
>>> data = data.batch(100, True)
"""
return BatchDataset(self, batch_size, drop_remainder, num_parallel_workers, per_batch_map, input_columns,
pad_info)
output_columns, column_order, pad_info)
@check_sync_wait
def sync_wait(self, condition_name, num_batch=1, callback=None):
@ -367,7 +379,7 @@ class Dataset:
>>> # declare a function which returns a Dataset object
>>> def flat_map_func(x):
>>> data_dir = text.to_str(x[0])
>>> d = ds.ImageFolderDatasetV2(data_dir)
>>> d = ds.ImageFolderDataset(data_dir)
>>> return d
>>> # data is a Dataset object
>>> data = ds.TextFileDataset(DATA_FILE)
@ -394,7 +406,7 @@ class Dataset:
return dataset
@check_map
def map(self, input_columns=None, operations=None, output_columns=None, columns_order=None,
def map(self, operations=None, input_columns=None, output_columns=None, column_order=None,
num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None):
"""
Apply each operation in operations to this dataset.
@ -409,23 +421,23 @@ class Dataset:
The columns outputted by the very last operation will be assigned names specified by
output_columns.
Only the columns specified in columns_order will be propagated to the child node. These
columns will be in the same order as specified in columns_order.
Only the columns specified in column_order will be propagated to the child node. These
columns will be in the same order as specified in column_order.
Args:
operations (Union[list[TensorOp], list[functions]]): List of operations to be
applied on the dataset. Operations are applied in the order they appear in this list.
input_columns (list[str]): List of the names of the columns that will be passed to
the first operation as input. The size of this list must match the number of
input columns expected by the first operator. (default=None, the first
operation will be passed however many columns that is required, starting from
the first column).
operations (Union[list[TensorOp], list[functions]]): List of operations to be
applied on the dataset. Operations are applied in the order they appear in this list.
output_columns (list[str], optional): List of names assigned to the columns outputted by
the last operation. This parameter is mandatory if len(input_columns) !=
len(output_columns). The size of this list must match the number of output
columns of the last operation. (default=None, output columns will have the same
name as the input columns, i.e., the columns will be replaced).
columns_order (list[str], optional): list of all the desired columns to propagate to the
column_order (list[str], optional): list of all the desired columns to propagate to the
child node. This list must be a subset of all the columns in the dataset after
all operations are applied. The order of the columns in each row propagated to the
child node follow the order they appear in this list. The parameter is mandatory
@ -446,7 +458,7 @@ class Dataset:
Examples:
>>> import mindspore.dataset as ds
>>> import mindspore.dataset.transforms.vision.c_transforms as c_transforms
>>> import mindspore.dataset.vision.c_transforms as c_transforms
>>>
>>> # data is an instance of Dataset which has 2 columns, "image" and "label".
>>> # ds_pyfunc is an instance of Dataset which has 3 columns, "col0", "col1", and "col2". Each column is
@ -468,33 +480,33 @@ class Dataset:
>>> input_columns = ["image"]
>>>
>>> # Applies decode_op on column "image". This column will be replaced by the outputed
>>> # column of decode_op. Since columns_order is not provided, both columns "image"
>>> # column of decode_op. Since column_order is not provided, both columns "image"
>>> # and "label" will be propagated to the child node in their original order.
>>> ds_decoded = data.map(input_columns, operations)
>>> ds_decoded = data.map(operations, input_columns)
>>>
>>> # Rename column "image" to "decoded_image"
>>> output_columns = ["decoded_image"]
>>> ds_decoded = data.map(input_columns, operations, output_columns)
>>> ds_decoded = data.map(operations, input_columns, output_columns)
>>>
>>> # Specify the order of the columns.
>>> columns_order ["label", "image"]
>>> ds_decoded = data.map(input_columns, operations, None, columns_order)
>>> column_order ["label", "image"]
>>> ds_decoded = data.map(operations, input_columns, None, column_order)
>>>
>>> # Rename column "image" to "decoded_image" and also specify the order of the columns.
>>> columns_order ["label", "decoded_image"]
>>> column_order ["label", "decoded_image"]
>>> output_columns = ["decoded_image"]
>>> ds_decoded = data.map(input_columns, operations, output_columns, columns_order)
>>> ds_decoded = data.map(operations, input_columns, output_columns, column_order)
>>>
>>> # Rename column "image" to "decoded_image" and keep only this column.
>>> columns_order ["decoded_image"]
>>> column_order ["decoded_image"]
>>> output_columns = ["decoded_image"]
>>> ds_decoded = data.map(input_columns, operations, output_columns, columns_order)
>>> ds_decoded = data.map(operations, input_columns, output_columns, column_order)
>>>
>>> # Simple example using pyfunc. Renaming columns and specifying column order
>>> # work in the same way as the previous examples.
>>> input_columns = ["col0"]
>>> operations = [(lambda x: x + 1)]
>>> ds_mapped = ds_pyfunc.map(input_columns, operations)
>>> ds_mapped = ds_pyfunc.map(operations, input_columns)
>>>
>>> # 2) Map example with more than one operation
>>>
@ -509,22 +521,22 @@ class Dataset:
>>> # outputted by decode_op is passed as input to random_jitter_op.
>>> # random_jitter_op will output one column. Column "image" will be replaced by
>>> # the column outputted by random_jitter_op (the very last operation). All other
>>> # columns are unchanged. Since columns_order is not specified, the order of the
>>> # columns are unchanged. Since column_order is not specified, the order of the
>>> # columns will remain the same.
>>> ds_mapped = data.map(input_columns, operations)
>>> ds_mapped = data.map(operations, input_columns)
>>>
>>> # Creates a dataset that is identical to ds_mapped, except the column "image"
>>> # that is outputted by random_jitter_op is renamed to "image_transformed".
>>> # Specifying column order works in the same way as examples in 1).
>>> output_columns = ["image_transformed"]
>>> ds_mapped_and_renamed = data.map(input_columns, operation, output_columns)
>>> ds_mapped_and_renamed = data.map(operation, input_columns, output_columns)
>>>
>>> # Multiple operations using pyfunc. Renaming columns and specifying column order
>>> # work in the same way as examples in 1).
>>> input_columns = ["col0"]
>>> operations = [(lambda x: x + x), (lambda x: x - 1)]
>>> output_columns = ["col0_mapped"]
>>> ds_mapped = ds_pyfunc.map(input_columns, operations, output_columns)
>>> ds_mapped = ds_pyfunc.map(operations, input_columns, output_columns)
>>>
>>> # 3) Example where number of input columns is not equal to number of output columns
>>>
@ -540,20 +552,21 @@ class Dataset:
>>> (lambda x: (x % 2, x % 3, x % 5, x % 7))]
>>>
>>> # Note: because the number of input columns is not the same as the number of
>>> # output columns, the output_columns and columns_order parameter must be
>>> # output columns, the output_columns and column_order parameter must be
>>> # specified. Otherwise, this map call will also result in an error.
>>> input_columns = ["col2", "col0"]
>>> output_columns = ["mod2", "mod3", "mod5", "mod7"]
>>>
>>> # Propagate all columns to the child node in this order:
>>> columns_order = ["col0", "col2", "mod2", "mod3", "mod5", "mod7", "col1"]
>>> ds_mapped = ds_pyfunc.map(input_columns, operations, output_columns, columns_order)
>>> column_order = ["col0", "col2", "mod2", "mod3", "mod5", "mod7", "col1"]
>>> ds_mapped = ds_pyfunc.map(operations, input_columns, output_columns, column_order)
>>>
>>> # Propagate some columns to the child node in this order:
>>> columns_order = ["mod7", "mod3", "col1"]
>>> ds_mapped = ds_pyfunc.map(input_columns, operations, output_columns, columns_order)
>>> column_order = ["mod7", "mod3", "col1"]
>>> ds_mapped = ds_pyfunc.map(operations, input_columns, output_columns, column_order)
"""
return MapDataset(self, input_columns, operations, output_columns, columns_order, num_parallel_workers,
return MapDataset(self, operations, input_columns, output_columns, column_order, num_parallel_workers,
python_multiprocessing, cache, callbacks)
@check_filter
@ -1012,7 +1025,7 @@ class Dataset:
def get_distribution(output_dataset):
dev_id = 0
if isinstance(output_dataset, (Cifar10Dataset, Cifar100Dataset, GeneratorDataset, ImageFolderDatasetV2,
if isinstance(output_dataset, (Cifar10Dataset, Cifar100Dataset, GeneratorDataset, ImageFolderDataset,
ManifestDataset, MnistDataset, VOCDataset, CocoDataset, CelebADataset,
MindDataset)):
sampler = output_dataset.sampler
@ -1412,7 +1425,7 @@ class MappableDataset(SourceDataset):
>>>
>>> dataset_dir = "/path/to/imagefolder_directory"
>>> # a SequentialSampler is created by default
>>> data = ds.ImageFolderDatasetV2(dataset_dir)
>>> data = ds.ImageFolderDataset(dataset_dir)
>>>
>>> # use a DistributedSampler instead of the SequentialSampler
>>> new_sampler = ds.DistributedSampler(10, 2)
@ -1501,7 +1514,7 @@ class MappableDataset(SourceDataset):
>>> dataset_dir = "/path/to/imagefolder_directory"
>>>
>>> # many datasets have shuffle on by default, set shuffle to False if split will be called!
>>> data = ds.ImageFolderDatasetV2(dataset_dir, shuffle=False)
>>> data = ds.ImageFolderDataset(dataset_dir, shuffle=False)
>>>
>>> # sets the seed, and tells split to use this seed when randomizing. This
>>> # is needed because we are sharding later
@ -1629,13 +1642,25 @@ class BatchDataset(DatasetOp):
last parameter of the callable should always be a BatchInfo object.
input_columns (list[str], optional): List of names of the input columns. The size of the list should
match with signature of per_batch_map callable.
output_columns (list[str], optional): List of names assigned to the columns outputted by
the last operation. This parameter is mandatory if len(input_columns) !=
len(output_columns). The size of this list must match the number of output
columns of the last operation. (default=None, output columns will have the same
name as the input columns, i.e., the columns will be replaced).
column_order (list[str], optional): list of all the desired columns to propagate to the
child node. This list must be a subset of all the columns in the dataset after
all operations are applied. The order of the columns in each row propagated to the
child node follow the order they appear in this list. The parameter is mandatory
if the len(input_columns) != len(output_columns). (default=None, all columns
will be propagated to the child node, the order of the columns will remain the
same).
pad_info (dict, optional): Whether to perform padding on selected columns. pad_info={"col1":([224,224],0)}
would pad column with name "col1" to a tensor of size [224,224] and fill the missing with 0.
"""
def __init__(self, input_dataset, batch_size, drop_remainder=False, num_parallel_workers=None,
per_batch_map=None, input_columns=None, pad_info=None):
per_batch_map=None, input_columns=None, output_columns=None, column_order=None, pad_info=None):
super().__init__(num_parallel_workers)
if BatchDataset._is_ancestor_of_repeat(input_dataset):
@ -1647,6 +1672,8 @@ class BatchDataset(DatasetOp):
self.drop_remainder = drop_remainder
self.per_batch_map = per_batch_map
self.input_columns = input_columns
self.output_columns = output_columns
self.column_order = column_order
self.pad_info = pad_info
self.children.append(input_dataset)
input_dataset.parent.append(self)
@ -1962,16 +1989,16 @@ class MapDataset(DatasetOp):
Args:
input_dataset (Dataset): Input Dataset to be mapped.
operations (TensorOp): A function mapping a nested structure of tensors
to another nested structure of tensor (default=None).
input_columns (list[str]): List of names of the input columns
(default=None, the operations will be applied on the first columns in the dataset).
The size of the list should match the number of inputs of the first operator.
operations (TensorOp): A function mapping a nested structure of tensors
to another nested structure of tensor (default=None).
output_columns (list[str], optional): list of names of the output columns.
The size of the list should match the number of outputs of the last operator
(default=None, output columns will be the input columns, i.e., the columns will
be replaced).
columns_order (list[str], optional): list of all the desired columns of the dataset (default=None).
column_order (list[str], optional): list of all the desired columns of the dataset (default=None).
The argument is mandatory if len(input_columns) != len(output_columns).
num_parallel_workers (int, optional): Number of workers to process the Dataset
in parallel (default=None).
@ -1982,29 +2009,29 @@ class MapDataset(DatasetOp):
callbacks: (DSCallback, list[DSCallback], optional): list of Dataset callbacks to be called (Default=None)
Raises:
ValueError: If len(input_columns) != len(output_columns) and columns_order is not specified.
ValueError: If len(input_columns) != len(output_columns) and column_order is not specified.
"""
def __init__(self, input_dataset, input_columns=None, operations=None, output_columns=None, columns_order=None,
def __init__(self, input_dataset, operations=None, input_columns=None, output_columns=None, column_order=None,
num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None):
super().__init__(num_parallel_workers)
self.children.append(input_dataset)
if input_columns is not None and not isinstance(input_columns, list):
input_columns = [input_columns]
self.input_columns = input_columns
if operations is not None and not isinstance(operations, list):
operations = [operations]
self.operations = operations
if input_columns is not None and not isinstance(input_columns, list):
input_columns = [input_columns]
self.input_columns = input_columns
if output_columns is not None and not isinstance(output_columns, list):
output_columns = [output_columns]
self.output_columns = output_columns
self.cache = cache
self.columns_order = columns_order
self.column_order = column_order
if self.input_columns and self.output_columns \
and len(self.input_columns) != len(self.output_columns) \
and self.columns_order is None:
raise ValueError("When (len(input_columns) != len(output_columns)), columns_order must be specified.")
and self.column_order is None:
raise ValueError("When (len(input_columns) != len(output_columns)), column_order must be specified.")
input_dataset.parent.append(self)
self._input_indexs = input_dataset.input_indexs
@ -2021,7 +2048,7 @@ class MapDataset(DatasetOp):
args["input_columns"] = self.input_columns
args["operations"] = self.operations
args["output_columns"] = self.output_columns
args["columns_order"] = self.columns_order
args["column_order"] = self.column_order
args["cache"] = self.cache.cache_client if self.cache is not None else None
if self.callbacks is not None:
@ -2048,7 +2075,7 @@ class MapDataset(DatasetOp):
new_op.children = copy.deepcopy(self.children, memodict)
new_op.input_columns = copy.deepcopy(self.input_columns, memodict)
new_op.output_columns = copy.deepcopy(self.output_columns, memodict)
new_op.columns_order = copy.deepcopy(self.columns_order, memodict)
new_op.column_order = copy.deepcopy(self.column_order, memodict)
new_op.num_parallel_workers = copy.deepcopy(self.num_parallel_workers, memodict)
new_op.parent = copy.deepcopy(self.parent, memodict)
new_op.ms_role = copy.deepcopy(self.ms_role, memodict)
@ -2646,7 +2673,7 @@ def _select_sampler(num_samples, input_sampler, shuffle, num_shards, shard_id, n
return samplers.SequentialSampler(num_samples=num_samples)
class ImageFolderDatasetV2(MappableDataset):
class ImageFolderDataset(MappableDataset):
"""
A source dataset that reads images from a tree of directories.
@ -2722,14 +2749,14 @@ class ImageFolderDatasetV2(MappableDataset):
>>> # path to imagefolder directory. This directory needs to contain sub-directories which contain the images
>>> dataset_dir = "/path/to/imagefolder_directory"
>>> # 1) read all samples (image files) in dataset_dir with 8 threads
>>> imagefolder_dataset = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8)
>>> imagefolder_dataset = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8)
>>> # 2) read all samples (image files) from folder cat and folder dog with label 0 and 1
>>> imagefolder_dataset = ds.ImageFolderDatasetV2(dataset_dir,class_indexing={"cat":0,"dog":1})
>>> imagefolder_dataset = ds.ImageFolderDataset(dataset_dir,class_indexing={"cat":0,"dog":1})
>>> # 3) read all samples (image files) in dataset_dir with extensions .JPEG and .png (case sensitive)
>>> imagefolder_dataset = ds.ImageFolderDatasetV2(dataset_dir, extensions=[".JPEG",".png"])
>>> imagefolder_dataset = ds.ImageFolderDataset(dataset_dir, extensions=[".JPEG",".png"])
"""
@check_imagefolderdatasetv2
@check_imagefolderdataset
def __init__(self, dataset_dir, num_samples=None, num_parallel_workers=None,
shuffle=None, sampler=None, extensions=None, class_indexing=None,
decode=False, num_shards=None, shard_id=None, cache=None):
@ -3168,6 +3195,7 @@ class SamplerFn:
"""
Multiprocessing or multithread generator function wrapper master process.
"""
def __init__(self, dataset, num_worker, multi_process):
self.workers = []
self.num_worker = num_worker

2
mindspore/dataset/engine/iterators.py
View File

@ -150,7 +150,7 @@ class Iterator:
op_type = OpName.SKIP
elif isinstance(dataset, de.TakeDataset):
op_type = OpName.TAKE
elif isinstance(dataset, de.ImageFolderDatasetV2):
elif isinstance(dataset, de.ImageFolderDataset):
op_type = OpName.IMAGEFOLDER
elif isinstance(dataset, de.GeneratorDataset):
op_type = OpName.GENERATOR

14
mindspore/dataset/engine/samplers.py
View File

@ -41,7 +41,7 @@ class Sampler:
>>> for i in range(self.dataset_size - 1, -1, -1):
>>> yield i
>>>
>>> ds = ds.ImageFolderDatasetV2(path, sampler=ReverseSampler())
>>> ds = ds.ImageFolderDataset(path, sampler=ReverseSampler())
"""
def __init__(self, num_samples=None):
@ -232,7 +232,7 @@ class DistributedSampler(BuiltinSampler):
>>>
>>> # creates a distributed sampler with 10 shards total. This shard is shard 5
>>> sampler = ds.DistributedSampler(10, 5)
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
Raises:
ValueError: If num_shards is not positive.
@ -315,7 +315,7 @@ class PKSampler(BuiltinSampler):
>>>
>>> # creates a PKSampler that will get 3 samples from every class.
>>> sampler = ds.PKSampler(3)
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
Raises:
ValueError: If num_val is not positive.
@ -387,7 +387,7 @@ class RandomSampler(BuiltinSampler):
>>>
>>> # creates a RandomSampler
>>> sampler = ds.RandomSampler()
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
Raises:
ValueError: If replacement is not boolean.
@ -447,7 +447,7 @@ class SequentialSampler(BuiltinSampler):
>>>
>>> # creates a SequentialSampler
>>> sampler = ds.SequentialSampler()
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
"""
def __init__(self, start_index=None, num_samples=None):
@ -510,7 +510,7 @@ class SubsetRandomSampler(BuiltinSampler):
>>>
>>> # creates a SubsetRandomSampler, will sample from the provided indices
>>> sampler = ds.SubsetRandomSampler()
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
"""
def __init__(self, indices, num_samples=None):
@ -573,7 +573,7 @@ class WeightedRandomSampler(BuiltinSampler):
>>>
>>> # creates a WeightedRandomSampler that will sample 4 elements without replacement
>>> sampler = ds.WeightedRandomSampler(weights, 4)
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
Raises:
ValueError: If num_samples is not positive.

13
mindspore/dataset/engine/serializer_deserializer.py
View File

@ -21,9 +21,10 @@ import sys
from mindspore import log as logger
from . import datasets as de
from ..transforms.vision.utils import Inter, Border
from ..vision.utils import Inter, Border
from ..core import config
def serialize(dataset, json_filepath=None):
"""
Serialize dataset pipeline into a json file.
@ -44,7 +45,7 @@ def serialize(dataset, json_filepath=None):
>>> DATA_DIR = "../../data/testMnistData"
>>> data = ds.MnistDataset(DATA_DIR, 100)
>>> one_hot_encode = C.OneHot(10) # num_classes is input argument
>>> data = data.map(input_column_names="label", operation=one_hot_encode)
>>> data = data.map(operation=one_hot_encode, input_column_names="label")
>>> data = data.batch(batch_size=10, drop_remainder=True)
>>>
>>> ds.engine.serialize(data, json_filepath="mnist_dataset_pipeline.json") # serialize it to json file
@ -77,7 +78,7 @@ def deserialize(input_dict=None, json_filepath=None):
>>> DATA_DIR = "../../data/testMnistData"
>>> data = ds.MnistDataset(DATA_DIR, 100)
>>> one_hot_encode = C.OneHot(10) # num_classes is input argument
>>> data = data.map(input_column_names="label", operation=one_hot_encode)
>>> data = data.map(operation=one_hot_encode, input_column_names="label")
>>> data = data.batch(batch_size=10, drop_remainder=True)
>>>
>>> # Use case 1: to/from json file
@ -254,7 +255,7 @@ def create_node(node):
pyobj = None
# Find a matching Dataset class and call the constructor with the corresponding args.
# When a new Dataset class is introduced, another if clause and parsing code needs to be added.
if dataset_op == 'ImageFolderDatasetV2':
if dataset_op == 'ImageFolderDataset':
sampler = construct_sampler(node.get('sampler'))
pyobj = pyclass(node['dataset_dir'], node.get('num_samples'), node.get('num_parallel_workers'),
node.get('shuffle'), sampler, node.get('extensions'),
@ -336,8 +337,8 @@ def create_node(node):
elif dataset_op == 'MapDataset':
tensor_ops = construct_tensor_ops(node.get('operations'))
pyobj = de.Dataset().map(node.get('input_columns'), tensor_ops, node.get('output_columns'),
node.get('columns_order'), node.get('num_parallel_workers'))
pyobj = de.Dataset().map(tensor_ops, node.get('input_columns'), node.get('output_columns'),
node.get('column_order'), node.get('num_parallel_workers'))
elif dataset_op == 'ShuffleDataset':
pyobj = de.Dataset().shuffle(node.get('buffer_size'))

24
mindspore/dataset/engine/validators.py
View File

@ -35,8 +35,8 @@ from . import cache_client
from .. import callback
def check_imagefolderdatasetv2(method):
"""A wrapper that wraps a parameter checker around the original Dataset(ImageFolderDatasetV2)."""
def check_imagefolderdataset(method):
"""A wrapper that wraps a parameter checker around the original Dataset(ImageFolderDataset)."""
@wraps(method)
def new_method(self, *args, **kwargs):
@ -474,8 +474,8 @@ def check_batch(method):
@wraps(method)
def new_method(self, *args, **kwargs):
[batch_size, drop_remainder, num_parallel_workers, per_batch_map,
input_columns, pad_info], param_dict = parse_user_args(method, *args, **kwargs)
[batch_size, drop_remainder, num_parallel_workers, per_batch_map, input_columns, output_columns,
column_order, pad_info], param_dict = parse_user_args(method, *args, **kwargs)
if not (isinstance(batch_size, int) or (callable(batch_size))):
raise TypeError("batch_size should either be an int or a callable.")
@ -510,6 +510,12 @@ def check_batch(method):
if len(input_columns) != (len(ins.signature(per_batch_map).parameters) - 1):
raise ValueError("the signature of per_batch_map should match with input columns")
if output_columns is not None:
raise ValueError("output_columns is currently not implemented.")
if column_order is not None:
raise ValueError("column_order is currently not implemented.")
return method(self, *args, **kwargs)
return new_method
@ -551,14 +557,14 @@ def check_map(method):
@wraps(method)
def new_method(self, *args, **kwargs):
[input_columns, _, output_columns, columns_order, num_parallel_workers, python_multiprocessing, cache,
[_, input_columns, output_columns, column_order, num_parallel_workers, python_multiprocessing, cache,
callbacks], _ = \
parse_user_args(method, *args, **kwargs)
nreq_param_columns = ['input_columns', 'output_columns', 'columns_order']
nreq_param_columns = ['input_columns', 'output_columns', 'column_order']
if columns_order is not None:
type_check(columns_order, (list,), "columns_order")
if column_order is not None:
type_check(column_order, (list,), "column_order")
if num_parallel_workers is not None:
check_num_parallel_workers(num_parallel_workers)
type_check(python_multiprocessing, (bool,), "python_multiprocessing")
@ -571,7 +577,7 @@ def check_map(method):
else:
type_check(callbacks, (callback.DSCallback,), "callbacks")
for param_name, param in zip(nreq_param_columns, [input_columns, output_columns, columns_order]):
for param_name, param in zip(nreq_param_columns, [input_columns, output_columns, column_order]):
if param is not None:
check_columns(param, param_name)
if callbacks is not None:

20
mindspore/dataset/text/transforms.py
View File

@ -103,7 +103,6 @@ class SlidingWindow(cde.SlidingWindowOp):
super().__init__(width, axis)
class Ngram(cde.NgramOp):
"""
TensorOp to generate n-gram from a 1-D string Tensor.
@ -161,8 +160,9 @@ class JiebaTokenizer(cde.JiebaTokenizerOp):
>>> # If with_offsets=False, then output three columns {["token", dtype=str], ["offsets_start", dtype=uint32],
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> data = data.map(operations=tokenizer_op, input_columns=["text"],
>>> output_columns=["token", "offsets_start", "offsets_limit"],
>>> column_order=["token", "offsets_start", "offsets_limit"])
"""
@check_jieba_init
@ -281,7 +281,7 @@ class UnicodeCharTokenizer(cde.UnicodeCharTokenizerOp):
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = text.UnicodeCharTokenizer(True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_with_offsets
@ -312,7 +312,7 @@ class WordpieceTokenizer(cde.WordpieceTokenizerOp):
>>> tokenizer_op = text.WordpieceTokenizer(vocab=vocab, unknown_token=['UNK'],
>>> max_bytes_per_token=100, with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_wordpiece_tokenizer
@ -377,7 +377,7 @@ if platform.system().lower() != 'windows':
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = text.WhitespaceTokenizer(True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_with_offsets
@ -403,7 +403,7 @@ if platform.system().lower() != 'windows':
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = text.UnicodeScriptTokenizerOp(keep_whitespace=True, with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_unicode_script_tokenizer
@ -496,7 +496,7 @@ if platform.system().lower() != 'windows':
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = text.RegexTokenizer(delim_pattern, keep_delim_pattern, with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_regex_tokenizer
@ -539,7 +539,7 @@ if platform.system().lower() != 'windows':
>>> preserve_unused_token=True,
>>> with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_basic_tokenizer
@ -592,7 +592,7 @@ if platform.system().lower() != 'windows':
>>> normalization_form=NormalizeForm.NONE, preserve_unused_token=True,
>>> with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_bert_tokenizer

2
mindspore/dataset/transforms/__init__.py
View File

@ -16,6 +16,6 @@ This module is to support common augmentations. C_transforms is a high performan
image augmentation module which is developed with C++ OpenCV. Py_transforms
provide more kinds of image augmentations which is developed with Python PIL.
"""
from . import vision
from .. import vision
from . import c_transforms
from . import py_transforms

4
mindspore/dataset/transforms/c_transforms.py
View File

@ -229,8 +229,8 @@ class Duplicate(cde.DuplicateOp):
>>> # +---------+
>>> # | [1,2,3] |
>>> # +---------+
>>> data = data.map(input_columns=["x"], operations=Duplicate(),
>>> output_columns=["x", "y"], columns_order=["x", "y"])
>>> data = data.map(operations=Duplicate(), input_columns=["x"],
>>> output_columns=["x", "y"], column_order=["x", "y"])
>>> # Data after
>>> # | x | y |
>>> # +---------+---------+

50
mindspore/dataset/transforms/py_transforms.py
View File

@ -17,9 +17,8 @@
This module py_transforms is implemented basing on Python. It provides common
operations including OneHotOp.
"""
from .validators import check_one_hot_op
from .vision import py_transforms_util as util
from .validators import check_one_hot_op, check_compose_list
from . import py_transforms_util as util
class OneHotOp:
@ -48,3 +47,48 @@ class OneHotOp:
label (numpy.ndarray), label after being Smoothed.
"""
return util.one_hot_encoding(label, self.num_classes, self.smoothing_rate)
class Compose:
"""
Compose a list of transforms.
.. Note::
Compose takes a list of transformations either provided in py_transforms or from user-defined implementation;
each can be an initialized transformation class or a lambda function, as long as the output from the last
transformation is a single tensor of type numpy.ndarray. See below for an example of how to use Compose
with py_transforms classes and check out FiveCrop or TenCrop for the use of them in conjunction with lambda
functions.
Args:
transforms (list): List of transformations to be applied.
Examples:
>>> import mindspore.dataset as ds
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>> dataset_dir = "path/to/imagefolder_directory"
>>> # create a dataset that reads all files in dataset_dir with 8 threads
>>> dataset = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8)
>>> # create a list of transformations to be applied to the image data
>>> transform = Compose([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor(),
>>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262)),
>>> py_transforms.RandomErasing()])
>>> # apply the transform to the dataset through dataset.map()
>>> dataset = dataset.map(operations=transform, input_columns="image")
"""
@check_compose_list
def __init__(self, transforms):
self.transforms = transforms
def __call__(self, img):
"""
Call method.
Returns:
lambda function, Lambda function that takes in an img to apply transformations on.
"""
return util.compose(img, self.transforms)

65
mindspore/dataset/transforms/py_transforms_util.py Normal file
View File

@ -0,0 +1,65 @@
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
Built-in py_transforms_utils functions.
"""
import numpy as np
from ..core.py_util_helpers import is_numpy
def compose(img, transforms):
"""
Compose a list of transforms and apply on the image.
Args:
img (numpy.ndarray): An image in Numpy ndarray.
transforms (list): A list of transform Class objects to be composed.
Returns:
img (numpy.ndarray), An augmented image in Numpy ndarray.
"""
if is_numpy(img):
for transform in transforms:
img = transform(img)
if is_numpy(img):
return img
raise TypeError('img should be Numpy ndarray. Got {}. Append ToTensor() to transforms'.format(type(img)))
raise TypeError('img should be Numpy ndarray. Got {}.'.format(type(img)))
def one_hot_encoding(label, num_classes, epsilon):
"""
Apply label smoothing transformation to the input label, and make label be more smoothing and continuous.
Args:
label (numpy.ndarray): label to be applied label smoothing.
num_classes (int): Num class of object in dataset, value should over 0.
epsilon (float): The adjustable Hyper parameter. Default is 0.0.
Returns:
img (numpy.ndarray), label after being one hot encoded and done label smoothed.
"""
if label > num_classes:
raise ValueError('the num_classes is smaller than the category number.')
num_elements = label.size
one_hot_label = np.zeros((num_elements, num_classes), dtype=int)
if isinstance(label, list) is False:
label = [label]
for index in range(num_elements):
one_hot_label[index, label[index]] = 1
return (1 - epsilon) * one_hot_label + epsilon / num_classes

16
mindspore/dataset/transforms/validators.py
View File

@ -200,3 +200,19 @@ def check_random_transform_ops(method):
return method(self, *args, **kwargs)
return new_method
def check_compose_list(method):
"""Wrapper method to check the transform list of Compose."""
@wraps(method)
def new_method(self, *args, **kwargs):
[transforms], _ = parse_user_args(method, *args, **kwargs)
type_check(transforms, (list,), transforms)
if not transforms:
raise ValueError("transforms list is empty.")
return method(self, *args, **kwargs)
return new_method

0
mindspore/dataset/transforms/vision/__init__.py → mindspore/dataset/vision/__init__.py
View File

7
mindspore/dataset/transforms/vision/c_transforms.py → mindspore/dataset/vision/c_transforms.py
View File

@ -25,11 +25,12 @@ to improve their training models.
Examples:
>>> import mindspore.dataset as ds
>>> import mindspore.dataset.transforms.c_transforms as c_transforms
>>> import mindspore.dataset.transforms.vision.c_transforms as c_vision
>>> import mindspore.dataset.vision.c_transforms as c_vision
>>> from mindspore.dataset.transforms.vision.utils import Border, ImageBatchFormat, Inter
>>> dataset_dir = "path/to/imagefolder_directory"
>>> # create a dataset that reads all files in dataset_dir with 8 threads
>>> data1 = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8)
>>> data1 = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8)
>>> # create a list of transformations to be applied to the image data
>>> transforms_list = [c_vision.Decode(),
>>> c_vision.Resize((256, 256)),
@ -1095,7 +1096,7 @@ class UniformAugment(cde.UniformAugOp):
num_ops (int, optional): Number of operations to be selected and applied (default=2).
Examples:
>>> import mindspore.dataset.transforms.vision.py_transforms as py_vision
>>> import mindspore.dataset.vision.py_transforms as py_vision
>>> transforms_list = [c_vision.RandomHorizontalFlip(),
>>> c_vision.RandomVerticalFlip(),
>>> c_vision.RandomColorAdjust(),

345
mindspore/dataset/transforms/vision/py_transforms.py → mindspore/dataset/vision/py_transforms.py
View File

@ -33,7 +33,7 @@ from .validators import check_prob, check_crop, check_resize_interpolation, chec
check_normalize_py, check_random_crop, check_random_color_adjust, check_random_rotation, \
check_transforms_list, check_random_apply, check_ten_crop, check_num_channels, check_pad, \
check_random_perspective, check_random_erasing, check_cutout, check_linear_transform, check_random_affine, \
check_mix_up, check_positive_degrees, check_uniform_augment_py, check_compose_list, check_auto_contrast
check_mix_up, check_positive_degrees, check_uniform_augment_py, check_auto_contrast
from .utils import Inter, Border
DE_PY_INTER_MODE = {Inter.NEAREST: Image.NEAREST,
@ -46,50 +46,6 @@ DE_PY_BORDER_TYPE = {Border.CONSTANT: 'constant',
Border.SYMMETRIC: 'symmetric'}
class ComposeOp:
"""
Compose a list of transforms.
.. Note::
ComposeOp takes a list of transformations either provided in py_transforms or from user-defined implementation;
each can be an initialized transformation class or a lambda function, as long as the output from the last
transformation is a single tensor of type numpy.ndarray. See below for an example of how to use ComposeOp
with py_transforms classes and check out FiveCrop or TenCrop for the use of them in conjunction with lambda
functions.
Args:
transforms (list): List of transformations to be applied.
Examples:
>>> import mindspore.dataset as ds
>>> import mindspore.dataset.transforms.vision.py_transforms as py_transforms
>>> dataset_dir = "path/to/imagefolder_directory"
>>> # create a dataset that reads all files in dataset_dir with 8 threads
>>> dataset = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8)
>>> # create a list of transformations to be applied to the image data
>>> transform = py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor(),
>>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262)),
>>> py_transforms.RandomErasing()])
>>> # apply the transform to the dataset through dataset.map()
>>> dataset = dataset.map(input_columns="image", operations=transform())
"""
@check_compose_list
def __init__(self, transforms):
self.transforms = transforms
def __call__(self):
"""
Call method.
Returns:
lambda function, Lambda function that takes in an image to apply transformations on.
"""
return lambda img: util.compose(img, self.transforms)
class ToTensor:
"""
Convert the input NumPy image array or PIL image of shape (H,W,C) to a NumPy ndarray of shape (C,H,W).
@ -103,9 +59,11 @@ class ToTensor:
output_type (Numpy datatype, optional): The datatype of the NumPy output (default=np.float32).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(), py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor()])
"""
def __init__(self, output_type=np.float32):
@ -132,11 +90,13 @@ class ToType:
output_type (Numpy datatype): The datatype of the NumPy output, e.g. numpy.float32.
Examples:
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>> import numpy as np
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor(),
>>> py_transforms.ToType(np.float32)])
>>>
>>> Compose([py_transforms.Decode(), py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor(),
>>> py_transforms.ToType(np.float32)])
"""
def __init__(self, output_type):
@ -179,9 +139,11 @@ class ToPIL:
Examples:
>>> # data is already decoded, but not in PIL image format
>>> py_transforms.ComposeOp([py_transforms.ToPIL(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.ToPIL(), py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor()])
"""
def __call__(self, img):
@ -202,7 +164,10 @@ class Decode:
Decode the input image to PIL image format in RGB mode.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor()])
"""
@ -233,10 +198,13 @@ class Normalize:
The standard deviation values must be in range (0.0, 1.0].
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor(),
>>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262))])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor(),
>>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262))])
"""
@check_normalize_py
@ -291,9 +259,12 @@ class RandomCrop:
value of edge.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomCrop(224),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomCrop(224),
>>> py_transforms.ToTensor()])
"""
@check_random_crop
@ -330,9 +301,12 @@ class RandomHorizontalFlip:
prob (float, optional): Probability of the image being flipped (default=0.5).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor()])
"""
@check_prob
@ -360,9 +334,12 @@ class RandomVerticalFlip:
prob (float, optional): Probability of the image being flipped (default=0.5).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomVerticalFlip(0.5),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomVerticalFlip(0.5),
>>> py_transforms.ToTensor()])
"""
@check_prob
@ -401,9 +378,12 @@ class Resize:
- Inter.BICUBIC, means interpolation method is bicubic interpolation.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.Resize(256),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.Resize(256),
>>> py_transforms.ToTensor()])
"""
@check_resize_interpolation
@ -448,9 +428,12 @@ class RandomResizedCrop:
crop area (default=10). If exceeded, fall back to use center crop instead.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomResizedCrop(224),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomResizedCrop(224),
>>> py_transforms.ToTensor()])
"""
@check_random_resize_crop
@ -486,9 +469,12 @@ class CenterCrop:
If size is a sequence of length 2, it should be (height, width).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.CenterCrop(64),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.CenterCrop(64),
>>> py_transforms.ToTensor()])
"""
@check_crop
@ -527,9 +513,12 @@ class RandomColorAdjust:
If it is a sequence, it should be [min, max] where -0.5 <= min <= max <= 0.5.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomColorAdjust(0.4, 0.4, 0.4, 0.1),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomColorAdjust(0.4, 0.4, 0.4, 0.1),
>>> py_transforms.ToTensor()])
"""
@check_random_color_adjust
@ -585,9 +574,12 @@ class RandomRotation:
If it is an int, it is used for all RGB channels. Default is 0.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomRotation(30),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomRotation(30),
>>> py_transforms.ToTensor()])
"""
@check_random_rotation
@ -619,10 +611,12 @@ class RandomOrder:
transforms (list): List of the transformations to be applied.
Examples:
>>> transforms_list = [py_transforms.CenterCrop(64), py_transforms.RandomRotation(30)]
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomOrder(transforms_list),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomOrder(transforms_list),
>>> py_transforms.ToTensor()])
"""
@check_transforms_list
@ -651,10 +645,12 @@ class RandomApply:
prob (float, optional): The probability to apply the transformation list (default=0.5).
Examples:
>>> transforms_list = [py_transforms.CenterCrop(64), py_transforms.RandomRotation(30)]
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomApply(transforms_list, prob=0.6),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomApply(transforms_list, prob=0.6),
>>> py_transforms.ToTensor()])
"""
@check_random_apply
@ -683,10 +679,12 @@ class RandomChoice:
transforms (list): List of transformations to be chosen from to apply.
Examples:
>>> transforms_list = [py_transforms.CenterCrop(64), py_transforms.RandomRotation(30)]
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomChoice(transforms_list),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomChoice(transforms_list),
>>> py_transforms.ToTensor()])
"""
@check_transforms_list
@ -716,10 +714,13 @@ class FiveCrop:
If size is a sequence of length 2, it should be (height, width).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.FiveCrop(size),
>>> # 4D stack of 5 images
>>> lambda images: numpy.stack([py_transforms.ToTensor()(image) for image in images])])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.FiveCrop(size),
>>> # 4D stack of 5 images
>>> lambda images: numpy.stack([py_transforms.ToTensor()(image) for image in images])])
"""
@check_crop
@ -752,10 +753,13 @@ class TenCrop:
if set to True (default=False).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.TenCrop(size),
>>> # 4D stack of 10 images
>>> lambda images: numpy.stack([py_transforms.ToTensor()(image) for image in images])])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.TenCrop(size),
>>> # 4D stack of 10 images
>>> lambda images: numpy.stack([py_transforms.ToTensor()(image) for image in images])])
"""
@check_ten_crop
@ -789,9 +793,12 @@ class Grayscale:
Default is 1. If set to 3, the returned image has 3 identical RGB channels.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.Grayscale(3),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.Grayscale(3),
>>> py_transforms.ToTensor()])
"""
@check_num_channels
@ -819,9 +826,12 @@ class RandomGrayscale:
prob (float, optional): Probability of the image being converted to grayscale (default=0.1).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomGrayscale(0.3),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomGrayscale(0.3),
>>> py_transforms.ToTensor()])
"""
@check_prob
@ -878,10 +888,13 @@ class Pad:
value of edge.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> # adds 10 pixels (default black) to each side of the border of the image
>>> py_transforms.Pad(padding=10),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> # adds 10 pixels (default black) to each side of the border of the image
>>> py_transforms.Pad(padding=10),
>>> py_transforms.ToTensor()])
"""
@check_pad
@ -922,9 +935,12 @@ class RandomPerspective:
- Inter.BICUBIC, means interpolation method is bicubic interpolation.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomPerspective(prob=0.1),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomPerspective(prob=0.1),
>>> py_transforms.ToTensor()])
"""
@check_random_perspective
@ -972,9 +988,12 @@ class RandomErasing:
erase_area (default=10). If exceeded, return the original image.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.ToTensor(),
>>> py_transforms.RandomErasing(value='random')])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.ToTensor(),
>>> py_transforms.RandomErasing(value='random')])
"""
@check_random_erasing
@ -1016,9 +1035,12 @@ class Cutout:
num_patches (int, optional): Number of patches to be cut out of an image (default=1).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.ToTensor(),
>>> py_transforms.Cutout(80)])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.ToTensor(),
>>> py_transforms.Cutout(80)])
"""
@check_cutout
@ -1043,7 +1065,8 @@ class Cutout:
bounded = False
for _ in range(self.num_patches):
i, j, erase_h, erase_w, erase_value = util.get_erase_params(np_img, (scale, scale), (1, 1), 0, bounded, 1)
i, j, erase_h, erase_w, erase_value = util.get_erase_params(np_img, (scale, scale), (1, 1), 0, bounded,
1)
np_img = util.erase(np_img, i, j, erase_h, erase_w, erase_value)
return np_img
@ -1061,10 +1084,13 @@ class LinearTransformation:
mean_vector (numpy.ndarray): a NumPy ndarray of shape (D,) where D = C x H x W.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.Resize(256),
>>> py_transforms.ToTensor(),
>>> py_transforms.LinearTransformation(transformation_matrix, mean_vector)])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.Resize(256),
>>> py_transforms.ToTensor(),
>>> py_transforms.LinearTransformation(transformation_matrix, mean_vector)])
"""
@check_linear_transform
@ -1133,9 +1159,12 @@ class RandomAffine:
TypeError: If fill_value is not a single integer or a 3-tuple.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomAffine(degrees=15, translate=(0.1, 0.1), scale=(0.9, 1.1)),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomAffine(degrees=15, translate=(0.1, 0.1), scale=(0.9, 1.1)),
>>> py_transforms.ToTensor()])
"""
@check_random_affine
@ -1278,9 +1307,12 @@ class RandomColor:
It should be in (min, max) format (default=(0.1,1.9)).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomColor((0.5,1.5)),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomColor((0.5,1.5)),
>>> py_transforms.ToTensor()])
"""
@check_positive_degrees
@ -1310,9 +1342,12 @@ class RandomSharpness:
It should be in (min, max) format (default=(0.1,1.9)).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomSharpness((0.5,1.5)),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomSharpness((0.5,1.5)),
>>> py_transforms.ToTensor()])
"""
@ -1343,9 +1378,12 @@ class AutoContrast:
ignore (Union[int, sequence], optional): Pixel values to ignore (default=None).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.AutoContrast(),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.AutoContrast(),
>>> py_transforms.ToTensor()])
"""
@ -1373,9 +1411,12 @@ class Invert:
Invert colors of input PIL image.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.Invert(),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.Invert(),
>>> py_transforms.ToTensor()])
"""
@ -1398,9 +1439,12 @@ class Equalize:
Equalize the histogram of input PIL image.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.Equalize(),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.Equalize(),
>>> py_transforms.ToTensor()])
"""
@ -1430,13 +1474,16 @@ class UniformAugment:
num_ops (int, optional): number of transforms to sequentially apply (default=2).
Examples:
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> transforms_list = [py_transforms.CenterCrop(64),
>>> py_transforms.RandomColor(),
>>> py_transforms.RandomSharpness(),
>>> py_transforms.RandomRotation(30)]
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.UniformAugment(transforms_list),
>>> py_transforms.ToTensor()])
>>> Compose([py_transforms.Decode(),
>>> py_transforms.UniformAugment(transforms_list),
>>> py_transforms.ToTensor()])
"""
@check_uniform_augment_py

60
mindspore/dataset/transforms/vision/py_transforms_util.py → mindspore/dataset/vision/py_transforms_util.py
View File

@ -24,6 +24,7 @@ import numpy as np
from PIL import Image, ImageOps, ImageEnhance, __version__
from .utils import Inter
from ..core.py_util_helpers import is_numpy
augment_error_message = 'img should be PIL image. Got {}. Use Decode() for encoded data or ToPIL() for decoded data.'
@ -41,39 +42,6 @@ def is_pil(img):
return isinstance(img, Image.Image)
def is_numpy(img):
"""
Check if the input image is NumPy format.
Args:
img: Image to be checked.
Returns:
Bool, True if input is NumPy image.
"""
return isinstance(img, np.ndarray)
def compose(img, transforms):
"""
Compose a list of transforms and apply on the image.
Args:
img (numpy.ndarray): An image in NumPy ndarray.
transforms (list): A list of transform Class objects to be composed.
Returns:
img (numpy.ndarray), An augmented image in NumPy ndarray.
"""
if is_numpy(img):
for transform in transforms:
img = transform(img)
if is_numpy(img):
return img
raise TypeError('img should be NumPy ndarray. Got {}. Append ToTensor() to transforms'.format(type(img)))
raise TypeError('img should be NumPy ndarray. Got {}.'.format(type(img)))
def normalize(img, mean, std):
"""
Normalize the image between [0, 1] with respect to mean and standard deviation.
@ -1221,32 +1189,6 @@ def random_affine(img, angle, translations, scale, shear, resample, fill_value=0
return img.transform(output_size, Image.AFFINE, matrix, resample, **kwargs)
def one_hot_encoding(label, num_classes, epsilon):
"""
Apply label smoothing transformation to the input label, and make label be more smoothing and continuous.
Args:
label (numpy.ndarray): label to be applied label smoothing.
num_classes (int): Num class of object in dataset, value should over 0.
epsilon (float): The adjustable Hyper parameter. Default is 0.0.
Returns:
img (numpy.ndarray), label after being one hot encoded and done label smoothed.
"""
if label > num_classes:
raise ValueError('the num_classes is smaller than the category number.')
num_elements = label.size
one_hot_label = np.zeros((num_elements, num_classes), dtype=int)
if isinstance(label, list) is False:
label = [label]
for index in range(num_elements):
one_hot_label[index, label[index]] = 1
return (1 - epsilon) * one_hot_label + epsilon / num_classes
def mix_up_single(batch_size, img, label, alpha=0.2):
"""
Apply mix up transformation to image and label in single batch internal, One hot encoding should done before this.

0
mindspore/dataset/transforms/vision/utils.py → mindspore/dataset/vision/utils.py
View File

19
mindspore/dataset/transforms/vision/validators.py → mindspore/dataset/vision/validators.py
View File

@ -19,10 +19,10 @@ from functools import wraps
import numpy as np
from mindspore._c_dataengine import TensorOp
from .utils import Inter, Border, ImageBatchFormat
from ...core.validator_helpers import check_value, check_uint8, FLOAT_MAX_INTEGER, check_pos_float32, \
from mindspore.dataset.core.validator_helpers import check_value, check_uint8, FLOAT_MAX_INTEGER, check_pos_float32, \
check_2tuple, check_range, check_positive, INT32_MAX, parse_user_args, type_check, type_check_list, \
check_tensor_op, UINT8_MAX, check_value_normalize_std
from .utils import Inter, Border, ImageBatchFormat
def check_crop_size(size):
@ -678,21 +678,6 @@ def check_positive_degrees(method):
return new_method
def check_compose_list(method):
"""Wrapper method to check the transform list of ComposeOp."""
@wraps(method)
def new_method(self, *args, **kwargs):
[transforms], _ = parse_user_args(method, *args, **kwargs)
type_check(transforms, (list,), transforms)
if not transforms:
raise ValueError("transforms list is empty.")
return method(self, *args, **kwargs)
return new_method
def check_random_select_subpolicy_op(method):
"""Wrapper method to check the parameters of RandomSelectSubpolicyOp."""

4
mindspore/train/callback/_summary_collector.py
View File

@ -727,7 +727,7 @@ class SummaryCollector(Callback):
Get dataset path of MindDataset object.
Args:
output_dataset (Union[Dataset, ImageFolderDatasetV2, MnistDataset, Cifar10Dataset, Cifar100Dataset,
output_dataset (Union[Dataset, ImageFolderDataset, MnistDataset, Cifar10Dataset, Cifar100Dataset,
VOCDataset, CelebADataset, MindDataset, ManifestDataset, TFRecordDataset, TextFileDataset]):
Refer to mindspore.dataset.Dataset.
@ -738,7 +738,7 @@ class SummaryCollector(Callback):
IndexError: it means get dataset path failed.
"""
dataset_package = import_module('mindspore.dataset')
dataset_dir_set = (dataset_package.ImageFolderDatasetV2, dataset_package.MnistDataset,
dataset_dir_set = (dataset_package.ImageFolderDataset, dataset_package.MnistDataset,
dataset_package.Cifar10Dataset, dataset_package.Cifar100Dataset,
dataset_package.VOCDataset, dataset_package.CelebADataset)
dataset_file_set = (dataset_package.MindDataset, dataset_package.ManifestDataset)

4
model_zoo/official/cv/faster_rcnn/src/dataset.py
View File

@ -449,7 +449,7 @@ def create_fasterrcnn_dataset(mindrecord_file, batch_size=2, repeat_num=12, devi
if is_training:
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "image_shape", "box", "label", "valid_num"],
columns_order=["image", "image_shape", "box", "label", "valid_num"],
column_order=["image", "image_shape", "box", "label", "valid_num"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
flip = (np.random.rand() < config.flip_ratio)
@ -467,7 +467,7 @@ def create_fasterrcnn_dataset(mindrecord_file, batch_size=2, repeat_num=12, devi
else:
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "image_shape", "box", "label", "valid_num"],
columns_order=["image", "image_shape", "box", "label", "valid_num"],
column_order=["image", "image_shape", "box", "label", "valid_num"],
operations=compose_map_func,
num_parallel_workers=num_parallel_workers)

6
model_zoo/official/cv/inceptionv3/src/dataset.py
View File

@ -37,10 +37,10 @@ def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1):
dataset
"""
if group_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
# define map operations
if do_train:
trans = [

4
model_zoo/official/cv/maskrcnn/src/dataset.py
View File

@ -505,7 +505,7 @@ def create_maskrcnn_dataset(mindrecord_file, batch_size=2, device_num=1, rank_id
if is_training:
ds = ds.map(input_columns=["image", "annotation", "mask", "mask_shape"],
output_columns=["image", "image_shape", "box", "label", "valid_num", "mask"],
columns_order=["image", "image_shape", "box", "label", "valid_num", "mask"],
column_order=["image", "image_shape", "box", "label", "valid_num", "mask"],
operations=compose_map_func,
python_multiprocessing=False,
num_parallel_workers=num_parallel_workers)
@ -514,7 +514,7 @@ def create_maskrcnn_dataset(mindrecord_file, batch_size=2, device_num=1, rank_id
else:
ds = ds.map(input_columns=["image", "annotation", "mask", "mask_shape"],
output_columns=["image", "image_shape", "box", "label", "valid_num", "mask"],
columns_order=["image", "image_shape", "box", "label", "valid_num", "mask"],
column_order=["image", "image_shape", "box", "label", "valid_num", "mask"],
operations=compose_map_func,
num_parallel_workers=num_parallel_workers)
ds = ds.batch(batch_size, drop_remainder=True)

24
model_zoo/official/cv/mobilenetv2/src/dataset.py
View File

@ -26,6 +26,7 @@ import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.c_transforms as C2
def create_dataset(dataset_path, do_train, config, repeat_num=1):
"""
create a train or eval dataset
@ -44,20 +45,19 @@ def create_dataset(dataset_path, do_train, config, repeat_num=1):
rank_size = int(os.getenv("RANK_SIZE", '1'))
rank_id = int(os.getenv("RANK_ID", '0'))
if rank_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=rank_size, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=rank_size, shard_id=rank_id)
elif config.platform == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
elif config.platform == "CPU":
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
resize_height = config.image_height
resize_width = config.image_width
@ -71,7 +71,8 @@ def create_dataset(dataset_path, do_train, config, repeat_num=1):
resize_op = C.Resize((256, 256))
center_crop = C.CenterCrop(resize_width)
rescale_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
normalize_op = C.Normalize(mean=[0.485*255, 0.456*255, 0.406*255], std=[0.229*255, 0.224*255, 0.225*255])
normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
change_swap_op = C.HWC2CHW()
if do_train:
@ -95,6 +96,7 @@ def create_dataset(dataset_path, do_train, config, repeat_num=1):
return ds
def extract_features(net, dataset_path, config):
features_folder = dataset_path + '_features'
if not os.path.exists(features_folder):
@ -110,13 +112,13 @@ def extract_features(net, dataset_path, config):
for data in pbar:
features_path = os.path.join(features_folder, f"feature_{i}.npy")
label_path = os.path.join(features_folder, f"label_{i}.npy")
if not(os.path.exists(features_path) and os.path.exists(label_path)):
if not (os.path.exists(features_path) and os.path.exists(label_path)):
image = data["image"]
label = data["label"]
features = model.predict(Tensor(image))
np.save(features_path, features.asnumpy())
np.save(label_path, label)
pbar.set_description("Process dataset batch: %d"%(i+1))
pbar.set_description("Process dataset batch: %d" % (i + 1))
i += 1
return step_size

23
model_zoo/official/cv/mobilenetv2_quant/src/dataset.py
View File

@ -21,7 +21,8 @@ import mindspore.common.dtype as mstype
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.c_transforms as C2
import mindspore.dataset.transforms.vision.py_transforms as P
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as P
def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1, batch_size=32):
@ -44,7 +45,7 @@ def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1,
if config.data_load_mode == "mindrecord":
load_func = partial(de.MindDataset, dataset_path, columns_list)
else:
load_func = partial(de.ImageFolderDatasetV2, dataset_path)
load_func = partial(de.ImageFolderDataset, dataset_path)
if do_train:
if rank_size == 1:
ds = load_func(num_parallel_workers=8, shuffle=True)
@ -56,10 +57,10 @@ def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1,
elif device_target == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
raise ValueError("Unsupported device_target.")
@ -118,12 +119,12 @@ def create_dataset_py(dataset_path, do_train, config, device_target, repeat_num=
rank_id = int(os.getenv("RANK_ID"))
if do_train:
if rank_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=rank_size, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=rank_size, shard_id=rank_id)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=False)
else:
raise ValueError("Unsupported device target.")
@ -149,9 +150,9 @@ def create_dataset_py(dataset_path, do_train, config, device_target, repeat_num=
else:
trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op]
compose = P.ComposeOp(trans)
compose = mindspore.dataset.transforms.py_transforms.Compose(trans)
ds = ds.map(input_columns="image", operations=compose(), num_parallel_workers=8, python_multiprocessing=True)
ds = ds.map(input_columns="image", operations=compose, num_parallel_workers=8, python_multiprocessing=True)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)

6
model_zoo/official/cv/mobilenetv3/src/dataset.py
View File

@ -37,10 +37,10 @@ def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1,
if device_target == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
raise ValueError("Unsupported device_target.")

16
model_zoo/official/cv/nasnet/src/dataset.py
View File

@ -37,24 +37,24 @@ def create_dataset(dataset_path, config, do_train, repeat_num=1):
rank = config.rank
group_size = config.group_size
if group_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=config.work_nums, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=config.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(config.image_size),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, saturation=0.5) # fast mode
#C.RandomColorAdjust(brightness=0.4, contrast=0.5, saturation=0.5, hue=0.2)
]
C.RandomColorAdjust(brightness=0.4, saturation=0.5) # fast mode
# C.RandomColorAdjust(brightness=0.4, contrast=0.5, saturation=0.5, hue=0.2)
]
else:
trans = [
C.Decode(),
C.Resize(int(config.image_size/0.875)),
C.Resize(int(config.image_size / 0.875)),
C.CenterCrop(config.image_size)
]
]
trans += [
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),

18
model_zoo/official/cv/resnet/src/dataset.py
View File

@ -98,10 +98,10 @@ def create_dataset2(dataset_path, do_train, repeat_num=1, batch_size=32, target=
device_num = get_group_size()
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
@ -153,10 +153,10 @@ def create_dataset3(dataset_path, do_train, repeat_num=1, batch_size=32, target=
device_num, rank_id = _get_rank_info()
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.475 * 255, 0.451 * 255, 0.392 * 255]
std = [0.275 * 255, 0.267 * 255, 0.278 * 255]
@ -207,10 +207,10 @@ def create_dataset4(dataset_path, do_train, repeat_num=1, batch_size=32, target=
if target == "Ascend":
device_num, rank_id = _get_rank_info()
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=12, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=12, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [123.68, 116.78, 103.94]
std = [1.0, 1.0, 1.0]

17
model_zoo/official/cv/resnet50_quant/src/dataset.py
View File

@ -21,7 +21,8 @@ import mindspore.common.dtype as mstype
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.c_transforms as C2
import mindspore.dataset.transforms.vision.py_transforms as P
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as P
from mindspore.communication.management import init, get_rank, get_group_size
from src.config import config_quant
@ -54,7 +55,7 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="
if config.data_load_mode == "mindrecord":
load_func = partial(de.MindDataset, dataset_path, columns_list)
else:
load_func = partial(de.ImageFolderDatasetV2, dataset_path)
load_func = partial(de.ImageFolderDataset, dataset_path)
if device_num == 1:
ds = load_func(num_parallel_workers=8, shuffle=True)
else:
@ -120,12 +121,12 @@ def create_dataset_py(dataset_path, do_train, repeat_num=1, batch_size=32, targe
if do_train:
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=False)
image_size = 224
@ -145,8 +146,8 @@ def create_dataset_py(dataset_path, do_train, repeat_num=1, batch_size=32, targe
else:
trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op]
compose = P.ComposeOp(trans)
ds = ds.map(input_columns="image", operations=compose(), num_parallel_workers=8, python_multiprocessing=True)
compose = mindspore.dataset.transforms.py_transforms.Compose(trans)
ds = ds.map(input_columns="image", operations=compose, num_parallel_workers=8, python_multiprocessing=True)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)

7
model_zoo/official/cv/resnet_thor/src/dataset.py
View File

@ -47,10 +47,10 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="
num_parallels = 4
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=num_parallels, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=num_parallels, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=num_parallels, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=num_parallels, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
@ -86,6 +86,7 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="
return ds
def _get_rank_info():
"""
get rank size and rank id

6
model_zoo/official/cv/resnext50/src/dataset.py
View File

@ -134,9 +134,9 @@ def classification_dataset(data_dir, image_size, per_batch_size, max_epoch, rank
transform_label = target_transform
if input_mode == 'folder':
de_dataset = de.ImageFolderDatasetV2(data_dir, num_parallel_workers=num_parallel_workers,
shuffle=shuffle, sampler=sampler, class_indexing=class_indexing,
num_shards=group_size, shard_id=rank)
de_dataset = de.ImageFolderDataset(data_dir, num_parallel_workers=num_parallel_workers,
shuffle=shuffle, sampler=sampler, class_indexing=class_indexing,
num_shards=group_size, shard_id=rank)
else:
dataset = TxtDataset(root, data_dir)
sampler = DistributedSampler(dataset, rank, group_size, shuffle=shuffle)

11
model_zoo/official/cv/shufflenetv2/src/dataset.py
View File

@ -30,6 +30,7 @@ class toBGR():
img = np.ascontiguousarray(img)
return img
def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1):
"""
create a train or eval dataset
@ -45,23 +46,23 @@ def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1):
dataset
"""
if group_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(224),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
]
]
else:
trans = [
C.Decode(),
C.Resize(256),
C.CenterCrop(224)
]
]
trans += [
toBGR(),
C.Rescale(1.0 / 255.0, 0.0),

2
model_zoo/official/cv/ssd/src/dataset.py
View File

@ -403,7 +403,7 @@ def create_ssd_dataset(mindrecord_file, batch_size=32, repeat_num=10, device_num
output_columns = ["img_id", "image", "image_shape"]
trans = [normalize_op, change_swap_op]
ds = ds.map(input_columns=["img_id", "image", "annotation"],
output_columns=output_columns, columns_order=output_columns,
output_columns=output_columns, column_order=output_columns,
operations=compose_map_func, python_multiprocessing=is_training,
num_parallel_workers=num_parallel_workers)
ds = ds.map(input_columns=["image"], operations=trans, python_multiprocessing=is_training,

6
model_zoo/official/cv/vgg16/src/dataset.py
View File

@ -149,9 +149,9 @@ def classification_dataset(data_dir, image_size, per_batch_size, rank=0, group_s
transform_label = target_transform
if input_mode == 'folder':
de_dataset = de.ImageFolderDatasetV2(data_dir, num_parallel_workers=num_parallel_workers,
shuffle=shuffle, sampler=sampler, class_indexing=class_indexing,
num_shards=group_size, shard_id=rank)
de_dataset = de.ImageFolderDataset(data_dir, num_parallel_workers=num_parallel_workers,
shuffle=shuffle, sampler=sampler, class_indexing=class_indexing,
num_shards=group_size, shard_id=rank)
else:
dataset = TxtDataset(root, data_dir)
sampler = DistributedSampler(dataset, rank, group_size, shuffle=shuffle)

2
model_zoo/official/cv/yolov3_darknet53/src/yolo_dataset.py
View File

@ -178,7 +178,7 @@ def create_yolo_dataset(image_dir, anno_path, batch_size, max_epoch, device_num,
compose_map_func = (lambda image, img_id: reshape_fn(image, img_id, config))
ds = ds.map(input_columns=["image", "img_id"],
output_columns=["image", "image_shape", "img_id"],
columns_order=["image", "image_shape", "img_id"],
column_order=["image", "image_shape", "img_id"],
operations=compose_map_func, num_parallel_workers=8)
ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=8)
ds = ds.batch(batch_size, drop_remainder=True)

2
model_zoo/official/cv/yolov3_darknet53_quant/src/yolo_dataset.py
View File

@ -175,7 +175,7 @@ def create_yolo_dataset(image_dir, anno_path, batch_size, max_epoch, device_num,
compose_map_func = (lambda image, img_id: reshape_fn(image, img_id, config))
ds = ds.map(input_columns=["image", "img_id"],
output_columns=["image", "image_shape", "img_id"],
columns_order=["image", "image_shape", "img_id"],
column_order=["image", "image_shape", "img_id"],
operations=compose_map_func, num_parallel_workers=8)
ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=8)
ds = ds.batch(batch_size, drop_remainder=True)

4
model_zoo/official/cv/yolov3_resnet18/src/dataset.py
View File

@ -303,7 +303,7 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=1, device_num=
hwc_to_chw = C.HWC2CHW()
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
columns_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
column_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=num_parallel_workers)
ds = ds.batch(batch_size, drop_remainder=True)
@ -311,6 +311,6 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=1, device_num=
else:
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "image_shape", "annotation"],
columns_order=["image", "image_shape", "annotation"],
column_order=["image", "image_shape", "annotation"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
return ds

18
model_zoo/official/nlp/bert/src/clue_classification_dataset_process.py
View File

@ -43,7 +43,7 @@ def process_tnews_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage
### Processing label
if data_usage == 'test':
dataset = dataset.map(input_columns=["id"], output_columns=["id", "label_id"],
columns_order=["id", "label_id", "sentence"], operations=ops.Duplicate())
column_order=["id", "label_id", "sentence"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["label_id"], operations=ops.Fill(0))
else:
label_vocab = text.Vocab.from_list(label_list)
@ -61,10 +61,10 @@ def process_tnews_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage
dataset = dataset.map(input_columns=["sentence"], output_columns=["text_ids"], operations=lookup)
dataset = dataset.map(input_columns=["text_ids"], operations=ops.PadEnd([max_seq_len], 0))
dataset = dataset.map(input_columns=["text_ids"], output_columns=["text_ids", "mask_ids"],
columns_order=["text_ids", "mask_ids", "label_id"], operations=ops.Duplicate())
column_order=["text_ids", "mask_ids", "label_id"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["mask_ids"], operations=ops.Mask(ops.Relational.NE, 0, mstype.int32))
dataset = dataset.map(input_columns=["text_ids"], output_columns=["text_ids", "segment_ids"],
columns_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate())
column_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["segment_ids"], operations=ops.Fill(0))
dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)
return dataset
@ -87,7 +87,7 @@ def process_cmnli_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage
### Processing label
if data_usage == 'test':
dataset = dataset.map(input_columns=["id"], output_columns=["id", "label_id"],
columns_order=["id", "label_id", "sentence1", "sentence2"], operations=ops.Duplicate())
column_order=["id", "label_id", "sentence1", "sentence2"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["label_id"], operations=ops.Fill(0))
else:
label_vocab = text.Vocab.from_list(label_list)
@ -110,26 +110,26 @@ def process_cmnli_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage
operations=ops.Concatenate(append=np.array(["[SEP]"], dtype='S')))
### Generating segment_ids
dataset = dataset.map(input_columns=["sentence1"], output_columns=["sentence1", "type_sentence1"],
columns_order=["sentence1", "type_sentence1", "sentence2", "label_id"],
column_order=["sentence1", "type_sentence1", "sentence2", "label_id"],
operations=ops.Duplicate())
dataset = dataset.map(input_columns=["sentence2"], output_columns=["sentence2", "type_sentence2"],
columns_order=["sentence1", "type_sentence1", "sentence2", "type_sentence2", "label_id"],
column_order=["sentence1", "type_sentence1", "sentence2", "type_sentence2", "label_id"],
operations=ops.Duplicate())
dataset = dataset.map(input_columns=["type_sentence1"], operations=[lookup, ops.Fill(0)])
dataset = dataset.map(input_columns=["type_sentence2"], operations=[lookup, ops.Fill(1)])
dataset = dataset.map(input_columns=["type_sentence1", "type_sentence2"], output_columns=["segment_ids"],
columns_order=["sentence1", "sentence2", "segment_ids", "label_id"],
column_order=["sentence1", "sentence2", "segment_ids", "label_id"],
operations=ops.Concatenate())
dataset = dataset.map(input_columns=["segment_ids"], operations=ops.PadEnd([max_seq_len], 0))
### Generating text_ids
dataset = dataset.map(input_columns=["sentence1", "sentence2"], output_columns=["text_ids"],
columns_order=["text_ids", "segment_ids", "label_id"],
column_order=["text_ids", "segment_ids", "label_id"],
operations=ops.Concatenate())
dataset = dataset.map(input_columns=["text_ids"], operations=lookup)
dataset = dataset.map(input_columns=["text_ids"], operations=ops.PadEnd([max_seq_len], 0))
### Generating mask_ids
dataset = dataset.map(input_columns=["text_ids"], output_columns=["text_ids", "mask_ids"],
columns_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate())
column_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["mask_ids"], operations=ops.Mask(ops.Relational.NE, 0, mstype.int32))
dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)
return dataset

4
model_zoo/official/recommend/deepfm/src/dataset.py
View File

@ -213,7 +213,7 @@ def _get_mindrecord_dataset(directory, train_mode=True, epochs=1, batch_size=100
np.array(y).flatten().reshape(batch_size, 39),
np.array(z).flatten().reshape(batch_size, 1))),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'],
column_order=['feat_ids', 'feat_vals', 'label'],
num_parallel_workers=8)
ds = ds.repeat(epochs)
return ds
@ -261,7 +261,7 @@ def _get_tf_dataset(directory, train_mode=True, epochs=1, batch_size=1000,
np.array(y).flatten().reshape(batch_size, 39),
np.array(z).flatten().reshape(batch_size, 1))),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'],
column_order=['feat_ids', 'feat_vals', 'label'],
num_parallel_workers=8)
ds = ds.repeat(epochs)
return ds

4
model_zoo/official/recommend/wide_and_deep/src/datasets.py
View File

@ -230,7 +230,7 @@ def _get_tf_dataset(data_dir, train_mode=True, epochs=1, batch_size=1000,
ds = ds.map(operations=_padding_func(batch_size, manual_shape, target_column),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8)
column_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8)
# if train_mode:
ds = ds.repeat(epochs)
return ds
@ -270,7 +270,7 @@ def _get_mindrecord_dataset(directory, train_mode=True, epochs=1, batch_size=100
ds = ds.batch(int(batch_size / line_per_sample), drop_remainder=True)
ds = ds.map(_padding_func(batch_size, manual_shape, target_column),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'],
column_order=['feat_ids', 'feat_vals', 'label'],
num_parallel_workers=8)
ds = ds.repeat(epochs)
return ds

2
model_zoo/official/recommend/wide_and_deep_multitable/src/datasets.py
View File

@ -263,7 +263,7 @@ def _get_tf_dataset(data_dir,
'multi_doc_ad_topic_id_mask', 'ad_id', 'display_ad_and_is_leak',
'display_id', 'is_leak'
],
columns_order=[
column_order=[
'label', 'continue_val', 'indicator_id', 'emb_128_id',
'emb_64_single_id', 'multi_doc_ad_category_id',
'multi_doc_ad_category_id_mask', 'multi_doc_event_entity_id',

2
tests/st/mem_reuse/resnet_cifar_memreuse.py
View File

@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype
import mindspore.context as context
import mindspore.dataset as de
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
from mindspore import Tensor
from mindspore.communication.management import init

2
tests/st/mem_reuse/resnet_cifar_normal.py
View File

@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype
import mindspore.context as context
import mindspore.dataset as de
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
from mindspore import Tensor
from mindspore.communication.management import init

4
tests/st/model_zoo_tests/wide_and_deep/python_file_for_ci/datasets.py
View File

@ -57,7 +57,7 @@ def _get_tf_dataset(data_dir, train_mode=True, epochs=1, batch_size=1000,
np.array(y).flatten().reshape(batch_size, 39),
np.array(z).flatten().reshape(batch_size, 1))),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8)
column_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8)
#if train_mode:
ds = ds.repeat(epochs)
return ds
@ -97,7 +97,7 @@ def _get_mindrecord_dataset(directory, train_mode=True, epochs=1, batch_size=100
np.array(y).flatten().reshape(batch_size, 39),
np.array(z).flatten().reshape(batch_size, 1))),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'],
column_order=['feat_ids', 'feat_vals', 'label'],
num_parallel_workers=8)
ds = ds.repeat(epochs)
return ds

6
tests/st/model_zoo_tests/yolov3/src/dataset.py
View File

@ -22,7 +22,7 @@ from matplotlib.colors import rgb_to_hsv, hsv_to_rgb
from PIL import Image
import mindspore.dataset as de
from mindspore.mindrecord import FileWriter
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.vision.c_transforms as C
from src.config import ConfigYOLOV3ResNet18
iter_cnt = 0
@ -305,7 +305,7 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=10, device_num
hwc_to_chw = C.HWC2CHW()
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
columns_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
column_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=num_parallel_workers)
ds = ds.batch(batch_size, drop_remainder=True)
@ -313,6 +313,6 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=10, device_num
else:
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "image_shape", "annotation"],
columns_order=["image", "image_shape", "annotation"],
column_order=["image", "image_shape", "annotation"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
return ds

2
tests/st/networks/models/deeplabv3/src/md_dataset.py
View File

@ -15,7 +15,7 @@
"""Dataset module."""
from PIL import Image
import mindspore.dataset as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.vision.c_transforms as C
import numpy as np
from .ei_dataset import HwVocRawDataset

8
tests/st/networks/models/resnet50/src/dataset.py
View File

@ -18,7 +18,7 @@
import os
import mindspore.common.dtype as mstype
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.vision.c_transforms as C
import mindspore.dataset.transforms.c_transforms as C2
@ -39,10 +39,10 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
device_num = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]

8
tests/st/networks/models/resnet50/src_thor/dataset.py
View File

@ -21,7 +21,7 @@ import mindspore.common.dtype as mstype
import mindspore.dataset as dataset
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.c_transforms as C2
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.vision.c_transforms as C
dataset.config.set_seed(1)
@ -43,10 +43,10 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
device_num = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]

4
tests/st/networks/test_gpu_lenet.py
View File

@ -21,11 +21,11 @@ import pytest
import mindspore.context as context
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import Tensor
from mindspore.common import dtype as mstype
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.nn import Dense, TrainOneStepCell, WithLossCell
from mindspore.nn.metrics import Accuracy
from mindspore.nn.optim import Momentum

6
tests/st/ops/ascend/test_tdt_data_ms.py
View File

@ -17,11 +17,11 @@ import numpy as np
import mindspore.context as context
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
from mindspore.common.api import _executor
from mindspore.common.tensor import Tensor
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.ops import operations as P
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
@ -83,8 +83,6 @@ if __name__ == '__main__':
class dataiter(nn.Cell):
def __init__(self):
super(dataiter, self).__init__()
def construct(self):
input_, _ = get_next()

4
tests/st/probability/dataset.py
View File

@ -17,9 +17,9 @@ Produce the dataset
"""
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.dataset.transforms.c_transforms as C
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.common import dtype as mstype

2
tests/st/probability/test_gpu_svi_cvae.py
View File

@ -16,7 +16,7 @@ import os
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import context, Tensor
from mindspore.ops import operations as P

2
tests/st/probability/test_gpu_svi_vae.py
View File

@ -16,7 +16,7 @@ import os
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import context, Tensor
from mindspore.ops import operations as P

2
tests/st/probability/test_gpu_vae_gan.py
View File

@ -18,7 +18,7 @@ The VAE interface can be called to construct VAE-GAN network.
import os
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import context
from mindspore.ops import operations as P

4
tests/st/probability/test_uncertainty.py
View File

@ -15,12 +15,12 @@
""" test uncertainty toolbox """
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import context, Tensor
from mindspore.common import dtype as mstype
from mindspore.common.initializer import TruncatedNormal
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.nn.probability.toolbox.uncertainty_evaluation import UncertaintyEvaluation
from mindspore.train.serialization import load_checkpoint, load_param_into_net

4
tests/st/ps/full_ps/test_full_ps_lenet.py
View File

@ -19,10 +19,10 @@ import argparse
import mindspore.context as context
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore.common import dtype as mstype
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.nn.metrics import Accuracy
from mindspore.train import Model
from mindspore.train.callback import LossMonitor

2
tests/st/pynative/test_pynative_resnet50.py
View File

@ -21,7 +21,7 @@ import pytest
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
import mindspore.ops.functional as F

4
tests/st/quantization/lenet_quant/dataset.py
View File

@ -17,9 +17,9 @@ Produce the dataset
"""
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.dataset.transforms.c_transforms as C
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.common import dtype as mstype

4
tests/st/summary/test_summary.py
View File

@ -25,8 +25,8 @@ from mindspore import nn, Tensor, context
from mindspore.nn.metrics import Accuracy
from mindspore.nn.optim import Momentum
from mindspore.dataset.transforms import c_transforms as C
from mindspore.dataset.transforms.vision import c_transforms as CV
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import c_transforms as CV
from mindspore.dataset.vision import Inter
from mindspore.common import dtype as mstype
from mindspore.common.initializer import TruncatedNormal
from mindspore.ops import operations as P

2
tests/st/tbe_networks/resnet_cifar.py
View File

@ -24,7 +24,7 @@ from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMoni
from mindspore.train.serialization import load_checkpoint, load_param_into_net
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
from mindspore import Tensor
from mindspore import context

2
tests/st/tbe_networks/test_resnet_cifar_1p.py
View File

@ -21,7 +21,7 @@ from resnet import resnet50
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
import mindspore.ops.functional as F
from mindspore import Tensor

2
tests/st/tbe_networks/test_resnet_cifar_8p.py
View File

@ -22,7 +22,7 @@ from resnet import resnet50
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
import mindspore.ops.functional as F
from mindspore import Tensor

9
tests/ut/python/dataset/test_HWC2CHW.py
View File

@ -17,8 +17,9 @@ Testing HWC2CHW op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import diff_mse, visualize_list, save_and_check_md5
@ -99,8 +100,8 @@ def test_HWC2CHW_comp(plot=False):
py_vision.ToTensor(),
py_vision.HWC2CHW()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
image_c_transposed = []
image_py_transposed = []

6
tests/ut/python/dataset/test_apply.py
View File

@ -15,7 +15,7 @@
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
DATA_DIR = "../data/dataset/testPK/data"
@ -46,8 +46,8 @@ def test_apply_generator_case():
def test_apply_imagefolder_case():
# apply dataset map operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3)
data2 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3)
data1 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=3)
data2 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=3)
decode_op = vision.Decode()
normalize_op = vision.Normalize([121.0, 115.0, 100.0], [70.0, 68.0, 71.0])

99
tests/ut/python/dataset/test_autocontrast.py
View File

@ -17,8 +17,9 @@ Testing AutoContrast op in DE
"""
import numpy as np
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as F
import mindspore.dataset.vision.c_transforms as C
from mindspore import log as logger
from util import visualize_list, visualize_one_channel_dataset, diff_mse, save_and_check_md5
@ -35,14 +36,14 @@ def test_auto_contrast_py(plot=False):
logger.info("Test AutoContrast Python Op")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
ds_original = ds.map(input_columns="image",
operations=transforms_original())
operations=transforms_original)
ds_original = ds_original.batch(512)
@ -55,15 +56,16 @@ def test_auto_contrast_py(plot=False):
axis=0)
# AutoContrast Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_auto_contrast = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=10.0, ignore=[10, 20]),
F.ToTensor()])
transforms_auto_contrast = \
mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=10.0, ignore=[10, 20]),
F.ToTensor()])
ds_auto_contrast = ds.map(input_columns="image",
operations=transforms_auto_contrast())
operations=transforms_auto_contrast)
ds_auto_contrast = ds_auto_contrast.batch(512)
@ -96,15 +98,15 @@ def test_auto_contrast_c(plot=False):
logger.info("Test AutoContrast C Op")
# AutoContrast Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224))])
python_op = F.AutoContrast(cutoff=10.0, ignore=[10, 20])
c_op = C.AutoContrast(cutoff=10.0, ignore=[10, 20])
transforms_op = F.ComposeOp([lambda img: F.ToPIL()(img.astype(np.uint8)),
python_op,
np.array])()
transforms_op = mindspore.dataset.transforms.py_transforms.Compose([lambda img: F.ToPIL()(img.astype(np.uint8)),
python_op,
np.array])
ds_auto_contrast_py = ds.map(input_columns="image",
operations=transforms_op)
@ -119,7 +121,7 @@ def test_auto_contrast_c(plot=False):
image,
axis=0)
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224))])
@ -159,17 +161,18 @@ def test_auto_contrast_one_channel_c(plot=False):
logger.info("Test AutoContrast C Op With One Channel Images")
# AutoContrast Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224))])
python_op = F.AutoContrast()
c_op = C.AutoContrast()
# not using F.ToTensor() since it converts to floats
transforms_op = F.ComposeOp([lambda img: (np.array(img)[:, :, 0]).astype(np.uint8),
F.ToPIL(),
python_op,
np.array])()
transforms_op = mindspore.dataset.transforms.py_transforms.Compose(
[lambda img: (np.array(img)[:, :, 0]).astype(np.uint8),
F.ToPIL(),
python_op,
np.array])
ds_auto_contrast_py = ds.map(input_columns="image",
operations=transforms_op)
@ -184,7 +187,7 @@ def test_auto_contrast_one_channel_c(plot=False):
image,
axis=0)
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
@ -248,7 +251,7 @@ def test_auto_contrast_invalid_ignore_param_c():
"""
logger.info("Test AutoContrast C Op with invalid ignore parameter")
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
@ -260,7 +263,7 @@ def test_auto_contrast_invalid_ignore_param_c():
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Argument ignore with value 255.5 is not of type" in str(error)
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
@ -279,7 +282,7 @@ def test_auto_contrast_invalid_cutoff_param_c():
"""
logger.info("Test AutoContrast C Op with invalid cutoff parameter")
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
@ -291,7 +294,7 @@ def test_auto_contrast_invalid_cutoff_param_c():
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Input cutoff is not within the required interval of (0 to 100)." in str(error)
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
@ -310,22 +313,22 @@ def test_auto_contrast_invalid_ignore_param_py():
"""
logger.info("Test AutoContrast python Op with invalid ignore parameter")
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(ignore=255.5),
F.ToTensor()])])
operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(ignore=255.5),
F.ToTensor()])])
except TypeError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Argument ignore with value 255.5 is not of type" in str(error)
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(ignore=(10, 100)),
F.ToTensor()])])
operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(ignore=(10, 100)),
F.ToTensor()])])
except TypeError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Argument ignore with value (10,100) is not of type" in str(error)
@ -337,22 +340,22 @@ def test_auto_contrast_invalid_cutoff_param_py():
"""
logger.info("Test AutoContrast python Op with invalid cutoff parameter")
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=-10.0),
F.ToTensor()])])
operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=-10.0),
F.ToTensor()])])
except ValueError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Input cutoff is not within the required interval of (0 to 100)." in str(error)
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=120.0),
F.ToTensor()])])
operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=120.0),
F.ToTensor()])])
except ValueError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Input cutoff is not within the required interval of (0 to 100)." in str(error)

16
tests/ut/python/dataset/test_batch.py
View File

@ -449,6 +449,22 @@ def test_batch_exception_13():
logger.info("Got an exception in DE: {}".format(str(e)))
assert "shard_id" in str(e)
# test non-functional parameters
try:
data1 = data1.batch(batch_size, output_columns="3")
sum([1 for _ in data1])
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "output_columns is currently not implemented." in str(e)
try:
data1 = data1.batch(batch_size, column_order="3")
sum([1 for _ in data1])
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "column_order is currently not implemented." in str(e)
if __name__ == '__main__':
test_batch_01()

18
tests/ut/python/dataset/test_bounding_box_augment.py
View File

@ -19,7 +19,7 @@ Testing the bounding box augment op in DE
import numpy as np
import mindspore.log as logger
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \
config_get_set_seed, config_get_set_num_parallel_workers, save_and_check_md5
@ -51,7 +51,7 @@ def test_bounding_box_augment_with_rotation_op(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "bounding_box_augment_rotation_c_result.npz"
@ -90,7 +90,7 @@ def test_bounding_box_augment_with_crop_op(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "bounding_box_augment_crop_c_result.npz"
@ -128,7 +128,7 @@ def test_bounding_box_augment_valid_ratio_c(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op]) # Add column for "bbox"
filename = "bounding_box_augment_valid_ratio_c_result.npz"
@ -165,7 +165,7 @@ def test_bounding_box_augment_op_coco_c(plot_vis=False):
dataCoco2 = dataCoco2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
unaugSamp, augSamp = [], []
@ -197,17 +197,17 @@ def test_bounding_box_augment_valid_edge_c(plot_vis=False):
# Add column for "bbox"
dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=lambda img, bbox:
(img, np.array([[0, 0, img.shape[1], img.shape[0], 0, 0, 0]]).astype(np.float32)))
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=lambda img, bbox:
(img, np.array([[0, 0, img.shape[1], img.shape[0], 0, 0, 0]]).astype(np.float32)))
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "bounding_box_augment_valid_edge_c_result.npz"
save_and_check_md5(dataVoc2, filename, generate_golden=GENERATE_GOLDEN)
@ -240,7 +240,7 @@ def test_bounding_box_augment_invalid_ratio_c():
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op]) # Add column for "bbox"
except ValueError as error:
logger.info("Got an exception in DE: {}".format(str(error)))

12
tests/ut/python/dataset/test_cache_map.py
View File

@ -18,7 +18,7 @@ Testing cache operator with mappable datasets
import os
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from mindspore import log as logger
from util import save_and_check_md5
@ -46,7 +46,7 @@ def test_cache_map_basic1():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR, cache=some_cache)
decode_op = c_vision.Decode()
ds1 = ds1.map(input_columns=["image"], operations=decode_op)
ds1 = ds1.repeat(4)
@ -75,7 +75,7 @@ def test_cache_map_basic2():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR)
decode_op = c_vision.Decode()
ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache)
ds1 = ds1.repeat(4)
@ -104,7 +104,7 @@ def test_cache_map_basic3():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR)
decode_op = c_vision.Decode()
ds1 = ds1.repeat(4)
ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache)
@ -128,7 +128,7 @@ def test_cache_map_basic4():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR, cache=some_cache)
decode_op = c_vision.Decode()
ds1 = ds1.repeat(4)
ds1 = ds1.map(input_columns=["image"], operations=decode_op)
@ -165,7 +165,7 @@ def test_cache_map_failure1():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR, cache=some_cache)
decode_op = c_vision.Decode()
ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache)
ds1 = ds1.repeat(4)

2
tests/ut/python/dataset/test_cache_nomap.py
View File

@ -19,7 +19,7 @@ import os
import pytest
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from mindspore import log as logger
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]

13
tests/ut/python/dataset/test_center_crop.py
View File

@ -17,8 +17,9 @@ Testing CenterCrop op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import diff_mse, visualize_list, save_and_check_md5
@ -93,8 +94,8 @@ def test_center_crop_comp(height=375, width=375, plot=False):
py_vision.CenterCrop([height, width]),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
image_c_cropped = []
image_py_cropped = []
@ -123,9 +124,9 @@ def test_crop_grayscale(height=375, width=375):
(lambda image: (image.transpose(1, 2, 0) * 255).astype(np.uint8))
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform())
data1 = data1.map(input_columns=["image"], operations=transform)
# If input is grayscale, the output dimensions should be single channel
crop_gray = vision.CenterCrop([height, width])

19
tests/ut/python/dataset/test_concat.py
View File

@ -17,7 +17,8 @@ import numpy as np
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as F
from mindspore import log as logger
@ -317,15 +318,15 @@ def test_concat_14():
DATA_DIR = "../data/dataset/testPK/data"
DATA_DIR2 = "../data/dataset/testImageNetData/train/"
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=3)
data2 = ds.ImageFolderDatasetV2(DATA_DIR2, num_samples=2)
data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=3)
data2 = ds.ImageFolderDataset(DATA_DIR2, num_samples=2)
transforms1 = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms1 = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
data1 = data1.map(input_columns=["image"], operations=transforms1())
data2 = data2.map(input_columns=["image"], operations=transforms1())
data1 = data1.map(input_columns=["image"], operations=transforms1)
data2 = data2.map(input_columns=["image"], operations=transforms1)
data3 = data1 + data2
expected, output = [], []
@ -351,7 +352,7 @@ def test_concat_15():
DATA_DIR = "../data/dataset/testPK/data"
DATA_DIR2 = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
data1 = ds.ImageFolderDatasetV2(DATA_DIR)
data1 = ds.ImageFolderDataset(DATA_DIR)
data2 = ds.TFRecordDataset(DATA_DIR2, columns_list=["image"])
data1 = data1.project(["image"])

4
tests/ut/python/dataset/test_concatenate_op.py
View File

@ -74,7 +74,7 @@ def test_concatenate_op_multi_input_string():
concatenate_op = data_trans.Concatenate(0, prepend=prepend_tensor, append=append_tensor)
data = data.map(input_columns=["col1", "col2"], columns_order=["out1"], output_columns=["out1"],
data = data.map(input_columns=["col1", "col2"], column_order=["out1"], output_columns=["out1"],
operations=concatenate_op)
expected = np.array(["dw", "df", "1", "2", "d", "3", "4", "e", "dwsdf", "df"], dtype='S')
for data_row in data:
@ -89,7 +89,7 @@ def test_concatenate_op_multi_input_numeric():
concatenate_op = data_trans.Concatenate(0, prepend=prepend_tensor)
data = data.map(input_columns=["col1", "col2"], columns_order=["out1"], output_columns=["out1"],
data = data.map(input_columns=["col1", "col2"], column_order=["out1"], output_columns=["out1"],
operations=concatenate_op)
expected = np.array([3, 5, 1, 2, 3, 4])
for data_row in data:

17
tests/ut/python/dataset/test_config.py
View File

@ -21,8 +21,9 @@ import glob
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import dataset_equal
@ -283,8 +284,8 @@ def test_deterministic_python_seed():
py_vision.RandomCrop([512, 512], [200, 200, 200, 200]),
py_vision.ToTensor(),
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
data1_output = []
# config.set_seed() calls random.seed()
for data_one in data1.create_dict_iterator(num_epochs=1):
@ -292,7 +293,7 @@ def test_deterministic_python_seed():
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
data2 = data2.map(input_columns=["image"], operations=transform)
# config.set_seed() calls random.seed(), resets seed for next dataset iterator
ds.config.set_seed(0)
@ -326,8 +327,8 @@ def test_deterministic_python_seed_multi_thread():
py_vision.RandomCrop([512, 512], [200, 200, 200, 200]),
py_vision.ToTensor(),
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform(), python_multiprocessing=True)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform, python_multiprocessing=True)
data1_output = []
# config.set_seed() calls random.seed()
for data_one in data1.create_dict_iterator(num_epochs=1):
@ -336,7 +337,7 @@ def test_deterministic_python_seed_multi_thread():
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
# If seed is set up on constructor
data2 = data2.map(input_columns=["image"], operations=transform(), python_multiprocessing=True)
data2 = data2.map(input_columns=["image"], operations=transform, python_multiprocessing=True)
# config.set_seed() calls random.seed()
ds.config.set_seed(0)

21
tests/ut/python/dataset/test_cut_out.py
View File

@ -18,8 +18,9 @@ Testing CutOut op in DE
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c
import mindspore.dataset.transforms.vision.py_transforms as f
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c
import mindspore.dataset.vision.py_transforms as f
from mindspore import log as logger
from util import visualize_image, visualize_list, diff_mse, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers
@ -43,8 +44,8 @@ def test_cut_out_op(plot=False):
f.ToTensor(),
f.RandomErasing(value='random')
]
transform_1 = f.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
@ -89,8 +90,8 @@ def test_cut_out_op_multicut(plot=False):
f.Decode(),
f.ToTensor(),
]
transform_1 = f.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
@ -144,8 +145,8 @@ def test_cut_out_md5():
f.ToTensor(),
f.Cutout(100)
]
transform = f.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename1 = "cut_out_01_c_result.npz"
@ -172,8 +173,8 @@ def test_cut_out_comp(plot=False):
f.ToTensor(),
f.Cutout(200)
]
transform_1 = f.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)

10
tests/ut/python/dataset/test_cutmix_batch_op.py
View File

@ -18,9 +18,9 @@ Testing the CutMixBatch op in DE
import numpy as np
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.dataset.transforms.c_transforms as data_trans
import mindspore.dataset.transforms.vision.utils as mode
import mindspore.dataset.vision.utils as mode
from mindspore import log as logger
from util import save_and_check_md5, diff_mse, visualize_list, config_get_set_seed, \
config_get_set_num_parallel_workers
@ -119,11 +119,11 @@ def test_cutmix_batch_success2(plot=False):
def test_cutmix_batch_success3(plot=False):
"""
Test CutMixBatch op with default values for alpha and prob on a batch of HWC images on ImageFolderDatasetV2
Test CutMixBatch op with default values for alpha and prob on a batch of HWC images on ImageFolderDataset
"""
logger.info("test_cutmix_batch_success3")
ds_original = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR2, shuffle=False)
ds_original = ds.ImageFolderDataset(dataset_dir=DATA_DIR2, shuffle=False)
decode_op = vision.Decode()
ds_original = ds_original.map(input_columns=["image"], operations=[decode_op])
ds_original = ds_original.batch(4, pad_info={}, drop_remainder=True)
@ -136,7 +136,7 @@ def test_cutmix_batch_success3(plot=False):
images_original = np.append(images_original, image, axis=0)
# CutMix Images
data1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR2, shuffle=False)
data1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR2, shuffle=False)
decode_op = vision.Decode()
data1 = data1.map(input_columns=["image"], operations=[decode_op])

2
tests/ut/python/dataset/test_dataset_numpy_slices.py
View File

@ -18,7 +18,7 @@ import numpy as np
import pandas as pd
import mindspore.dataset as de
from mindspore import log as logger
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
def test_numpy_slices_list_1():

4
tests/ut/python/dataset/test_datasets_celeba.py
View File

@ -12,9 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
DATA_DIR = "../data/dataset/testCelebAData/"

2
tests/ut/python/dataset/test_datasets_coco.py
View File

@ -14,7 +14,7 @@
# ==============================================================================
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
DATA_DIR = "../data/dataset/testCOCO/train/"
DATA_DIR_2 = "../data/dataset/testCOCO/train"

16
tests/ut/python/dataset/test_datasets_generator.py
View File

@ -244,7 +244,7 @@ def test_generator_8():
data1 = data1.map(input_columns="col0", output_columns="out0", operations=(lambda x: x * 3),
num_parallel_workers=2)
data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x * 7, x)),
num_parallel_workers=2, columns_order=["out0", "out1", "out2"])
num_parallel_workers=2, column_order=["out0", "out1", "out2"])
data1 = data1.map(input_columns="out2", output_columns="out2", operations=(lambda x: x + 1),
num_parallel_workers=2)
@ -299,7 +299,7 @@ def test_generator_10():
# apply dataset operations
data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"])
data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x, x * 5)),
columns_order=['col0', 'out1', 'out2'], num_parallel_workers=2)
column_order=['col0', 'out1', 'out2'], num_parallel_workers=2)
# Expected column order is |col0|out1|out2|
i = 0
@ -318,17 +318,17 @@ def test_generator_11():
Test map column order when len(input_columns) != len(output_columns).
"""
logger.info("Test map column order when len(input_columns) != len(output_columns), "
"and columns_order drops some columns.")
"and column_order drops some columns.")
# apply dataset operations
data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"])
data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x, x * 5)),
columns_order=['out1', 'out2'], num_parallel_workers=2)
column_order=['out1', 'out2'], num_parallel_workers=2)
# Expected column order is |out1|out2|
i = 0
for item in data1.create_tuple_iterator(num_epochs=1):
# len should be 2 because col0 is dropped (not included in columns_order)
# len should be 2 because col0 is dropped (not included in column_order)
assert len(item) == 2
golden = np.array([[i, i + 1], [i + 2, i + 3]])
np.testing.assert_array_equal(item[0], golden)
@ -358,7 +358,7 @@ def test_generator_12():
i = i + 1
data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"])
data1 = data1.map(operations=(lambda x: (x * 5)), columns_order=["col1", "col0"], num_parallel_workers=2)
data1 = data1.map(operations=(lambda x: (x * 5)), column_order=["col1", "col0"], num_parallel_workers=2)
# Expected column order is |col0|col1|
i = 0
@ -392,7 +392,7 @@ def test_generator_13():
i = i + 1
for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary
# len should be 2 because col0 is dropped (not included in columns_order)
# len should be 2 because col0 is dropped (not included in column_order)
assert len(item) == 2
golden = np.array([i * 5])
np.testing.assert_array_equal(item["out0"], golden)
@ -508,7 +508,7 @@ def test_generator_error_3():
for _ in data1:
pass
assert "When (len(input_columns) != len(output_columns)), columns_order must be specified." in str(info.value)
assert "When (len(input_columns) != len(output_columns)), column_order must be specified." in str(info.value)
def test_generator_error_4():

8
tests/ut/python/dataset/test_datasets_get_dataset_size.py
View File

@ -27,16 +27,16 @@ CIFAR100_DATA_DIR = "../data/dataset/testCifar100Data"
def test_imagenet_rawdata_dataset_size():
ds_total = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR)
ds_total = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR)
assert ds_total.get_dataset_size() == 6
ds_shard_1_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=1, shard_id=0)
ds_shard_1_0 = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR, num_shards=1, shard_id=0)
assert ds_shard_1_0.get_dataset_size() == 6
ds_shard_2_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=2, shard_id=0)
ds_shard_2_0 = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR, num_shards=2, shard_id=0)
assert ds_shard_2_0.get_dataset_size() == 3
ds_shard_3_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=3, shard_id=0)
ds_shard_3_0 = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR, num_shards=3, shard_id=0)
assert ds_shard_3_0.get_dataset_size() == 2

42
tests/ut/python/dataset/test_datasets_imagefolder.py
View File

@ -24,7 +24,7 @@ def test_imagefolder_basic():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR)
data1 = ds.ImageFolderDataset(DATA_DIR)
data1 = data1.repeat(repeat_count)
num_iter = 0
@ -44,7 +44,7 @@ def test_imagefolder_numsamples():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10, num_parallel_workers=2)
data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=10, num_parallel_workers=2)
data1 = data1.repeat(repeat_count)
num_iter = 0
@ -58,7 +58,7 @@ def test_imagefolder_numsamples():
assert num_iter == 10
random_sampler = ds.RandomSampler(num_samples=3, replacement=True)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_parallel_workers=2, sampler=random_sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, num_parallel_workers=2, sampler=random_sampler)
num_iter = 0
for item in data1.create_dict_iterator(num_epochs=1):
@ -67,7 +67,7 @@ def test_imagefolder_numsamples():
assert num_iter == 3
random_sampler = ds.RandomSampler(num_samples=3, replacement=False)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_parallel_workers=2, sampler=random_sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, num_parallel_workers=2, sampler=random_sampler)
num_iter = 0
for item in data1.create_dict_iterator(num_epochs=1):
@ -82,7 +82,7 @@ def test_imagefolder_numshards():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3)
data1 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=3)
data1 = data1.repeat(repeat_count)
num_iter = 0
@ -102,7 +102,7 @@ def test_imagefolder_shardid():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=1)
data1 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=1)
data1 = data1.repeat(repeat_count)
num_iter = 0
@ -122,7 +122,7 @@ def test_imagefolder_noshuffle():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, shuffle=False)
data1 = ds.ImageFolderDataset(DATA_DIR, shuffle=False)
data1 = data1.repeat(repeat_count)
num_iter = 0
@ -142,7 +142,7 @@ def test_imagefolder_extrashuffle():
repeat_count = 2
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, shuffle=True)
data1 = ds.ImageFolderDataset(DATA_DIR, shuffle=True)
data1 = data1.shuffle(buffer_size=5)
data1 = data1.repeat(repeat_count)
@ -164,7 +164,7 @@ def test_imagefolder_classindex():
# apply dataset operations
class_index = {"class3": 333, "class1": 111}
data1 = ds.ImageFolderDatasetV2(DATA_DIR, class_indexing=class_index, shuffle=False)
data1 = ds.ImageFolderDataset(DATA_DIR, class_indexing=class_index, shuffle=False)
data1 = data1.repeat(repeat_count)
golden = [111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111,
@ -189,7 +189,7 @@ def test_imagefolder_negative_classindex():
# apply dataset operations
class_index = {"class3": -333, "class1": 111}
data1 = ds.ImageFolderDatasetV2(DATA_DIR, class_indexing=class_index, shuffle=False)
data1 = ds.ImageFolderDataset(DATA_DIR, class_indexing=class_index, shuffle=False)
data1 = data1.repeat(repeat_count)
golden = [111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111,
@ -214,7 +214,7 @@ def test_imagefolder_extensions():
# apply dataset operations
ext = [".jpg", ".JPEG"]
data1 = ds.ImageFolderDatasetV2(DATA_DIR, extensions=ext)
data1 = ds.ImageFolderDataset(DATA_DIR, extensions=ext)
data1 = data1.repeat(repeat_count)
num_iter = 0
@ -235,7 +235,7 @@ def test_imagefolder_decode():
# apply dataset operations
ext = [".jpg", ".JPEG"]
data1 = ds.ImageFolderDatasetV2(DATA_DIR, extensions=ext, decode=True)
data1 = ds.ImageFolderDataset(DATA_DIR, extensions=ext, decode=True)
data1 = data1.repeat(repeat_count)
num_iter = 0
@ -262,7 +262,7 @@ def test_sequential_sampler():
# apply dataset operations
sampler = ds.SequentialSampler()
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
result = []
@ -283,7 +283,7 @@ def test_random_sampler():
# apply dataset operations
sampler = ds.RandomSampler()
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
@ -304,7 +304,7 @@ def test_distributed_sampler():
# apply dataset operations
sampler = ds.DistributedSampler(10, 1)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
@ -325,7 +325,7 @@ def test_pk_sampler():
# apply dataset operations
sampler = ds.PKSampler(3)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
@ -347,7 +347,7 @@ def test_subset_random_sampler():
# apply dataset operations
indices = [0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 11]
sampler = ds.SubsetRandomSampler(indices)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
@ -369,7 +369,7 @@ def test_weighted_random_sampler():
# apply dataset operations
weights = [1.0, 0.1, 0.02, 0.3, 0.4, 0.05, 1.2, 0.13, 0.14, 0.015, 0.16, 1.1]
sampler = ds.WeightedRandomSampler(weights, 11)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
@ -389,7 +389,7 @@ def test_imagefolder_rename():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10)
data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=10)
data1 = data1.repeat(repeat_count)
num_iter = 0
@ -421,8 +421,8 @@ def test_imagefolder_zip():
repeat_count = 2
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10)
data2 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10)
data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=10)
data2 = ds.ImageFolderDataset(DATA_DIR, num_samples=10)
data1 = data1.repeat(repeat_count)
# rename dataset2 for no conflict

6
tests/ut/python/dataset/test_datasets_sharding.py
View File

@ -20,9 +20,9 @@ def test_imagefolder_shardings(print_res=False):
image_folder_dir = "../data/dataset/testPK/data"
def sharding_config(num_shards, shard_id, num_samples, shuffle, class_index, repeat_cnt=1):
data1 = ds.ImageFolderDatasetV2(image_folder_dir, num_samples=num_samples, num_shards=num_shards,
shard_id=shard_id,
shuffle=shuffle, class_indexing=class_index, decode=True)
data1 = ds.ImageFolderDataset(image_folder_dir, num_samples=num_samples, num_shards=num_shards,
shard_id=shard_id,
shuffle=shuffle, class_indexing=class_index, decode=True)
data1 = data1.repeat(repeat_cnt)
res = []
for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary

2
tests/ut/python/dataset/test_datasets_voc.py
View File

@ -13,7 +13,7 @@
# limitations under the License.
# ==============================================================================
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
DATA_DIR = "../data/dataset/testVOC2012"
IMAGE_SHAPE = [2268, 2268, 2268, 2268, 642, 607, 561, 596, 612, 2268]

2
tests/ut/python/dataset/test_decode.py
View File

@ -18,7 +18,7 @@ Testing Decode op in DE
import cv2
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
from util import diff_mse

2
tests/ut/python/dataset/test_deviceop_cpu.py
View File

@ -15,7 +15,7 @@
import time
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]

2
tests/ut/python/dataset/test_duplicate_op.py
View File

@ -24,7 +24,7 @@ import mindspore.dataset.transforms.c_transforms as ops
def compare(array):
data = ds.NumpySlicesDataset([array], column_names="x")
array = np.array(array)
data = data.map(input_columns=["x"], output_columns=["x", "y"], columns_order=["x", "y"],
data = data.map(input_columns=["x"], output_columns=["x", "y"], column_order=["x", "y"],
operations=ops.Duplicate())
for d in data.create_dict_iterator(num_epochs=1):
np.testing.assert_array_equal(array, d["x"])

2
tests/ut/python/dataset/test_epoch_ctrl.py
View File

@ -21,7 +21,7 @@ import numpy as np
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]

59
tests/ut/python/dataset/test_equalize.py
View File

@ -18,8 +18,9 @@ Testing Equalize op in DE
import numpy as np
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as C
import mindspore.dataset.vision.py_transforms as F
from mindspore import log as logger
from util import visualize_list, visualize_one_channel_dataset, diff_mse, save_and_check_md5
@ -36,14 +37,14 @@ def test_equalize_py(plot=False):
logger.info("Test Equalize")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
ds_original = ds.map(input_columns="image",
operations=transforms_original())
operations=transforms_original)
ds_original = ds_original.batch(512)
@ -56,15 +57,15 @@ def test_equalize_py(plot=False):
axis=0)
# Color Equalized Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_equalize = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.Equalize(),
F.ToTensor()])
transforms_equalize = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.Equalize(),
F.ToTensor()])
ds_equalize = ds.map(input_columns="image",
operations=transforms_equalize())
operations=transforms_equalize)
ds_equalize = ds_equalize.batch(512)
@ -93,7 +94,7 @@ def test_equalize_c(plot=False):
logger.info("Test Equalize cpp op")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = [C.Decode(), C.Resize(size=[224, 224])]
@ -111,7 +112,7 @@ def test_equalize_c(plot=False):
axis=0)
# Equalize Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transform_equalize = [C.Decode(), C.Resize(size=[224, 224]),
C.Equalize()]
@ -145,7 +146,7 @@ def test_equalize_py_c(plot=False):
logger.info("Test Equalize cpp and python op")
# equalize Images in cpp
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(), C.Resize((224, 224))])
@ -163,17 +164,17 @@ def test_equalize_py_c(plot=False):
axis=0)
# Equalize images in python
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(), C.Resize((224, 224))])
transforms_p_equalize = F.ComposeOp([lambda img: img.astype(np.uint8),
F.ToPIL(),
F.Equalize(),
np.array])
transforms_p_equalize = mindspore.dataset.transforms.py_transforms.Compose([lambda img: img.astype(np.uint8),
F.ToPIL(),
F.Equalize(),
np.array])
ds_p_equalize = ds.map(input_columns="image",
operations=transforms_p_equalize())
operations=transforms_p_equalize)
ds_p_equalize = ds_p_equalize.batch(512)
@ -204,7 +205,7 @@ def test_equalize_one_channel():
c_op = C.Equalize()
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
@ -253,12 +254,12 @@ def test_equalize_md5_py():
logger.info("Test Equalize")
# First dataset
data1 = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
transforms = F.ComposeOp([F.Decode(),
F.Equalize(),
F.ToTensor()])
data1 = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Equalize(),
F.ToTensor()])
data1 = data1.map(input_columns="image", operations=transforms())
data1 = data1.map(input_columns="image", operations=transforms)
# Compare with expected md5 from images
filename = "equalize_01_result.npz"
save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN)
@ -271,7 +272,7 @@ def test_equalize_md5_c():
logger.info("Test Equalize cpp op with md5 check")
# Generate dataset
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_equalize = [C.Decode(),
C.Resize(size=[224, 224]),

2
tests/ut/python/dataset/test_exceptions.py
View File

@ -15,7 +15,7 @@
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]

2
tests/ut/python/dataset/test_filterop.py
View File

@ -16,7 +16,7 @@
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as cde
import mindspore.dataset.vision.c_transforms as cde
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json"

19
tests/ut/python/dataset/test_five_crop.py
View File

@ -18,7 +18,8 @@ import pytest
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as vision
from mindspore import log as logger
from util import visualize_list, save_and_check_md5
@ -39,8 +40,8 @@ def test_five_crop_op(plot=False):
vision.Decode(),
vision.ToTensor(),
]
transform_1 = vision.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
@ -49,8 +50,8 @@ def test_five_crop_op(plot=False):
vision.FiveCrop(200),
lambda images: np.stack([vision.ToTensor()(image) for image in images]) # 4D stack of 5 images
]
transform_2 = vision.ComposeOp(transforms_2)
data2 = data2.map(input_columns=["image"], operations=transform_2())
transform_2 = mindspore.dataset.transforms.py_transforms.Compose(transforms_2)
data2 = data2.map(input_columns=["image"], operations=transform_2)
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)):
@ -83,8 +84,8 @@ def test_five_crop_error_msg():
vision.FiveCrop(200),
vision.ToTensor()
]
transform = vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
with pytest.raises(RuntimeError) as info:
for _ in data:
@ -108,8 +109,8 @@ def test_five_crop_md5():
vision.FiveCrop(100),
lambda images: np.stack([vision.ToTensor()(image) for image in images]) # 4D stack of 5 images
]
transform = vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename = "five_crop_01_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)

4
tests/ut/python/dataset/test_flat_map.py
View File

@ -27,7 +27,7 @@ def test_flat_map_1():
def flat_map_func(x):
data_dir = x[0].item().decode('utf8')
d = ds.ImageFolderDatasetV2(data_dir)
d = ds.ImageFolderDataset(data_dir)
return d
data = ds.TextFileDataset(DATA_FILE)
@ -47,7 +47,7 @@ def test_flat_map_2():
def flat_map_func_1(x):
data_dir = x[0].item().decode('utf8')
d = ds.ImageFolderDatasetV2(data_dir)
d = ds.ImageFolderDataset(data_dir)
return d
def flat_map_func_2(x):

6
tests/ut/python/dataset/test_get_col_names.py
View File

@ -15,7 +15,7 @@
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
CELEBA_DIR = "../data/dataset/testCelebAData"
CIFAR10_DIR = "../data/dataset/testCifar10Data"
@ -75,7 +75,7 @@ def test_get_column_name_generator():
def test_get_column_name_imagefolder():
data = ds.ImageFolderDatasetV2(IMAGE_FOLDER_DIR)
data = ds.ImageFolderDataset(IMAGE_FOLDER_DIR)
assert data.get_col_names() == ["image", "label"]
@ -105,7 +105,7 @@ def test_get_column_name_map():
assert data.get_col_names() == ["col1", "label"]
data = ds.Cifar10Dataset(CIFAR10_DIR)
data = data.map(input_columns=["image"], operations=center_crop_op, output_columns=["col1", "col2"],
columns_order=["col2", "col1"])
column_order=["col2", "col1"])
assert data.get_col_names() == ["col2", "col1"]

4
tests/ut/python/dataset/test_get_size.py
View File

@ -150,13 +150,13 @@ def test_manifest():
def test_imagefolder():
data = ds.ImageFolderDatasetV2("../data/dataset/testPK/data/")
data = ds.ImageFolderDataset("../data/dataset/testPK/data/")
assert data.get_dataset_size() == 44
assert data.num_classes() == 4
data = data.shuffle(100)
assert data.num_classes() == 4
data = ds.ImageFolderDatasetV2("../data/dataset/testPK/data/", num_samples=10)
data = ds.ImageFolderDataset("../data/dataset/testPK/data/", num_samples=10)
assert data.get_dataset_size() == 10
assert data.num_classes() == 4

59
tests/ut/python/dataset/test_invert.py
View File

@ -18,8 +18,9 @@ Testing Invert op in DE
import numpy as np
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as F
import mindspore.dataset.vision.c_transforms as C
from mindspore import log as logger
from util import visualize_list, save_and_check_md5, diff_mse
@ -35,14 +36,14 @@ def test_invert_py(plot=False):
logger.info("Test Invert Python op")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
ds_original = ds.map(input_columns="image",
operations=transforms_original())
operations=transforms_original)
ds_original = ds_original.batch(512)
@ -55,15 +56,15 @@ def test_invert_py(plot=False):
axis=0)
# Color Inverted Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_invert = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.Invert(),
F.ToTensor()])
transforms_invert = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.Invert(),
F.ToTensor()])
ds_invert = ds.map(input_columns="image",
operations=transforms_invert())
operations=transforms_invert)
ds_invert = ds_invert.batch(512)
@ -92,7 +93,7 @@ def test_invert_c(plot=False):
logger.info("Test Invert cpp op")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = [C.Decode(), C.Resize(size=[224, 224])]
@ -110,7 +111,7 @@ def test_invert_c(plot=False):
axis=0)
# Invert Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transform_invert = [C.Decode(), C.Resize(size=[224, 224]),
C.Invert()]
@ -144,7 +145,7 @@ def test_invert_py_c(plot=False):
logger.info("Test Invert cpp and python op")
# Invert Images in cpp
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(), C.Resize((224, 224))])
@ -162,17 +163,17 @@ def test_invert_py_c(plot=False):
axis=0)
# invert images in python
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(), C.Resize((224, 224))])
transforms_p_invert = F.ComposeOp([lambda img: img.astype(np.uint8),
F.ToPIL(),
F.Invert(),
np.array])
transforms_p_invert = mindspore.dataset.transforms.py_transforms.Compose([lambda img: img.astype(np.uint8),
F.ToPIL(),
F.Invert(),
np.array])
ds_p_invert = ds.map(input_columns="image",
operations=transforms_p_invert())
operations=transforms_p_invert)
ds_p_invert = ds_p_invert.batch(512)
@ -203,7 +204,7 @@ def test_invert_one_channel():
c_op = C.Invert()
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
@ -224,13 +225,13 @@ def test_invert_md5_py():
logger.info("Test Invert python op with md5 check")
# Generate dataset
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_invert = F.ComposeOp([F.Decode(),
F.Invert(),
F.ToTensor()])
transforms_invert = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Invert(),
F.ToTensor()])
data = ds.map(input_columns="image", operations=transforms_invert())
data = ds.map(input_columns="image", operations=transforms_invert)
# Compare with expected md5 from images
filename = "invert_01_result_py.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
@ -243,7 +244,7 @@ def test_invert_md5_c():
logger.info("Test Invert cpp op with md5 check")
# Generate dataset
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_invert = [C.Decode(),
C.Resize(size=[224, 224]),

29
tests/ut/python/dataset/test_linear_transformation.py
View File

@ -17,7 +17,8 @@ Testing LinearTransformation op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import diff_mse, visualize_list, save_and_check_md5
@ -46,11 +47,11 @@ def test_linear_transformation_op(plot=False):
py_vision.CenterCrop([height, weight]),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform())
data1 = data1.map(input_columns=["image"], operations=transform)
# Note: if transformation matrix is diagonal matrix with all 1 in diagonal,
# the output matrix in expected to be the same as the input matrix.
data1 = data1.map(input_columns=["image"],
@ -58,7 +59,7 @@ def test_linear_transformation_op(plot=False):
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
data2 = data2.map(input_columns=["image"], operations=transform)
image_transformed = []
image = []
@ -96,8 +97,8 @@ def test_linear_transformation_md5():
py_vision.ToTensor(),
py_vision.LinearTransformation(transformation_matrix, mean_vector)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename = "linear_transformation_01_result.npz"
@ -126,8 +127,8 @@ def test_linear_transformation_exception_01():
py_vision.ToTensor(),
py_vision.LinearTransformation(None, mean_vector)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
except TypeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Argument transformation_matrix with value None is not of type (<class 'numpy.ndarray'>,)" in str(e)
@ -155,8 +156,8 @@ def test_linear_transformation_exception_02():
py_vision.ToTensor(),
py_vision.LinearTransformation(transformation_matrix, None)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
except TypeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Argument mean_vector with value None is not of type (<class 'numpy.ndarray'>,)" in str(e)
@ -185,8 +186,8 @@ def test_linear_transformation_exception_03():
py_vision.ToTensor(),
py_vision.LinearTransformation(transformation_matrix, mean_vector)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "square matrix" in str(e)
@ -215,8 +216,8 @@ def test_linear_transformation_exception_04():
py_vision.ToTensor(),
py_vision.LinearTransformation(transformation_matrix, mean_vector)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "should match" in str(e)

Some files were not shown because too many files have changed in this diff Show More