p85971406
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!35134 [MD] Adding docstrings for minddata UT Python Part 3 

Merge pull request !35134 from davidanugraha/add_dataset_test_comment_part3
This commit is contained in:
i-robot 2022-05-30 22:29:05 +00:00 committed by Gitee
parent 226cc15439 34d3755e20
commit a4785b9baa
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
53 changed files with 1995 additions and 821 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
tests/ut/python/dataset/test_apply.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -28,6 +28,11 @@ def generator_1d():
def test_apply_generator_case():
"""
Feature: Apply op
Description: Test apply op on GeneratorDataset
Expectation: Output is equal to the expected output
"""
# apply dataset operations
data1 = ds.GeneratorDataset(generator_1d, ["data"])
data2 = ds.GeneratorDataset(generator_1d, ["data"])
@ -46,6 +51,11 @@ def test_apply_generator_case():
def test_apply_imagefolder_case():
"""
Feature: Apply op
Description: Test apply op on ImageFolderDataset
Expectation: Output is equal to the expected output
"""
# apply dataset map operations
data1 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=3)
data2 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=3)
@ -70,6 +80,11 @@ def test_apply_imagefolder_case():
def test_apply_flow_case_0(id_=0):
"""
Feature: Apply op
Description: Test control flow operation by applying batch op
Expectation: Output is equal to the expected output
"""
# apply control flow operations
data1 = ds.GeneratorDataset(generator_1d, ["data"])
@ -101,6 +116,11 @@ def test_apply_flow_case_0(id_=0):
def test_apply_flow_case_1(id_=1):
"""
Feature: Apply op
Description: Test control flow operation by applying repeat op
Expectation: Output is equal to the expected output
"""
# apply control flow operations
data1 = ds.GeneratorDataset(generator_1d, ["data"])
@ -132,6 +152,11 @@ def test_apply_flow_case_1(id_=1):
def test_apply_flow_case_2(id_=2):
"""
Feature: Apply op
Description: Test control flow operation by applying batch op then repeat op
Expectation: Output is equal to the expected output
"""
# apply control flow operations
data1 = ds.GeneratorDataset(generator_1d, ["data"])
@ -163,6 +188,11 @@ def test_apply_flow_case_2(id_=2):
def test_apply_flow_case_3(id_=3):
"""
Feature: Apply op
Description: Test control flow operation by applying shuffle op
Expectation: Output is equal to the expected output
"""
# apply control flow operations
data1 = ds.GeneratorDataset(generator_1d, ["data"])
@ -194,6 +224,11 @@ def test_apply_flow_case_3(id_=3):
def test_apply_exception_case():
"""
Feature: Apply op
Description: Test apply op using invalid inputs
Expectation: Correct error is raised as expected
"""
# apply exception operations
data1 = ds.GeneratorDataset(generator_1d, ["data"])

72
tests/ut/python/dataset/test_bucket_batch_by_length.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -42,6 +42,11 @@ def generate_3_columns(n):
def test_bucket_batch_invalid_input():
"""
Feature: bucket_batch_by_length op
Description: Test bucket_batch_by_length op with invalid inputs
Expectation: Correct error is raised as expected
"""
dataset = ds.GeneratorDataset((lambda: generate_sequential_same_shape(10)), ["col1"])
column_names = ["col1"]
@ -119,6 +124,11 @@ def test_bucket_batch_invalid_input():
def test_bucket_batch_multi_bucket_no_padding():
"""
Feature: bucket_batch_by_length op
Description: Test bucket_batch_by_length op with multi buckets but without padding
Expectation: Output is equal to the expected output
"""
dataset = ds.GeneratorDataset((lambda: generate_sequential_same_shape(10)), ["col1"])
column_names = ["col1"]
@ -142,6 +152,11 @@ def test_bucket_batch_multi_bucket_no_padding():
def test_bucket_batch_multi_bucket_no_padding_repeat():
"""
Feature: bucket_batch_by_length op
Description: Test bucket_batch_by_length op with multi buckets but without padding, followed by repeat op
Expectation: Output is equal to the expected output
"""
dataset = ds.GeneratorDataset((lambda: generate_sequential_same_shape(10)), ["col1"])
column_names = ["col1"]
@ -170,6 +185,11 @@ def test_bucket_batch_multi_bucket_no_padding_repeat():
def test_bucket_batch_multi_bucket_with_padding():
"""
Feature: bucket_batch_by_length op
Description: Test bucket_batch_by_length op with multi buckets and padding
Expectation: Output is equal to the expected output
"""
dataset = ds.GeneratorDataset((lambda: generate_sequential(10)), ["col1"])
column_names = ["col1"]
@ -201,6 +221,11 @@ def test_bucket_batch_multi_bucket_with_padding():
def test_bucket_batch_single_bucket_no_padding():
"""
Feature: bucket_batch_by_length op
Description: Test bucket_batch_by_length op with single bucket but without padding
Expectation: Output is equal to the expected output
"""
dataset = ds.GeneratorDataset((lambda: generate_sequential_same_shape(10)), ["col1"])
column_names = ["col1"]
@ -222,6 +247,11 @@ def test_bucket_batch_single_bucket_no_padding():
def test_bucket_batch_single_bucket_with_padding():
"""
Feature: bucket_batch_by_length op
Description: Test bucket_batch_by_length op with single bucket and padding
Expectation: Output is equal to the expected output
"""
dataset = ds.GeneratorDataset((lambda: generate_sequential(9)), ["col1"])
column_names = ["col1"]
@ -252,6 +282,11 @@ def test_bucket_batch_single_bucket_with_padding():
def test_bucket_batch_pad_to_bucket_boundary():
"""
Feature: bucket_batch_by_length op
Description: Test bucket_batch_by_length op with pad_to_bucket_boundary
Expectation: Output is equal to the expected output
"""
dataset = ds.GeneratorDataset((lambda: generate_sequential(9)), ["col1"])
column_names = ["col1"]
@ -283,6 +318,11 @@ def test_bucket_batch_pad_to_bucket_boundary():
def test_bucket_batch_default_pad():
"""
Feature: bucket_batch_by_length op
Description: Test bucket_batch_by_length op with default pad
Expectation: Output is equal to the expected output
"""
dataset = ds.GeneratorDataset((lambda: generate_sequential(15)), ["col1"])
column_names = ["col1"]
@ -319,6 +359,11 @@ def test_bucket_batch_default_pad():
def test_bucket_batch_drop_remainder():
"""
Feature: bucket_batch_by_length op
Description: Test bucket_batch_by_length op with drop_remainder
Expectation: Output is equal to the expected output
"""
dataset = ds.GeneratorDataset((lambda: generate_sequential_same_shape(27)), ["col1"])
column_names = ["col1"]
@ -350,6 +395,11 @@ def test_bucket_batch_drop_remainder():
def test_bucket_batch_default_length_function():
"""
Feature: bucket_batch_by_length op
Description: Test bucket_batch_by_length op with default element_length_function
Expectation: Output is equal to the expected output
"""
dataset = ds.GeneratorDataset((lambda: generate_sequential(9)), ["col1"])
column_names = ["col1"]
@ -380,6 +430,11 @@ def test_bucket_batch_default_length_function():
def test_bucket_batch_multi_column():
"""
Feature: bucket_batch_by_length op
Description: Test bucket_batch_by_length op with multi columns
Expectation: Output is equal to the expected output
"""
dataset = ds.GeneratorDataset((lambda: generate_2_columns(10)), ["same_shape", "variable_shape"])
column_names = ["same_shape"]
@ -417,6 +472,11 @@ def test_bucket_batch_multi_column():
def test_bucket_batch_three_columns():
"""
Feature: bucket_batch_by_length op
Description: Test bucket_batch_by_length op with three columns
Expectation: Output is equal to the expected output
"""
dataset = ds.GeneratorDataset((lambda: generate_3_columns(10)), ["same_shape", "same_shape2", "variable_shape"])
column_names = ["same_shape2"]
@ -458,6 +518,11 @@ def test_bucket_batch_three_columns():
def test_bucket_batch_get_dataset_size():
"""
Feature: bucket_batch_by_length op
Description: Test bucket_batch_by_length op followed by get_dataset_size op
Expectation: Output is equal to the expected output
"""
dataset = ds.GeneratorDataset((lambda: generate_sequential_same_shape(10)), ["col1"])
column_names = ["col1"]
@ -478,6 +543,11 @@ def test_bucket_batch_get_dataset_size():
def test_bucket_batch_invalid_column():
"""
Feature: bucket_batch_by_length op
Description: Test bucket_batch_by_length op with invalid column
Expectation: Error is raised as expected
"""
dataset = ds.GeneratorDataset((lambda: generate_sequential_same_shape(10)), ["col1"])
column_names = ["invalid_column"]

707
tests/ut/python/dataset/test_cache_map.py
View File

File diff suppressed because it is too large Load Diff

536
tests/ut/python/dataset/test_cache_nomap.py
View File

File diff suppressed because it is too large Load Diff

47
tests/ut/python/dataset/test_config.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -43,7 +43,9 @@ def config_error_func(config_interface, input_args, err_type, except_err_msg):
def test_basic():
"""
Test basic configuration functions
Feature: Config
Description: Test basic configuration functions
Expectation: Output is equal to the expected value
"""
# Save original configuration values
num_parallel_workers_original = ds.config.get_num_parallel_workers()
@ -80,14 +82,18 @@ def test_basic():
def test_get_seed():
"""
This gets the seed value without explicitly setting a default, expect int.
Feature: Config
Description: Test get_seed value without explicitly setting a default
Expectation: Expecting to get an int
"""
assert isinstance(ds.config.get_seed(), int)
def test_pipeline():
"""
Test that our configuration pipeline works when we set parameters at different locations in dataset code
Feature: Config
Description: Test that the config pipeline works when parameters are set at different locations in dataset code
Expectation: Output is equal to the expected value
"""
# Save original configuration values
num_parallel_workers_original = ds.config.get_num_parallel_workers()
@ -120,7 +126,9 @@ def test_pipeline():
def test_deterministic_run_fail():
"""
Test RandomCrop with seed, expected to fail
Feature: Config
Description: Test RandomCrop with seed
Expectation: Exception is raised as expected
"""
logger.info("test_deterministic_run_fail")
@ -162,7 +170,9 @@ def test_deterministic_run_fail():
def test_seed_undeterministic():
"""
Test seed with num parallel workers in c, this test is expected to fail some of the time
Feature: Config
Description: Test seed with num_parallel_workers in Cpp
Expectation: Exception is raised some of the time
"""
logger.info("test_seed_undeterministic")
@ -202,7 +212,9 @@ def test_seed_undeterministic():
def test_seed_deterministic():
"""
Test deterministic run with setting the seed, only works with num_parallel worker = 1
Feature: Config
Description: Test deterministic run with setting the seed
Expectation: Runs successfully if num_parallel_worker=1
"""
logger.info("test_seed_deterministic")
@ -237,7 +249,9 @@ def test_seed_deterministic():
def test_deterministic_run_distribution():
"""
Test deterministic run with with setting the seed being used in a distribution
Feature: Config
Description: Test deterministic run with with setting the seed being used in a distribution
Expectation: Output is equal to the expected output
"""
logger.info("test_deterministic_run_distribution")
@ -272,7 +286,9 @@ def test_deterministic_run_distribution():
def test_deterministic_python_seed():
"""
Test deterministic execution with seed in python
Feature: Config
Description: Test deterministic execution with seed in Python
Expectation: Output is equal to the expected output
"""
logger.info("test_deterministic_python_seed")
@ -317,7 +333,9 @@ def test_deterministic_python_seed():
def test_deterministic_python_seed_multi_thread():
"""
Test deterministic execution with seed in python, this fails with multi-thread pyfunc run
Feature: Config
Description: Test deterministic execution with seed in Python with multi-thread PyFunc run
Expectation: Exception is raised as expected
"""
logger.info("test_deterministic_python_seed_multi_thread")
@ -376,7 +394,9 @@ def test_deterministic_python_seed_multi_thread():
def test_auto_num_workers_error():
"""
Test auto_num_workers error
Feature: Config
Description: Test set_auto_num_workers with invalid input
Expectation: Error is raised as expected
"""
err_msg = ""
try:
@ -389,9 +409,10 @@ def test_auto_num_workers_error():
def test_auto_num_workers():
"""
Test auto_num_workers can be set.
Feature: Config
Description: Test set_auto_num_workers with no argument
Expectation: Output is equal to the expected value
"""
saved_config = ds.config.get_auto_num_workers()
assert isinstance(saved_config, bool)
# change to a different config

18
tests/ut/python/dataset/test_cut_out.py
View File

@ -32,7 +32,9 @@ GENERATE_GOLDEN = False
def test_cut_out_op(plot=False):
"""
Test CutOut
Feature: CutOut op
Description: Test CutOut op by comparing between Python and Cpp implementation
Expectation: Both outputs are equal to each other
"""
logger.info("test_cut_out")
@ -80,7 +82,9 @@ def test_cut_out_op(plot=False):
def test_cut_out_op_multicut(plot=False):
"""
Test CutOut
Feature: CutOut op
Description: Test CutOut where Python is implemented without RandomErasing and Cpp is implemented with num_patches
Expectation: Both outputs are equal to each other
"""
logger.info("test_cut_out")
@ -128,7 +132,9 @@ def test_cut_out_op_multicut(plot=False):
def test_cut_out_md5():
"""
Test CutOut with md5 check
Feature: CutOut op
Description: Test CutOut with md5 comparison check
Expectation: Passes the md5 check test
"""
logger.info("test_cut_out_md5")
original_seed = config_get_set_seed(2)
@ -165,7 +171,7 @@ def test_cut_out_comp_hwc(plot=False):
"""
Feature: CutOut op
Description: Test CutOut with HWC input, Decode(to_pil=True) & ToTensor versus Decode(to_pil=False) comparison
Expectation: Test succeeds. Manual confirmation of logged info. Manual visualization confirmation.
Expectation: Test succeeds. Manual confirmation of logged info. Manual visualization confirmation
"""
logger.info("test_cut_out_comp")
@ -214,8 +220,8 @@ def test_cut_out_comp_chw():
"""
Feature: CutOut op
Description: Test CutOut with CHW input, Decode(to_pil=True) & ToTensor versus Decode(to_pil=False) & HWC2CHW
comparison.
Expectation: Test succeeds. Manual confirmation of logged info.
comparison
Expectation: Test succeeds. Manual confirmation of logged info
"""
logger.info("test_cut_out_comp_chw")

66
tests/ut/python/dataset/test_cutmix_batch_op.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -34,7 +34,9 @@ GENERATE_GOLDEN = False
def test_cutmix_batch_success1(plot=False):
"""
Test CutMixBatch op with specified alpha and prob parameters on a batch of CHW images
Feature: CutMixBatch op
Description: Test CutMixBatch op with specified alpha and prob parameters on a batch of CHW images
Expectation: Output is equal to the expected output
"""
logger.info("test_cutmix_batch_success1")
# Original Images
@ -76,7 +78,9 @@ def test_cutmix_batch_success1(plot=False):
def test_cutmix_batch_success2(plot=False):
"""
Test CutMixBatch op with default values for alpha and prob on a batch of rescaled HWC images
Feature: CutMixBatch op
Description: Test CutMixBatch op with default values for alpha and prob on a batch of rescaled HWC images
Expectation: Output is equal to the expected output
"""
logger.info("test_cutmix_batch_success2")
@ -119,7 +123,9 @@ def test_cutmix_batch_success2(plot=False):
def test_cutmix_batch_success3(plot=False):
"""
Test CutMixBatch op with default values for alpha and prob on a batch of HWC images on ImageFolderDataset
Feature: CutMixBatch op
Description: Test with default values for alpha and prob on a batch of HWC images on ImagesFolderDataset
Expectation: Output is equal to the expected output
"""
logger.info("test_cutmix_batch_success3")
@ -171,7 +177,9 @@ def test_cutmix_batch_success3(plot=False):
def test_cutmix_batch_success4(plot=False):
"""
Test CutMixBatch on a dataset where OneHot returns a 2D vector
Feature: CutMixBatch op
Description: Test CutMixBatch op on a dataset where OneHot returns a 2D vector
Expectation: Output is equal to the expected output
"""
logger.info("test_cutmix_batch_success4")
@ -223,7 +231,9 @@ def test_cutmix_batch_success4(plot=False):
def test_cutmix_batch_nhwc_md5():
"""
Test CutMixBatch on a batch of HWC images with MD5:
Feature: CutMixBatch op
Description: Test CutMixBatch op on a batch of HWC images with md5 comparison check
Expectation: Passes the md5 check test
"""
logger.info("test_cutmix_batch_nhwc_md5")
original_seed = config_get_set_seed(0)
@ -248,7 +258,9 @@ def test_cutmix_batch_nhwc_md5():
def test_cutmix_batch_nchw_md5():
"""
Test CutMixBatch on a batch of CHW images with MD5:
Feature: CutMixBatch op
Description: Test CutMixBatch op on a batch of CHW images with md5 comparison check
Expectation: Passes the md5 check test
"""
logger.info("test_cutmix_batch_nchw_md5")
original_seed = config_get_set_seed(0)
@ -274,8 +286,9 @@ def test_cutmix_batch_nchw_md5():
def test_cutmix_batch_fail1():
"""
Test CutMixBatch Fail 1
We expect this to fail because the images and labels are not batched
Feature: CutMixBatch op
Description: Test CutMixBatch op where images and labels are not batched
Expectation: Correct error is raised as expected
"""
logger.info("test_cutmix_batch_fail1")
@ -298,8 +311,9 @@ def test_cutmix_batch_fail1():
def test_cutmix_batch_fail2():
"""
Test CutMixBatch Fail 2
We expect this to fail because alpha is negative
Feature: CutMixBatch op
Description: Test CutMixBatch op where alpha is negative
Expectation: Correct error is raised as expected
"""
logger.info("test_cutmix_batch_fail2")
@ -316,8 +330,9 @@ def test_cutmix_batch_fail2():
def test_cutmix_batch_fail3():
"""
Test CutMixBatch Fail 2
We expect this to fail because prob is larger than 1
Feature: CutMixBatch op
Description: Test CutMixBatch op where prob is larger than 1
Expectation: Correct error is raised as expected
"""
logger.info("test_cutmix_batch_fail3")
@ -334,8 +349,9 @@ def test_cutmix_batch_fail3():
def test_cutmix_batch_fail4():
"""
Test CutMixBatch Fail 2
We expect this to fail because prob is negative
Feature: CutMixBatch op
Description: Test CutMixBatch op where prob is negative
Expectation: Correct error is raised as expected
"""
logger.info("test_cutmix_batch_fail4")
@ -352,8 +368,9 @@ def test_cutmix_batch_fail4():
def test_cutmix_batch_fail5():
"""
Test CutMixBatch op
We expect this to fail because label column is not passed to cutmix_batch
Feature: CutMixBatch op
Description: Test CutMixBatch op where label column is not passed to cutmix_batch
Expectation: Correct error is raised as expected
"""
logger.info("test_cutmix_batch_fail5")
@ -379,8 +396,9 @@ def test_cutmix_batch_fail5():
def test_cutmix_batch_fail6():
"""
Test CutMixBatch op
We expect this to fail because image_batch_format passed to CutMixBatch doesn't match the format of the images
Feature: CutMixBatch op
Description: Test op where image_batch_format passed to CutMixBatch doesn't match the format of the images
Expectation: Correct error is raised as expected
"""
logger.info("test_cutmix_batch_fail6")
@ -406,8 +424,9 @@ def test_cutmix_batch_fail6():
def test_cutmix_batch_fail7():
"""
Test CutMixBatch op
We expect this to fail because labels are not in one-hot format
Feature: CutMixBatch op
Description: Test CutMixBatch op where labels are not in one-hot format
Expectation: Correct error is raised as expected
"""
logger.info("test_cutmix_batch_fail7")
@ -431,8 +450,9 @@ def test_cutmix_batch_fail7():
def test_cutmix_batch_fail8():
"""
Test CutMixBatch Fail 8
We expect this to fail because alpha is zero
Feature: CutMixBatch op
Description: Test CutMixBatch op where alpha is 0
Expectation: Correct error is raised as expected
"""
logger.info("test_cutmix_batch_fail8")

23
tests/ut/python/dataset/test_deviceop_cpu.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -26,7 +26,9 @@ TF_SCHEMA_FILE = "../data/dataset/testTFTestAllTypes/datasetSchema.json"
def test_case_0():
"""
Test Repeat
Feature: Device_que op
Description: Test device_que op after a repeat op
Expectation: Runs successfully
"""
# apply dataset operations
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
@ -43,7 +45,9 @@ def test_case_0():
def test_case_1():
"""
Test Batch
Feature: Device_que op
Description: Test device_que op after a batch op
Expectation: Runs successfully
"""
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
# define data augmentation parameters
@ -68,7 +72,9 @@ def test_case_1():
def test_case_2():
"""
Test Batch & Repeat
Feature: Device_que op
Description: Test device_que op after a batch op then a repeat op
Expectation: Runs successfully
"""
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
# define data augmentation parameters
@ -96,7 +102,9 @@ def test_case_2():
def test_case_3():
"""
Test Repeat & Batch
Feature: Device_que op
Description: Test device_que op after a repeat op then a batch op
Expectation: Runs successfully
"""
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
# define data augmentation parameters
@ -122,6 +130,11 @@ def test_case_3():
def test_case_tf_file():
"""
Feature: Device_que op
Description: Test device_que op using TFRecordDataset
Expectation: Runs successfully
"""
data = ds.TFRecordDataset(TF_FILES, TF_SCHEMA_FILE, shuffle=ds.Shuffle.FILES)
data = data.to_device()

8
tests/ut/python/dataset/test_five_crop.py
View File

@ -31,7 +31,9 @@ GENERATE_GOLDEN = False
def test_five_crop_op(plot=False):
"""
Test FiveCrop
Feature: FiveCrop op
Description: Test FiveCrop op basic usage
Expectation: Output is the same as expected output
"""
logger.info("test_five_crop")
@ -101,7 +103,9 @@ def test_five_crop_error_msg():
def test_five_crop_md5():
"""
Test FiveCrop with md5 check
Feature: FiveCrop op
Description: Test FiveCrop op with md5 check
Expectation: Passes the md5 check test
"""
logger.info("test_five_crop_md5")

14
tests/ut/python/dataset/test_gaussian_blur.py
View File

@ -1,4 +1,4 @@
# Copyright 2021 Huawei Technologies Co., Ltd
# Copyright 2021-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -30,7 +30,9 @@ IMAGE_FILE = "../data/dataset/apple.jpg"
def test_gaussian_blur_pipeline(plot=False):
"""
Test GaussianBlur of C implementation
Feature: GaussianBlur
Description: Test GaussianBlur of Cpp implementation
Expectation: Output is the same as expected output
"""
logger.info("test_gaussian_blur_pipeline")
@ -63,7 +65,9 @@ def test_gaussian_blur_pipeline(plot=False):
def test_gaussian_blur_eager():
"""
Test GaussianBlur with eager mode
Feature: GaussianBlur
Description: Test GaussianBlur in eager mode
Expectation: Output is the same as expected output
"""
logger.info("test_gaussian_blur_eager")
img = cv2.imread(IMAGE_FILE)
@ -76,7 +80,9 @@ def test_gaussian_blur_eager():
def test_gaussian_blur_exception():
"""
Test GaussianBlur with invalid parameters
Feature: GaussianBlur
Description: Test GaussianBlur with invalid parameters
Expectation: Error is raised as expected
"""
logger.info("test_gaussian_blur_exception")
try:

16
tests/ut/python/dataset/test_map_offload.py
View File

@ -27,7 +27,7 @@ BATCH_SIZE = 2
def test_offload():
"""
Feature: test map offload flag.
Feature: Test map offload flag.
Description: Input is image dataset.
Expectation: Output should be same with activated or deactivated offload.
"""
@ -159,7 +159,7 @@ def test_offload_column_mapping():
def test_offload_concat_dataset_1():
"""
Feature: test map offload flag for concatenated dataset.
Feature: Test map offload flag for concatenated dataset.
Description: Input is image dataset.
Expectation: Should raise RuntimeError.
"""
@ -185,7 +185,7 @@ def test_offload_concat_dataset_1():
def test_offload_concat_dataset_2():
"""
Feature: test map offload flag for concatenated dataset.
Feature: Test map offload flag for concatenated dataset.
Description: Input is image dataset.
Expectation: Should raise RuntimeError.
"""
@ -210,7 +210,7 @@ def test_offload_concat_dataset_2():
def test_offload_normalize_op():
"""
Feature: test map offload Normalize op.
Feature: Test map offload Normalize op.
Description: Input is image dataset.
Expectation: Output should be same with activated or deactivated offload for Normalize op.
"""
@ -239,7 +239,7 @@ def test_offload_normalize_op():
def test_offload_rescale_op():
"""
Feature: test map offload Rescale op.
Feature: Test map offload Rescale op.
Description: Input is image dataset.
Expectation: Output should be same with activated or deactivated offload for Rescale op.
"""
@ -268,7 +268,7 @@ def test_offload_rescale_op():
def test_offload_typecast_op():
"""
Feature: test map offload TypeCast op.
Feature: Test map offload TypeCast op.
Description: Input is image dataset.
Expectation: Output should be the same with activated or deactivated offload for TypeCast op.
"""
@ -352,7 +352,7 @@ def test_offload_not_end_of_pipeline():
def test_offload_dim_check():
"""
Feature: test input has the required number of dimensions for offload operation.
Feature: Test input has the required number of dimensions for offload operation.
Description: Input is image dataset.
Expectation: Should raise ValueError.
"""
@ -369,7 +369,7 @@ def test_offload_dim_check():
def test_offload_random_sharpness_op():
"""
Feature: test map offload RandomSharpness op.
Feature: Test map offload RandomSharpness op.
Description: Input is image dataset.
Expectation: Output should be same with activated or deactivated offload for RandomSharpness op.
"""

14
tests/ut/python/dataset/test_minddataset_multi_images.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -23,7 +23,11 @@ CV_FILE_NAME = "../data/mindrecord/testTwoImageData/twobytes.mindrecord"
def test_cv_minddataset_reader_two_png_tutorial():
"""tutorial for cv minderdataset."""
"""
Feature: MindDataset
Description: Test for CV MindDataset basic reader with two pngs tutorial
Expectation: Runs successfully
"""
columns_list = ["id", "file_name", "label_name", "img_data", "label_data"]
num_readers = 1
data_set = ds.MindDataset(CV_FILE_NAME, columns_list, num_readers)
@ -44,7 +48,11 @@ def test_cv_minddataset_reader_two_png_tutorial():
def test_cv_minddataset_reader_two_png_tutorial_just_image2():
"""tutorial for cv minderdataset."""
"""
Feature: MindDataset
Description: Test for CV MindDataset basic reader with two pngs tutorial but only using image and label data
Expectation: Runs successfully
"""
columns_list = ["img_data", "label_data"]
num_readers = 1
data_set = ds.MindDataset(CV_FILE_NAME, columns_list, num_readers)

7
tests/ut/python/dataset/test_minddataset_multi_images_and_ndarray.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -26,6 +26,11 @@ FILES_NUM = 1
def test_cv_minddataset_reader_multi_image_and_ndarray_tutorial():
"""
Feature: MindDataset
Description: Test for MindDataset reader for multiple images and ndarray tutorial
Expectation: Runs successfully
"""
try:
file_name = os.environ.get('PYTEST_CURRENT_TEST').split(':')[-1].split(' ')[0]
writer = FileWriter(file_name, FILES_NUM)

49
tests/ut/python/dataset/test_mixup_op.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -33,7 +33,9 @@ GENERATE_GOLDEN = False
def test_mixup_batch_success1(plot=False):
"""
Test MixUpBatch op with specified alpha parameter
Feature: MixUpBatch op
Description: Test MixUpBatch op with specified alpha parameter
Expectation: Output is the same as expected output
"""
logger.info("test_mixup_batch_success1")
@ -75,7 +77,9 @@ def test_mixup_batch_success1(plot=False):
def test_mixup_batch_success2(plot=False):
"""
Test MixUpBatch op with specified alpha parameter on ImageFolderDataset
Feature: MixUpBatch op
Description: Test MixUpBatch op with specified alpha parameter on ImageFolderDataset
Expectation: Output is the same as expected output
"""
logger.info("test_mixup_batch_success2")
@ -123,8 +127,9 @@ def test_mixup_batch_success2(plot=False):
def test_mixup_batch_success3(plot=False):
"""
Test MixUpBatch op without specified alpha parameter.
Alpha parameter will be selected by default in this case
Feature: MixUpBatch op
Description: Test MixUpBatch op without specified alpha parameter (alpha parameter will be selected by default)
Expectation: Output is the same as expected output
"""
logger.info("test_mixup_batch_success3")
@ -166,8 +171,9 @@ def test_mixup_batch_success3(plot=False):
def test_mixup_batch_success4(plot=False):
"""
Test MixUpBatch op on a dataset where OneHot returns a 2D vector.
Alpha parameter will be selected by default in this case
Feature: MixUpBatch op
Description: Test MixUpBatch op on dataset where OneHot returns a 2D vector (alpha parameter selected by default)
Expectation: Output is the same as expected output
"""
logger.info("test_mixup_batch_success4")
@ -215,7 +221,9 @@ def test_mixup_batch_success4(plot=False):
def test_mixup_batch_md5():
"""
Test MixUpBatch with MD5:
Feature: MixUpBatch op
Description: Test MixUpBatch op with md5
Expectation: Passes the md5 check test
"""
logger.info("test_mixup_batch_md5")
original_seed = config_get_set_seed(0)
@ -240,8 +248,9 @@ def test_mixup_batch_md5():
def test_mixup_batch_fail1():
"""
Test MixUpBatch Fail 1
We expect this to fail because the images and labels are not batched
Feature: MixUpBatch op
Description: Test MixUpBatch op with images and labels that are not batched
Expectation: Error is raised as expected
"""
logger.info("test_mixup_batch_fail1")
@ -275,8 +284,9 @@ def test_mixup_batch_fail1():
def test_mixup_batch_fail2():
"""
Test MixUpBatch Fail 2
We expect this to fail because alpha is negative
Feature: MixUpBatch op
Description: Test MixUpBatch op with negative alpha parameter
Expectation: Error is raised as expected
"""
logger.info("test_mixup_batch_fail2")
@ -304,8 +314,9 @@ def test_mixup_batch_fail2():
def test_mixup_batch_fail3():
"""
Test MixUpBatch op
We expect this to fail because label column is not passed to mixup_batch
Feature: MixUpBatch op
Description: Test MixUpBatch op where label column is not passed
Expectation: Error is raised as expected
"""
logger.info("test_mixup_batch_fail3")
# Original Images
@ -341,8 +352,9 @@ def test_mixup_batch_fail3():
def test_mixup_batch_fail4():
"""
Test MixUpBatch Fail 2
We expect this to fail because alpha is zero
Feature: MixUpBatch op
Description: Test MixUpBatch op where alpha parameter is 0
Expectation: Error is raised as expected
"""
logger.info("test_mixup_batch_fail4")
@ -370,8 +382,9 @@ def test_mixup_batch_fail4():
def test_mixup_batch_fail5():
"""
Test MixUpBatch Fail 5
We expect this to fail because labels are not OntHot encoded
Feature: MixUpBatch op
Description: Test MixUpBatch op where labels are not OneHot encoded
Expectation: Error is raised as expected
"""
logger.info("test_mixup_batch_fail5")

14
tests/ut/python/dataset/test_mu_law_decoding.py
View File

@ -1,4 +1,4 @@
# Copyright 2021 Huawei Technologies Co., Ltd
# Copyright 2021-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -25,7 +25,9 @@ from mindspore import log as logger
def test_mu_law_decoding():
"""
Test mu_law_decoding_op (pipeline).
Feature: MuLawDecoding op
Description: Test MuLawDecoding op in pipeline mode
Expectation: Output is the same as expected output
"""
logger.info("Test MuLawDecoding.")
@ -47,7 +49,9 @@ def test_mu_law_decoding():
def test_mu_law_decoding_eager():
"""
Test mu_law_decoding_op callable (eager).
Feature: MuLawDecoding op
Description: Test MuLawDecoding op in eager mode
Expectation: Output is the same as expected output
"""
logger.info("Test MuLawDecoding callable.")
@ -62,7 +66,9 @@ def test_mu_law_decoding_eager():
def test_mu_law_decoding_uncallable():
"""
Test mu_law_decoding_op not callable.
Feature: MuLawDecoding op
Description: Test parameter check of MuLawDecoding op
Expectation: Error is raised as expected
"""
logger.info("Test MuLawDecoding not callable.")

24
tests/ut/python/dataset/test_ngram_op.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -22,7 +22,9 @@ import mindspore.dataset.text as text
def test_ngram_callable():
"""
Test ngram op is callable
Feature: Ngram op
Description: Test Ngram op basic usage with valid input
Expectation: Output is the same as expected output
"""
op = text.Ngram(2, separator="-")
@ -39,7 +41,11 @@ def test_ngram_callable():
def test_multiple_ngrams():
""" test n-gram where n is a list of integers"""
"""
Feature: Ngram op
Description: Test Ngram op where n is a list of integers
Expectation: Output is the same as expected output
"""
plates_mottos = ["WildRose Country", "Canada's Ocean Playground", "Land of Living Skies"]
n_gram_mottos = []
n_gram_mottos.append(
@ -66,7 +72,11 @@ def test_multiple_ngrams():
def test_simple_ngram():
""" test simple gram with only one n value"""
"""
Feature: Ngram op
Description: Test Ngram op with only one n value
Expectation: Output is the same as expected output
"""
plates_mottos = ["Friendly Manitoba", "Yours to Discover", "Land of Living Skies",
"Birthplace of the Confederation"]
n_gram_mottos = [[""]]
@ -88,7 +98,11 @@ def test_simple_ngram():
def test_corner_cases():
""" testing various corner cases and exceptions"""
"""
Feature: Ngram op
Description: Test Ngram op with various corner cases and exceptions
Expectation: Output is the same as expected output or error is raised when appropriate
"""
def test_config(input_line, n, l_pad=("", 0), r_pad=("", 0), sep=" "):
def gen(texts):

42
tests/ut/python/dataset/test_normalize_op.py
View File

@ -162,7 +162,9 @@ def test_normalize_op_chw(plot=False):
def test_decode_op():
"""
Test Decode op
Feature: Decode op
Description: Test Decode op basic usage
Expectation: Dataset pipeline runs successfully and results are verified
"""
logger.info("Test Decode")
@ -184,7 +186,9 @@ def test_decode_op():
def test_decode_normalize_op():
"""
Test Decode op followed by Normalize op
Feature: Decode op and Normalize op
Description: Test Decode op followed by Normalize op in one map
Expectation: Dataset pipeline runs successfully and results are verified
"""
logger.info("Test [Decode, Normalize] in one Map")
@ -207,8 +211,9 @@ def test_decode_normalize_op():
def test_normalize_md5_01():
"""
Test Normalize with md5 check: valid mean and std
expected to pass
Feature: Normalize Op
Description: Test Normalize op with md5 check with valid mean and std
Expectation: Passes the md5 check test
"""
logger.info("test_normalize_md5_01")
data_c = util_test_normalize([121.0, 115.0, 100.0], [70.0, 68.0, 71.0], False)
@ -223,8 +228,9 @@ def test_normalize_md5_01():
def test_normalize_md5_02():
"""
Test Normalize with md5 check: len(mean)=len(std)=1 with RGB images
expected to pass
Feature: Normalize Op
Description: Test Normalize op with md5 check with len(mean)=len(std)=1 with RBG images
Expectation: Passes the md5 check test
"""
logger.info("test_normalize_md5_02")
data_py = util_test_normalize([0.475], [0.275], True)
@ -312,8 +318,9 @@ def test_normalize_exception_invalid_range():
def test_normalize_grayscale_md5_01():
"""
Test Normalize with md5 check: len(mean)=len(std)=1 with 1 channel grayscale images
expected to pass
Feature: Normalize Op
Description: Test Normalize op with md5 check with len(mean)=len(std)=1 with 1 channel grayscale images
Expectation: Passes the md5 check test
"""
logger.info("test_normalize_grayscale_md5_01")
data = util_test_normalize_grayscale(1, [0.5], [0.175])
@ -324,8 +331,9 @@ def test_normalize_grayscale_md5_01():
def test_normalize_grayscale_md5_02():
"""
Test Normalize with md5 check: len(mean)=len(std)=3 with 3 channel grayscale images
expected to pass
Feature: Normalize Op
Description: Test Normalize op with md5 check with len(mean)=len(std)=3 with 3 channels grayscale images
Expectation: Passes the md5 check test
"""
logger.info("test_normalize_grayscale_md5_02")
data = util_test_normalize_grayscale(3, [0.5, 0.5, 0.5], [0.175, 0.235, 0.512])
@ -336,8 +344,9 @@ def test_normalize_grayscale_md5_02():
def test_normalize_grayscale_exception():
"""
Test Normalize: len(mean)=len(std)=3 with 1 channel grayscale images
expected to raise RuntimeError
Feature: Normalize Op
Description: Test Normalize op with md5 check with len(mean)=len(std)=3 with 1 channel grayscale images
Expectation: Error is raised as expected
"""
logger.info("test_normalize_grayscale_exception")
try:
@ -348,6 +357,11 @@ def test_normalize_grayscale_exception():
def test_multiple_channels():
"""
Feature: Normalize Op
Description: Test Normalize op with multiple channels
Expectation: Output is equal to the expected output
"""
logger.info("test_multiple_channels")
def util_test(item, mean, std):
@ -381,7 +395,7 @@ def test_multiple_channels():
def test_normalize_eager_hwc():
"""
Feature: Normalize op
Description: Test eager support for Normalize C implementation with HWC input
Description: Test eager support for Normalize Cpp implementation with HWC input
Expectation: Receive non-None output image from op
"""
img_in = Image.open("../data/dataset/apple.jpg").convert("RGB")
@ -395,7 +409,7 @@ def test_normalize_eager_hwc():
def test_normalize_eager_chw():
"""
Feature: Normalize op
Description: Test eager support for Normalize C implementation with CHW input
Description: Test eager support for Normalize Cpp implementation with CHW input
Expectation: Receive non-None output image from op
"""
img_in = Image.open("../data/dataset/apple.jpg").convert("RGB")

18
tests/ut/python/dataset/test_normalizepad_op.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -165,7 +165,9 @@ def test_normalizepad_op_comp_chw():
def test_decode_normalizepad_op():
"""
Test Decode op followed by NormalizePad op
Feature: NormalizePad op
Description: Test Decode op followed by NormalizePad op
Expectation: Passes the md5 check test
"""
logger.info("Test [Decode, Normalize] in one Map")
@ -188,8 +190,8 @@ def test_decode_normalizepad_op():
def test_normalizepad_exception_unequal_size_1():
"""
Feature: Normalize op
Description: Test Normalize with error input: len(mean) != len(std)
Feature: NormalizePad op
Description: Test NormalizePad with error input: len(mean) != len(std)
Expectation: ValueError raised
"""
logger.info("test_normalizepad_exception_unequal_size_1")
@ -214,8 +216,8 @@ def test_normalizepad_exception_unequal_size_1():
def test_normalizepad_exception_unequal_size_2():
"""
Feature: Normalize op
Description: Test Normalize with error input: len(mean) != len(std)
Feature: NormalizePad op
Description: Test NormalizePad with error input: len(mean) != len(std)
Expectation: ValueError raised
"""
logger.info("test_normalizepad_exception_unequal_size_2")
@ -240,8 +242,8 @@ def test_normalizepad_exception_unequal_size_2():
def test_normalizepad_exception_invalid_range():
"""
Feature: Normalize op
Description: Test Normalize with error input: value is not in range [0,1]
Feature: NormalizePad op
Description: Test NormalizePad with error input: value is not in range [0,1]
Expectation: ValueError raised
"""
logger.info("test_normalizepad_exception_invalid_range")

17
tests/ut/python/dataset/test_num_samples.py
View File

@ -1,4 +1,4 @@
# Copyright 2021 Huawei Technologies Co., Ltd
# Copyright 2021-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -17,6 +17,11 @@ from mindspore import log as logger
def test_num_samples():
"""
Feature: num_samples
Description: Test num_samples parameter in ManifestDataset
Expectation: Output is the same as expected output
"""
manifest_file = "../data/dataset/testManifestData/test5trainimgs.json"
num_samples = 1
# sampler = ds.DistributedSampler(num_shards=1, shard_id=0, shuffle=False, num_samples=3, offset=1)
@ -30,6 +35,11 @@ def test_num_samples():
def test_num_samples_tf():
"""
Feature: num_samples
Description: Test num_samples parameter in TFRecordDataset
Expectation: Output is the same as expected output
"""
logger.info("test_tfrecord_read_all_dataset")
schema_file = "../data/dataset/testTFTestAllTypes/datasetSchemaNoRow.json"
files = ["../data/dataset/testTFTestAllTypes/test.data"]
@ -42,6 +52,11 @@ def test_num_samples_tf():
def test_num_samples_image_folder():
"""
Feature: num_samples
Description: Test num_samples parameter in ImageFolderDataset
Expectation: Output is the same as expected output
"""
data_dir = "../data/dataset/testPK/data"
ds1 = ds.ImageFolderDataset(data_dir, num_samples=2, num_shards=2, shard_id=0)
count = 0

7
tests/ut/python/dataset/test_opt.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -26,8 +26,9 @@ def generator_1d():
def test_case_0():
"""
Test 1D Generator.
Test without explicit kwargs for input args.
Feature: GeneratorDataset
Description: Test 1D Generator without explicit kwargs for input args
Expectation: Output is the same as expected output
"""
original_seed = config_get_set_seed(55)
original_num_parallel_workers = config_get_set_num_parallel_workers(1)

18
tests/ut/python/dataset/test_pad.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -30,7 +30,9 @@ GENERATE_GOLDEN = False
def test_pad_op():
"""
Test Pad op
Feature: Pad op
Description: Test Pad op between Python and Cpp implementation
Expectation: Both outputs are the same as expected
"""
logger.info("test_random_color_jitter_op")
@ -73,7 +75,9 @@ def test_pad_op():
def test_pad_op2():
"""
Test Pad op2
Feature: Pad op
Description: Test Pad op parameter with size 2
Expectation: Output's shape is the same as expected output's shape
"""
logger.info("test padding parameter with size 2")
@ -94,7 +98,9 @@ def test_pad_op2():
def test_pad_grayscale():
"""
Tests that the pad works for grayscale images
Feature: Pad op
Description: Test Pad op for grayscale images
Expectation: Output's shape is the same as expected output
"""
# Note: image.transpose performs channel swap to allow py transforms to
@ -140,7 +146,9 @@ def test_pad_grayscale():
def test_pad_md5():
"""
Test Pad with md5 check
Feature: Pad op
Description: Test Pad op with md5 check
Expectation: Passes the md5 check test
"""
logger.info("test_pad_md5")

6
tests/ut/python/dataset/test_python_tokenizer.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -24,7 +24,9 @@ DATA_FILE = "../data/dataset/testTokenizerData/1.txt"
def test_whitespace_tokenizer_ch():
"""
Test PythonTokenizer
Feature: PythonTokenizer
Description: Test PythonTokenizer using English and Chinese text based on whitespace separator
Expectation: Output is the same as expected output
"""
whitespace_strs = [["Welcome", "to", "Beijing!"],
["北京欢迎您！"],

7
tests/ut/python/dataset/test_rename.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -24,6 +24,11 @@ SCHEMA_DIR_2 = "../data/dataset/testTFBert5Rows2/datasetSchema.json"
def test_rename():
"""
Feature: Rename op
Description: Test rename op followed by repeat
Expectation: Output is the same as expected output
"""
data1 = ds.TFRecordDataset(DATA_DIR_2, SCHEMA_DIR_2, shuffle=False)
data2 = ds.TFRecordDataset(DATA_DIR_2, SCHEMA_DIR_2, shuffle=False)

91
tests/ut/python/dataset/test_repeat.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -33,7 +33,9 @@ GENERATE_GOLDEN = False
def test_tf_repeat_01():
"""
Test a simple repeat operation.
Feature: Repeat op
Description: Test repeat op under simple case
Expectation: Output is the same as expected output
"""
logger.info("Test Simple Repeat")
# define parameters
@ -49,7 +51,9 @@ def test_tf_repeat_01():
def test_tf_repeat_02():
"""
Test Infinite Repeat.
Feature: Repeat op
Description: Test repeat op with infinite count
Expectation: Runs successfully
"""
logger.info("Test Infinite Repeat")
# define parameters
@ -69,7 +73,9 @@ def test_tf_repeat_02():
def test_tf_repeat_03():
"""
Test Repeat then Batch.
Feature: Repeat op
Description: Test repeat op then batch op
Expectation: Output is the same as expected output
"""
logger.info("Test Repeat then Batch")
data1 = ds.TFRecordDataset(DATA_DIR_TF2, SCHEMA_DIR_TF2, shuffle=False)
@ -92,7 +98,9 @@ def test_tf_repeat_03():
def test_tf_repeat_04():
"""
Test a simple repeat operation with column list.
Feature: Repeat op
Description: Test repeat op under simple case with column list
Expectation: Output is the same as expected output
"""
logger.info("Test Simple Repeat Column List")
# define parameters
@ -112,6 +120,11 @@ def generator():
def test_nested_repeat1():
"""
Feature: Repeat op
Description: Test nested repeat with count > 1
Expectation: Output is the same as expected output
"""
logger.info("test_nested_repeat1")
data = ds.GeneratorDataset(generator, ["data"])
data = data.repeat(2)
@ -124,6 +137,11 @@ def test_nested_repeat1():
def test_nested_repeat2():
"""
Feature: Repeat op
Description: Test nested repeat with count = 1
Expectation: Output is the same as expected output
"""
logger.info("test_nested_repeat2")
data = ds.GeneratorDataset(generator, ["data"])
data = data.repeat(1)
@ -136,6 +154,11 @@ def test_nested_repeat2():
def test_nested_repeat3():
"""
Feature: Repeat op
Description: Test nested repeat with first repeat with count = 1 and second repeat with count > 1
Expectation: Output is the same as expected output
"""
logger.info("test_nested_repeat3")
data = ds.GeneratorDataset(generator, ["data"])
data = data.repeat(1)
@ -148,6 +171,11 @@ def test_nested_repeat3():
def test_nested_repeat4():
"""
Feature: Repeat op
Description: Test nested repeat with first repeat with count > 1 and second input with count = 1
Expectation: Output is the same as expected output
"""
logger.info("test_nested_repeat4")
data = ds.GeneratorDataset(generator, ["data"])
data = data.repeat(2)
@ -160,6 +188,11 @@ def test_nested_repeat4():
def test_nested_repeat5():
"""
Feature: Repeat op
Description: Test nested repeat after a batch operation
Expectation: Output is the same as expected output
"""
logger.info("test_nested_repeat5")
data = ds.GeneratorDataset(generator, ["data"])
data = data.batch(3)
@ -173,6 +206,11 @@ def test_nested_repeat5():
def test_nested_repeat6():
"""
Feature: Repeat op
Description: Test nested repeat with batch op (repeat -> batch -> repeat)
Expectation: Output is the same as expected output
"""
logger.info("test_nested_repeat6")
data = ds.GeneratorDataset(generator, ["data"])
data = data.repeat(2)
@ -186,6 +224,11 @@ def test_nested_repeat6():
def test_nested_repeat7():
"""
Feature: Repeat op
Description: Test nested repeat followed by a batch op
Expectation: Output is the same as expected output
"""
logger.info("test_nested_repeat7")
data = ds.GeneratorDataset(generator, ["data"])
data = data.repeat(2)
@ -199,6 +242,11 @@ def test_nested_repeat7():
def test_nested_repeat8():
"""
Feature: Repeat op
Description: Test nested repeat after a batch operation with drop_remainder=False
Expectation: Output is the same as expected output
"""
logger.info("test_nested_repeat8")
data = ds.GeneratorDataset(generator, ["data"])
data = data.batch(2, drop_remainder=False)
@ -215,6 +263,11 @@ def test_nested_repeat8():
def test_nested_repeat9():
"""
Feature: Repeat op
Description: Test nested repeat with first repeat with no count
Expectation: Output is the same as expected output
"""
logger.info("test_nested_repeat9")
data = ds.GeneratorDataset(generator, ["data"])
data = data.repeat()
@ -227,6 +280,11 @@ def test_nested_repeat9():
def test_nested_repeat10():
"""
Feature: Repeat op
Description: Test nested repeat with second repeat with no count
Expectation: Output is the same as expected output
"""
logger.info("test_nested_repeat10")
data = ds.GeneratorDataset(generator, ["data"])
data = data.repeat(3)
@ -239,6 +297,11 @@ def test_nested_repeat10():
def test_nested_repeat11():
"""
Feature: Repeat op
Description: Test nested repeat (4 repeat ops)
Expectation: Output is the same as expected output
"""
logger.info("test_nested_repeat11")
data = ds.GeneratorDataset(generator, ["data"])
data = data.repeat(2)
@ -253,6 +316,11 @@ def test_nested_repeat11():
def test_repeat_count1():
"""
Feature: Repeat op
Description: Test repeat after multiple operations, then followed by batch op
Expectation: Output is the same as expected output
"""
data1 = ds.TFRecordDataset(DATA_DIR_TF2, SCHEMA_DIR_TF2, shuffle=False)
data1_size = data1.get_dataset_size()
logger.info("dataset size is {}".format(data1_size))
@ -276,6 +344,11 @@ def test_repeat_count1():
def test_repeat_count2():
"""
Feature: Repeat op
Description: Test repeat after multiple operations and a batch op
Expectation: Output is the same as expected output
"""
data1 = ds.TFRecordDataset(DATA_DIR_TF2, SCHEMA_DIR_TF2, shuffle=False)
data1_size = data1.get_dataset_size()
logger.info("dataset size is {}".format(data1_size))
@ -300,7 +373,9 @@ def test_repeat_count2():
def test_repeat_count0():
"""
Test Repeat with invalid count 0.
Feature: Repeat op
Description: Test repeat with invalid count = 0
Expectation: Error is raised as expected
"""
logger.info("Test Repeat with invalid count 0")
with pytest.raises(ValueError) as info:
@ -311,7 +386,9 @@ def test_repeat_count0():
def test_repeat_countneg2():
"""
Test Repeat with invalid count -2.
Feature: Repeat op
Description: Test nested repeat with invalid count = -2
Expectation: Error is raised as expected
"""
logger.info("Test Repeat with invalid count -2")
with pytest.raises(ValueError) as info:

10
tests/ut/python/dataset/test_rescale_op.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -54,7 +54,9 @@ def get_rescaled(image_id):
def test_rescale_op(plot=False):
"""
Test rescale
Feature: Rescale op
Description: Test rescale op basic usage
Expectation: Output is the same as expected output
"""
logger.info("Test rescale")
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
@ -84,7 +86,9 @@ def test_rescale_op(plot=False):
def test_rescale_md5():
"""
Test Rescale with md5 comparison
Feature: Rescale op
Description: Test rescale op with md5 check
Expectation: Passes the md5 check test
"""
logger.info("Test Rescale with md5 comparison")

21
tests/ut/python/dataset/test_resize.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -30,6 +30,11 @@ GENERATE_GOLDEN = False
def test_resize_op(plot=False):
"""
Feature: Resize op
Description: Test Resize op basic usage
Expectation: The dataset is processed as expected
"""
def test_resize_op_parameters(test_name, size, plot):
"""
Test resize_op
@ -61,7 +66,9 @@ def test_resize_op(plot=False):
def test_resize_op_ANTIALIAS():
"""
Test resize_op
Feature: Resize op
Description: Test Resize op basic usage where image interpolation mode is Inter.ANTIALIAS
Expectation: The dataset is processed as expected
"""
logger.info("Test resize for ANTIALIAS")
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
@ -79,6 +86,11 @@ def test_resize_op_ANTIALIAS():
logger.info("use Resize by Inter.ANTIALIAS process {} images.".format(num_iter))
def test_resize_md5(plot=False):
"""
Feature: Resize op
Description: Test Resize op using md5 check
Expectation: Passes the md5 check test
"""
def test_resize_md5_parameters(test_name, size, filename, seed, plot):
"""
Test Resize with md5 check
@ -116,6 +128,11 @@ def test_resize_md5(plot=False):
def test_resize_op_invalid_input():
"""
Feature: Resize op
Description: Test Resize op with invalid input
Expectation: Correct error is raised as expected
"""
def test_invalid_input(test_name, size, interpolation, error, error_msg):
logger.info("Test Resize with bad input: {0}".format(test_name))
with pytest.raises(error) as error_info:

19
tests/ut/python/dataset/test_riaa_biquad.py
View File

@ -1,4 +1,4 @@
# Copyright 2021 Huawei Technologies Co., Ltd
# Copyright 2021-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -35,7 +35,11 @@ def count_unequal_element(data_expected, data_me, rtol, atol):
def test_riaa_biquad_eager():
""" mindspore eager mode normal testcase:riaa_biquad op"""
"""
Feature: RiaaBiquad op
Description: Test RiaaBiquad op in eager mode under normal test case
Expectation: Output is equal to the expected output
"""
# Original waveform
waveform = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float64)
# Expect waveform
@ -48,7 +52,11 @@ def test_riaa_biquad_eager():
def test_riaa_biquad_pipeline():
""" mindspore pipeline mode normal testcase:riaa_biquad op"""
"""
Feature: RiaaBiquad op
Description: Test RiaaBiquad op in pipeline mode under normal test case
Expectation: Output is equal to the expected output
"""
# Original waveform
waveform = np.array([[1.47, 4.722, 5.863], [0.492, 0.235, 0.56]], dtype=np.float32)
# Expect waveform
@ -65,6 +73,11 @@ def test_riaa_biquad_pipeline():
def test_riaa_biquad_invalid_parameter():
"""
Feature: RiaaBiquad op
Description: Test RiaaBiquad op with invalid parameter
Expectation: Error is raised as expected
"""
def test_invalid_input(test_name, sample_rate, error, error_msg):
logger.info("Test RiaaBiquad with bad input: {0}".format(test_name))
with pytest.raises(error) as error_info:

18
tests/ut/python/dataset/test_rotate.py
View File

@ -1,4 +1,4 @@
# Copyright 2021 Huawei Technologies Co., Ltd
# Copyright 2021-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -30,7 +30,9 @@ IMAGE_FILE = "../data/dataset/apple.jpg"
def test_rotate_pipeline_with_expanding(plot=False):
"""
Test Rotate of C implementation with expanding
Feature: Rotate
Description: Test Rotate of Cpp implementation in pipeline mode with expanding
Expectation: Output is the same as expected output
"""
logger.info("test_rotate_pipeline_with_expanding")
@ -63,7 +65,9 @@ def test_rotate_pipeline_with_expanding(plot=False):
def test_rotate_pipeline_without_expanding():
"""
Test Rotate of C implementation without expanding
Feature: Rotate
Description: Test Rotate of Cpp implementation in pipeline mode without expanding
Expectation: Output is the same as expected output
"""
logger.info("test_rotate_pipeline_without_expanding")
@ -83,7 +87,9 @@ def test_rotate_pipeline_without_expanding():
def test_rotate_eager():
"""
Test Rotate with eager mode
Feature: Rotate
Description: Test Rotate in eager mode
Expectation: Output is the same as expected output
"""
logger.info("test_rotate_eager")
img = cv2.imread(IMAGE_FILE)
@ -94,7 +100,9 @@ def test_rotate_eager():
def test_rotate_exception():
"""
Test Rotate with invalid parameters
Feature: Rotate
Description: Test Rotate with invalid parameters
Expectation: Correct error is raised as expected
"""
logger.info("test_rotate_exception")
try:

40
tests/ut/python/dataset/test_sampler_chain.py
View File

@ -175,7 +175,9 @@ def test_numpyslices_sampler_chain_multi_add_child():
def test_imagefolder_sampler_chain():
"""
Test ImageFolderDataset sampler chain
Feature: Chained Sampler
Description: ImageFolderDataset with sampler chain; add child sampler with 2 statements
Expectation: Data verified to be correct
"""
logger.info("test_imagefolder_sampler_chain")
@ -200,7 +202,9 @@ def test_imagefolder_sampler_chain():
def test_mnist_sampler_chain():
"""
Test Mnist sampler chain
Feature: Chained Sampler
Description: MnistDataset with sampler chain; add child sampler with 2 statements
Expectation: Data verified to be correct
"""
logger.info("test_mnist_sampler_chain")
@ -226,7 +230,9 @@ def test_mnist_sampler_chain():
def test_manifest_sampler_chain():
"""
Test Manifest sampler chain
Feature: Chained Sampler
Description: ManifestDataset with sampler chain; add child sampler with 2 statements
Expectation: Data verified to be correct
"""
logger.info("test_manifest_sampler_chain")
@ -252,7 +258,9 @@ def test_manifest_sampler_chain():
def test_coco_sampler_chain():
"""
Test Coco sampler chain
Feature: Chained Sampler
Description: CocoDataset with sampler chain; add child sampler with 2 statements
Expectation: Data verified to be correct
"""
logger.info("test_coco_sampler_chain")
@ -280,7 +288,9 @@ def test_coco_sampler_chain():
def test_cifar_sampler_chain():
"""
Test Cifar sampler chain, including nested child sampler
Feature: Chained Sampler
Description: CifarDataset with sampler chain, including nested child sampler
Expectation: Data verified to be correct
"""
logger.info("test_cifar_sampler_chain")
@ -309,7 +319,9 @@ def test_cifar_sampler_chain():
def test_voc_sampler_chain():
"""
Test VOC sampler chain
Feature: Chained Sampler
Description: VOCDataset with sampler chain; add child sampler with 2 statements
Expectation: Data verified to be correct
"""
logger.info("test_voc_sampler_chain")
@ -336,7 +348,9 @@ def test_voc_sampler_chain():
def test_numpyslices_sampler_chain_batch():
"""
Test NumpySlicesDataset sampler chaining, with batch
Feature: Chained Sampler
Description: NumpySlicesDataset with sampler chain with batch
Expectation: Data verified to be correct
"""
logger.info("test_numpyslices_sampler_chain_batch")
@ -367,7 +381,9 @@ def test_numpyslices_sampler_chain_batch():
def test_sampler_chain_errors():
"""
Test error cases for sampler chains
Feature: Chained Sampler
Description: Test error cases for sampler chains
Expectation: Correct error is raised as expected
"""
logger.info("test_sampler_chain_errors")
@ -396,7 +412,9 @@ def test_sampler_chain_errors():
def test_manifest_sampler_chain_repeat():
"""
Test ManifestDataset sampler chain DistributedSampler->SequentialSampler, with repeat
Feature: Chained Sampler
Description: Test ManifestDataset sampler chain DistributedSampler -> SequentialSampler with repeat
Expectation: Data verified to be correct
"""
logger.info("test_manifest_sampler_chain_batch")
manifest_file = "../data/dataset/testManifestData/test5trainimgs.json"
@ -425,7 +443,9 @@ def test_manifest_sampler_chain_repeat():
def test_manifest_sampler_chain_batch_repeat():
"""
Test ManifestDataset sampler chain DistributedSampler->SequentialSampler, with batch then repeat
Feature: Chained Sampler
Description: Test ManifestDataset sampler chain DistributedSampler -> SequentialSampler, with batch then repeat
Expectation: Data verified to be correct
"""
logger.info("test_manifest_sampler_chain_batch_repeat")
manifest_file = "../data/dataset/testManifestData/test5trainimgs.json"

17
tests/ut/python/dataset/test_schema.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -24,11 +24,21 @@ SCHEMA_FILE = "../data/dataset/testTFTestAllTypes/datasetSchema.json"
def test_schema_simple():
"""
Feature: Schema
Description: Test Schema simple case
Expectation: Runs successfully
"""
logger.info("test_schema_simple")
ds.Schema(SCHEMA_FILE)
def test_schema_file_vs_string():
"""
Feature: Schema
Description: Test Schema by comparing file and string
Expectation: Both datasets are equal
"""
logger.info("test_schema_file_vs_string")
schema1 = ds.Schema(SCHEMA_FILE)
@ -44,6 +54,11 @@ def test_schema_file_vs_string():
def test_schema_exception():
"""
Feature: Schema
Description: Test Schema with invalid inputs
Expectation: Correct error is raised as expected
"""
logger.info("test_schema_exception")
with pytest.raises(TypeError) as info:

50
tests/ut/python/dataset/test_shuffle.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -24,7 +24,9 @@ GENERATE_GOLDEN = False
def test_shuffle_01():
"""
Test shuffle: buffer_size < number-of-rows-in-dataset
Feature: Shuffle op
Description: Test shuffle op where buffer_size < number-of-rows-in-dataset
Expectation: Output is equal to the expected output
"""
logger.info("test_shuffle_01")
# define parameters
@ -42,7 +44,9 @@ def test_shuffle_01():
def test_shuffle_02():
"""
Test shuffle: buffer_size = number-of-rows-in-dataset
Feature: Shuffle op
Description: Test shuffle op where buffer_size == number-of-rows-in-dataset
Expectation: Output is equal to the expected output
"""
logger.info("test_shuffle_02")
# define parameters
@ -60,7 +64,9 @@ def test_shuffle_02():
def test_shuffle_03():
"""
Test shuffle: buffer_size=2 (minimum size), number-of-rows-in-dataset > 2
Feature: Shuffle op
Description: Test shuffle op where buffer_size=2 (minimum size) and number-of-rows-in-dataset > 2
Expectation: Output is equal to the expected output
"""
logger.info("test_shuffle_03")
# define parameters
@ -78,7 +84,9 @@ def test_shuffle_03():
def test_shuffle_04():
"""
Test shuffle: buffer_size=2 (minimum size), number-of-rows-in-dataset = 2
Feature: Shuffle op
Description: Test shuffle op where buffer_size=2 (minimum size) and number-of-rows-in-dataset=2
Expectation: Output is equal to the expected output
"""
logger.info("test_shuffle_04")
# define parameters
@ -96,7 +104,9 @@ def test_shuffle_04():
def test_shuffle_05():
"""
Test shuffle: buffer_size > number-of-rows-in-dataset
Feature: Shuffle op
Description: Test shuffle op where buffer_size > number-of-rows-in-dataset
Expectation: Output is equal to the expected output
"""
logger.info("test_shuffle_05")
# define parameters
@ -114,7 +124,9 @@ def test_shuffle_05():
def test_shuffle_06():
"""
Test shuffle: with set seed, both datasets
Feature: Shuffle op
Description: Test shuffle op with set seed for both datasets
Expectation: Output is equal to the expected output
"""
logger.info("test_shuffle_06")
# define parameters
@ -136,7 +148,9 @@ def test_shuffle_06():
def test_shuffle_exception_01():
"""
Test shuffle exception: buffer_size<0
Feature: Shuffle op
Description: Test shuffle op where buffer_size < 0
Expectation: Exception is raised as expected
"""
logger.info("test_shuffle_exception_01")
@ -154,7 +168,9 @@ def test_shuffle_exception_01():
def test_shuffle_exception_02():
"""
Test shuffle exception: buffer_size=0
Feature: Shuffle op
Description: Test shuffle op where buffer_size=0
Expectation: Exception is raised as expected
"""
logger.info("test_shuffle_exception_02")
@ -172,7 +188,9 @@ def test_shuffle_exception_02():
def test_shuffle_exception_03():
"""
Test shuffle exception: buffer_size=1
Feature: Shuffle op
Description: Test shuffle op where buffer_size=1
Expectation: Exception is raised as expected
"""
logger.info("test_shuffle_exception_03")
@ -190,7 +208,9 @@ def test_shuffle_exception_03():
def test_shuffle_exception_05():
"""
Test shuffle exception: Missing mandatory buffer_size input parameter
Feature: Shuffle op
Description: Test shuffle op where mandatory buffer_size input parameter is missing
Expectation: Exception is raised as expected
"""
logger.info("test_shuffle_exception_05")
@ -208,7 +228,9 @@ def test_shuffle_exception_05():
def test_shuffle_exception_06():
"""
Test shuffle exception: buffer_size wrong type, boolean value False
Feature: Shuffle op
Description: Test shuffle op where buffer_size has wrong type, boolean value False
Expectation: Exception is raised as expected
"""
logger.info("test_shuffle_exception_06")
@ -226,7 +248,9 @@ def test_shuffle_exception_06():
def test_shuffle_exception_07():
"""
Test shuffle exception: buffer_size wrong type, boolean value True
Feature: Shuffle op
Description: Test shuffle op where buffer_size has wrong type, boolean value True
Expectation: Exception is raised as expected
"""
logger.info("test_shuffle_exception_07")

66
tests/ut/python/dataset/test_skip.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -25,7 +25,9 @@ SCHEMA_DIR_TF2 = "../data/dataset/test_tf_file_3_images/datasetSchema.json"
def test_tf_skip():
"""
a simple skip operation.
Feature: Skip op
Description: Test simple skip op usage with TFRecordDataset
Expectation: Output is equal to the expected output
"""
data1 = ds.TFRecordDataset(DATA_DIR_TF2, SCHEMA_DIR_TF2, shuffle=False)
@ -51,6 +53,11 @@ def generator_md():
def test_generator_skip():
"""
Feature: Skip op
Description: Test simple skip op usage with GeneratorDataset with num_parallel_workers=4
Expectation: Output is equal to the expected output
"""
ds1 = ds.GeneratorDataset(generator_md, ["data"], num_parallel_workers=4)
# Here ds1 should be [3, 4]
@ -64,6 +71,11 @@ def test_generator_skip():
def test_skip_1():
"""
Feature: Skip op
Description: Test skip op using input count > 0
Expectation: Output is equal to the expected output
"""
ds1 = ds.GeneratorDataset(generator_md, ["data"])
# Here ds1 should be []
@ -76,6 +88,11 @@ def test_skip_1():
def test_skip_2():
"""
Feature: Skip op
Description: Test skip op using input count=0
Expectation: Output is equal to the expected output
"""
ds1 = ds.GeneratorDataset(generator_md, ["data"])
# Here ds1 should be [0, 1, 2, 3, 4]
@ -89,6 +106,11 @@ def test_skip_2():
def test_skip_repeat_1():
"""
Feature: Skip op
Description: Test skip op after a repeat op
Expectation: Output is equal to the expected output
"""
ds1 = ds.GeneratorDataset(generator_md, ["data"])
# Here ds1 should be [0, 1, 2, 3, 4, 0, 1, 2, 3, 4]
@ -105,6 +127,11 @@ def test_skip_repeat_1():
def test_skip_repeat_2():
"""
Feature: Skip op
Description: Test skip op followed by a repeat op
Expectation: Output is equal to the expected output
"""
ds1 = ds.GeneratorDataset(generator_md, ["data"])
# Here ds1 should be [3, 4]
@ -121,6 +148,11 @@ def test_skip_repeat_2():
def test_skip_repeat_3():
"""
Feature: Skip op
Description: Test skip op by applying repeat -> skip -> repeat
Expectation: Output is equal to the expected output
"""
ds1 = ds.GeneratorDataset(generator_md, ["data"])
# Here ds1 should be [0, 1, 2, 3, 4, 0, 1, 2, 3, 4]
@ -140,6 +172,11 @@ def test_skip_repeat_3():
def test_skip_take_1():
"""
Feature: Skip op
Description: Test skip op after applying take op
Expectation: Output is equal to the expected output
"""
ds1 = ds.GeneratorDataset(generator_md, ["data"])
# Here ds1 should be [0, 1, 2, 3]
@ -156,6 +193,11 @@ def test_skip_take_1():
def test_skip_take_2():
"""
Feature: Skip op
Description: Test skip op followed by a take op
Expectation: Output is equal to the expected output
"""
ds1 = ds.GeneratorDataset(generator_md, ["data"])
# Here ds1 should be [2, 3, 4]
@ -177,6 +219,11 @@ def generator_1d():
def test_skip_filter_1():
"""
Feature: Skip op
Description: Test skip op followed by a filter op
Expectation: Output is equal to the expected output
"""
dataset = ds.GeneratorDataset(generator_1d, ['data'])
dataset = dataset.skip(5)
dataset = dataset.filter(predicate=lambda data: data < 11, num_parallel_workers=4)
@ -188,6 +235,11 @@ def test_skip_filter_1():
def test_skip_filter_2():
"""
Feature: Skip op
Description: Test skip op after filter op is applied
Expectation: Output is equal to the expected output
"""
dataset = ds.GeneratorDataset(generator_1d, ['data'])
dataset = dataset.filter(predicate=lambda data: data < 11, num_parallel_workers=4)
dataset = dataset.skip(5)
@ -199,6 +251,11 @@ def test_skip_filter_2():
def test_skip_exception_1():
"""
Feature: Skip op
Description: Test skip op using input count=-1
Expectation: Error is raised as expected
"""
data1 = ds.GeneratorDataset(generator_md, ["data"])
try:
@ -212,6 +269,11 @@ def test_skip_exception_1():
def test_skip_exception_2():
"""
Feature: Skip op
Description: Test skip op using input count=-2
Expectation: Error is raised as expected
"""
ds1 = ds.GeneratorDataset(generator_md, ["data"])
with pytest.raises(ValueError) as e:

27
tests/ut/python/dataset/test_sliding_window.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -23,7 +23,9 @@ import mindspore.dataset.text as text
def test_sliding_window_callable():
"""
Test sliding window op is callable
Feature: SlidingWindow op
Description: Test SlidingWindow op with 1D input, 2D input, and multiple tensors
Expectation: Output is equal to the expected output for 1D, but error is raised for 2D and multiple tensors
"""
op = text.SlidingWindow(2, 0)
@ -45,7 +47,11 @@ def test_sliding_window_callable():
def test_sliding_window_string():
""" test sliding_window with string type"""
"""
Feature: SlidingWindow op
Description: Test SlidingWindow op with string type
Expectation: Output is equal to the expected output
"""
inputs = [["大", "家", "早", "上", "好"]]
expect = np.array([['大', '家'], ['家', '早'], ['早', '上'], ['上', '好']])
@ -63,6 +69,11 @@ def test_sliding_window_string():
def test_sliding_window_number():
"""
Feature: SlidingWindow op
Description: Test SlidingWindow op with int type
Expectation: Output is equal to the expected output
"""
inputs = [1]
expect = np.array([[1]])
@ -77,6 +88,11 @@ def test_sliding_window_number():
def test_sliding_window_big_width():
"""
Feature: SlidingWindow op
Description: Test SlidingWindow op with big width parameter
Expectation: Output is equal to the expected output
"""
inputs = [[1, 2, 3, 4, 5]]
expect = np.array([])
@ -88,6 +104,11 @@ def test_sliding_window_big_width():
def test_sliding_window_exception():
"""
Feature: SlidingWindow op
Description: Test SlidingWindow op with invalid inputs
Expectation: Correct error is raised as expected
"""
try:
_ = text.SlidingWindow(0, 0)
assert False

14
tests/ut/python/dataset/test_soft_dvpp.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -26,7 +26,9 @@ SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json"
def test_soft_dvpp_decode_resize_jpeg(plot=False):
"""
Test SoftDvppDecodeResizeJpeg op
Feature: SoftDvppDecodeResizeJpeg op
Description: Compare image using SoftDvppDecodeResizeJpeg op with a tuple input with [Decode, Resize] op
Expectation: Both outputs are equal to each other
"""
logger.info("test_random_decode_resize_op")
@ -58,7 +60,9 @@ def test_soft_dvpp_decode_resize_jpeg(plot=False):
def test_soft_dvpp_decode_random_crop_resize_jpeg(plot=False):
"""
Test SoftDvppDecodeRandomCropResizeJpeg op
Feature: SoftDvppDecodeResizeJpeg op
Description: Compare image using SoftDvppDecodeResizeJpeg op with SoftDvppDecodeRandomCropResizeJpeg op
Expectation: Both outputs are equal to each other
"""
logger.info("test_random_decode_resize_op")
@ -89,7 +93,9 @@ def test_soft_dvpp_decode_random_crop_resize_jpeg(plot=False):
def test_soft_dvpp_decode_resize_jpeg_supplement(plot=False):
"""
Test SoftDvppDecodeResizeJpeg op
Feature: SoftDvppDecodeResizeJpeg op
Description: Compare image using SoftDvppDecodeResizeJpeg op with an int input with [Decode, Resize] op
Expectation: Both outputs are equal to each other
"""
logger.info("test_random_decode_resize_op")

52
tests/ut/python/dataset/test_sync_wait.py
View File

@ -1,4 +1,4 @@
# Copyright 2019i-2022 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
@ -38,7 +38,9 @@ class Augment:
def test_simple_sync_wait():
"""
Test simple sync wait: test sync in dataset pipeline
Feature: Sync_wait op
Description: Test sync_wait op in dataset pipeline
Expectation: Runs successfully
"""
logger.info("test_simple_sync_wait")
batch_size = 4
@ -58,7 +60,9 @@ def test_simple_sync_wait():
def test_simple_shuffle_sync():
"""
Test simple shuffle sync: test shuffle before sync
Feature: Sync_wait op
Description: Test sync_wait op after shuffle op
Expectation: Runs successfully
"""
logger.info("test_simple_shuffle_sync")
shuffle_size = 4
@ -81,7 +85,9 @@ def test_simple_shuffle_sync():
def test_two_sync():
"""
Test two sync: dataset pipeline with with two sync_operators
Feature: Sync_wait op
Description: Test sync_wait op in dataset pipeline with two sync_wait ops
Expectation: Runs successfully
"""
logger.info("test_two_sync")
batch_size = 6
@ -109,7 +115,9 @@ def test_two_sync():
def test_sync_epoch():
"""
Test sync wait with epochs: test sync with epochs in dataset pipeline
Feature: Sync_wait op
Description: Test sync_wait op with epochs in dataset pipeline
Expectation: Runs successfully
"""
logger.info("test_sync_epoch")
batch_size = 30
@ -132,7 +140,9 @@ def test_sync_epoch():
def test_multiple_iterators():
"""
Test sync wait with multiple iterators: will start multiple
Feature: Sync_wait op
Description: Test sync_wait op with multiple iterators
Expectation: Runs successfully
"""
logger.info("test_sync_epoch")
batch_size = 30
@ -161,7 +171,9 @@ def test_multiple_iterators():
def test_sync_exception_01():
"""
Test sync: with shuffle in sync mode
Feature: Sync_wait op
Description: Test sync_wait op followed by shuffle op
Expectation: Error is raised as expected
"""
logger.info("test_sync_exception_01")
shuffle_size = 4
@ -179,7 +191,9 @@ def test_sync_exception_01():
def test_sync_exception_02():
"""
Test sync: with duplicated condition name
Feature: Sync_wait op
Description: Test sync_wait op with duplicated condition name
Expectation: Error is raised as expected
"""
logger.info("test_sync_exception_02")
@ -197,7 +211,9 @@ def test_sync_exception_02():
def test_sync_exception_03():
"""
Test sync: with wrong batch size
Feature: Sync_wait op
Description: Test sync_wait op with wrong batch size
Expectation: Error is raised as expected
"""
logger.info("test_sync_exception_03")
@ -212,7 +228,9 @@ def test_sync_exception_03():
def test_sync_exception_04():
"""
Test sync: with negative batch size in update
Feature: Sync_wait op
Description: Test sync_wait op with negative batch size in update
Expectation: Error is raised as expected
"""
logger.info("test_sync_exception_04")
@ -233,7 +251,9 @@ def test_sync_exception_04():
def test_sync_exception_05():
"""
Test sync: with wrong condition name in update
Feature: Sync_wait op
Description: Test sync_wait op with wrong condition name in update
Expectation: Error is raised as expected
"""
logger.info("test_sync_exception_05")
@ -253,7 +273,11 @@ def test_sync_exception_05():
def test_simple_sync_wait_empty_condition_name():
""" callback is none, sync_wait and sync_update's condition_name is empty string ('') """
"""
Feature: Sync_wait op
Description: Test where callback is none, and sync_wait and sync_update condition_name is empty string ('')
Expectation: Runs successfully
"""
logger.info("test_simple_sync_wait_empty_condition_name")
batch_size = 10
dataset = ds.GeneratorDataset(gen, column_names=["input"])
@ -272,7 +296,9 @@ def test_simple_sync_wait_empty_condition_name():
def test_sync_exception_06():
"""
Test sync: with string batch size
Feature: Sync_wait op
Description: Test sync_wait op with string batch size
Expectation: Error is raised as expected
"""
logger.info("test_sync_exception_03")

78
tests/ut/python/dataset/test_take.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -38,7 +38,9 @@ def filter_func_ge(data):
def test_take_01():
"""
Test take: origin there are 3 row, and take 1 row, in this case: will not meet eoe and eof
Feature: Take op
Description: Test take op where originally there are 3 rows and take 1 row. In this case, will not meet EOE and EOF
Expectation: Output is equal to the expected output
"""
logger.info("test_take_01")
data1 = ds.GeneratorDataset(generator, ["data"])
@ -55,7 +57,9 @@ def test_take_01():
def test_take_02():
"""
Test take: origin there are 3 row, and take 2 row, in this case: will meet eoe
Feature: Take op
Description: Test take op where originally there are 3 rows and take 2 rows. In this case, will meet EOE
Expectation: Output is equal to the expected output
"""
logger.info("test_take_02")
data1 = ds.GeneratorDataset(generator, ["data"])
@ -72,7 +76,9 @@ def test_take_02():
def test_take_03():
"""
Test take: origin there are 3 row, and take 3 row, in this case: will meet eoe and eof
Feature: Take op
Description: Test take op where originally there are 3 rows and take 3 rows. In this case, will meet EOE and EOF
Expectation: Output is equal to the expected output
"""
logger.info("test_take_03")
data1 = ds.GeneratorDataset(generator, ["data"])
@ -89,7 +95,9 @@ def test_take_03():
def test_take_04():
"""
Test take: origin there are 3 row, and take 4 row, this is more than the total rows
Feature: Take op
Description: Test take op where originally there are 3 rows and take 4 rows (more than the total rows)
Expectation: Output is equal to the expected output
"""
logger.info("test_take_04")
data1 = ds.GeneratorDataset(generator, ["data"])
@ -106,7 +114,9 @@ def test_take_04():
def test_take_05():
"""
Test take: there is no repeat op
Feature: Take op
Description: Test take op where there is no repeat op
Expectation: Output is equal to the expected output
"""
logger.info("test_take_05")
data1 = ds.GeneratorDataset(generator, ["data"])
@ -122,7 +132,9 @@ def test_take_05():
def test_take_06():
"""
Test take: repeat is before take
Feature: Take op
Description: Test take op where repeat op is done before take op
Expectation: Output is equal to the expected output
"""
logger.info("test_take_06")
data1 = ds.GeneratorDataset(generator, ["data"])
@ -139,7 +151,9 @@ def test_take_06():
def test_take_07():
"""
Test take: take is before batch, that mean take(N), N refer to rows num
Feature: Take op
Description: Test take op where take op is before batch op and have take(N) where N refers to rows num
Expectation: Output is equal to the expected output
"""
logger.info("test_take_07")
data1 = ds.GeneratorDataset(generator, ["data"])
@ -151,7 +165,9 @@ def test_take_07():
def test_take_08():
"""
Test take: take is after batch, that mean take(N), N refer to batches num
Feature: Take op
Description: Test take op where take op is after batch op and have take(N) where N refers to batches num
Expectation: Output is equal to the expected output
"""
logger.info("test_take_08")
data1 = ds.GeneratorDataset(generator, ["data"])
@ -163,7 +179,9 @@ def test_take_08():
def test_take_09():
"""
Test take: take count is -1, and read the whole dataset, take after repeat
Feature: Take op
Description: Test take op where take count is -1 and read the the whole dataset, take op is after repeat op
Expectation: Output is equal to the expected output
"""
logger.info("test_take_09")
data1 = ds.GeneratorDataset(generator, ["data"])
@ -180,7 +198,9 @@ def test_take_09():
def test_take_10():
"""
Test take: take count is -1, and read the whole dataset, take before repeat
Feature: Take op
Description: Test take op where take count is -1 and read the the whole dataset, take op is before repeat op
Expectation: Output is equal to the expected output
"""
logger.info("test_take_10")
data1 = ds.GeneratorDataset(generator, ["data"])
@ -197,7 +217,9 @@ def test_take_10():
def test_take_11():
"""
Test take: batch first, then do repeat and take operation
Feature: Take op
Description: Test take op where batch op is first, followed by repeat op, then take op
Expectation: Output is equal to the expected output
"""
logger.info("test_take_11")
data1 = ds.GeneratorDataset(generator, ["data"])
@ -215,7 +237,9 @@ def test_take_11():
def test_take_12():
"""
Test take: take first, then do batch and repeat operation
Feature: Take op
Description: Test take op where take op is first, followed by batch op, then repeat op
Expectation: Output is equal to the expected output
"""
logger.info("test_take_12")
data1 = ds.GeneratorDataset(generator, ["data"])
@ -233,7 +257,9 @@ def test_take_12():
def test_take_13():
"""
Test take: skip first, then do take, batch and repeat operation
Feature: Take op
Description: Test take op where skip op is first, followed by take op, then batch op, finally repeat op
Expectation: Output is equal to the expected output
"""
logger.info("test_take_13")
data1 = ds.GeneratorDataset(generator, ["data"])
@ -252,7 +278,9 @@ def test_take_13():
def test_take_14():
"""
Test take: take first, then do batch, skip and repeat operation
Feature: Take op
Description: Test take op where take op is first, followed by batch op, then skip op, finally repeat op
Expectation: Output is equal to the expected output
"""
logger.info("test_take_14")
data1 = ds.GeneratorDataset(generator, ["data"])
@ -271,7 +299,9 @@ def test_take_14():
def test_take_15():
"""
Test take: large amount data, take a part, then do skip operation
Feature: Take op
Description: Test take op with large amount of data, first take op then skip op
Expectation: Output is equal to the expected output
"""
logger.info("test_take_15")
data1 = ds.GeneratorDataset(generator_10, ["data"])
@ -288,7 +318,9 @@ def test_take_15():
def test_take_16():
"""
Test take: large amount data, skip a part, then do take operation
Feature: Take op
Description: Test take op with large amount of data, first skip op then take op
Expectation: Output is equal to the expected output
"""
logger.info("test_take_16")
data1 = ds.GeneratorDataset(generator_10, ["data"])
@ -305,7 +337,9 @@ def test_take_16():
def test_take_17():
"""
Test take: take first, then do filter operation
Feature: Take op
Description: Test take op with take op first then filter op
Expectation: Output is equal to the expected output
"""
logger.info("test_take_17")
data1 = ds.GeneratorDataset(generator_10, ["data"])
@ -322,7 +356,9 @@ def test_take_17():
def test_take_18():
"""
Test take: take first, then do filter, skip, batch and repeat operation
Feature: Take op
Description: Test take op with take op first, then filter op, skip op, batch op, and repeat op
Expectation: Output is equal to the expected output
"""
logger.info("test_take_18")
data1 = ds.GeneratorDataset(generator_10, ["data"])
@ -343,7 +379,9 @@ def test_take_18():
def test_take_19():
"""
Test take: take is after batch, that mean take(N), N refer to batches num
Feature: Take op
Description: Test take op where take op is after batch op, meaning take(N) where N refers to batches num
Expectation: Error is raised as expected
"""
logger.info("test_take_19")
with pytest.raises(ValueError) as info:

29
tests/ut/python/dataset/test_ten_crop.py
View File

@ -73,7 +73,9 @@ def util_test_ten_crop(crop_size, vertical_flip=False, plot=False):
def test_ten_crop_op_square(plot=False):
"""
Tests TenCrop for a square crop
Feature: TenCrop op
Description: Test TenCrop op for a square crop
Expectation: Output's shape is equal to the expected output's shape
"""
logger.info("test_ten_crop_op_square")
@ -82,7 +84,9 @@ def test_ten_crop_op_square(plot=False):
def test_ten_crop_op_rectangle(plot=False):
"""
Tests TenCrop for a rectangle crop
Feature: TenCrop op
Description: Test TenCrop op for a rectangle crop
Expectation: Output's shape is equal to the expected output's shape
"""
logger.info("test_ten_crop_op_rectangle")
@ -91,7 +95,9 @@ def test_ten_crop_op_rectangle(plot=False):
def test_ten_crop_op_vertical_flip(plot=False):
"""
Tests TenCrop with vertical flip set to True
Feature: TenCrop op
Description: Test TenCrop op with vertical flip set to True
Expectation: Output's shape is equal to the expected output's shape
"""
logger.info("test_ten_crop_op_vertical_flip")
@ -100,8 +106,9 @@ def test_ten_crop_op_vertical_flip(plot=False):
def test_ten_crop_md5():
"""
Tests TenCrops for giving the same results in multiple runs.
Since TenCrop is a deterministic function, we expect it to return the same result for a specific input every time
Feature: TenCrop op
Description: Test TenCrop op for giving the same results in multiple run for a specific input (since deterministic)
Expectation: Passes the md5 check test
"""
logger.info("test_ten_crop_md5")
@ -120,7 +127,9 @@ def test_ten_crop_md5():
def test_ten_crop_list_size_error_msg():
"""
Tests TenCrop error message when the size arg has more than 2 elements
Feature: TenCrop op
Description: Test TenCrop op when size arg has more than 2 elements
Expectation: Error is raised as expected
"""
logger.info("test_ten_crop_list_size_error_msg")
@ -136,7 +145,9 @@ def test_ten_crop_list_size_error_msg():
def test_ten_crop_invalid_size_error_msg():
"""
Tests TenCrop error message when the size arg is not positive
Feature: TenCrop op
Description: Test TenCrop op when size arg is not positive
Expectation: Error is raised as expected
"""
logger.info("test_ten_crop_invalid_size_error_msg")
@ -162,8 +173,8 @@ def test_ten_crop_invalid_size_error_msg():
def test_ten_crop_wrong_img_error_msg():
"""
Feature: TenCrop op
Description: Test TenCrop op when the input image is not in the correct format.
Expectation: Invalid input is detected
Description: Test TenCrop op when the input image is not in the correct format
Expectation: Error is raised as expected
"""
logger.info("test_ten_crop_wrong_img_error_msg")

17
tests/ut/python/dataset/test_Tensor.py → tests/ut/python/dataset/test_tensor.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -18,6 +18,11 @@ import mindspore._c_dataengine as cde
def test_shape():
"""
Feature: TensorShape op
Description: Test TensorShape op basic usage
Expectation: Output is equal to the expected output
"""
x = [2, 3]
s = cde.TensorShape(x)
assert s.as_list() == x
@ -25,6 +30,11 @@ def test_shape():
def test_basic():
"""
Feature: Tensor
Description: Test Tensor basic usage
Expectation: Output is equal to the expected output
"""
x = np.array([1, 2, 3, 4, 5])
n = cde.Tensor(x)
arr = np.array(n, copy=False)
@ -43,6 +53,11 @@ def test_basic():
def test_strides():
"""
Feature: Tensor
Description: Test Tensor basic usage with strides
Expectation: Output is equal to the expected output
"""
x = np.array([[1, 2, 3], [4, 5, 6]])
n1 = cde.Tensor(x[:, 1])
arr = np.array(n1, copy=False)

17
tests/ut/python/dataset/test_tensor_empty.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -18,6 +18,11 @@ import mindspore.dataset as ds
def test_tensor_empty():
"""
Feature: Tensor
Description: Test empty tensor using GeneratorDataset
Expectation: Output is equal to the expected output
"""
def gen():
for _ in range(4):
(yield np.array([], dtype=np.int64), np.array([], dtype='S').reshape([0, 4]), np.array([1],
@ -32,6 +37,11 @@ def test_tensor_empty():
def test_tensor_empty_map():
"""
Feature: Tensor
Description: Test empty tensor using GeneratorDataset and map it using a function op
Expectation: Output is equal to the expected output
"""
def gen():
for _ in range(4):
(yield np.array([], dtype=np.int64), np.array([], dtype='S'), np.array([1], dtype=np.float64))
@ -53,6 +63,11 @@ def test_tensor_empty_map():
def test_tensor_empty_batch():
"""
Feature: Tensor
Description: Test empty tensor using GeneratorDataset and apply batch op
Expectation: Output is equal to the expected output
"""
def gen():
for _ in range(4):
(yield np.array([], dtype=np.int64), np.array([], dtype='S').reshape([0, 4]), np.array([1],

10
tests/ut/python/dataset/test_text_basic_tokenizer.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -120,7 +120,9 @@ def check_basic_tokenizer_with_offsets(first, last, expected_tokens, expected_of
def test_basic_tokenizer_with_offsets():
"""
Test BasicTokenizer
Feature: BasicTokenizer
Description: Test BasicTokenizer by setting with_offsets to True
Expectation: Output is equal to the expected output
"""
for paras in test_paras:
check_basic_tokenizer_with_offsets(**paras)
@ -128,7 +130,9 @@ def test_basic_tokenizer_with_offsets():
def test_basic_tokenizer_default():
"""
Test BasicTokenizer
Feature: BasicTokenizer
Description: Test BasicTokenizer with default parameters
Expectation: Output is equal to the expected output
"""
for paras in test_paras:
check_basic_tokenizer_default(**paras)

14
tests/ut/python/dataset/test_text_bert_tokenizer.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -229,7 +229,9 @@ def check_bert_tokenizer_with_offsets(first, last, expect_str,
def test_bert_tokenizer_default():
"""
Test WordpieceTokenizer when with_offsets=False
Feature: BertTokenizer
Description: Test BertTokenizer by setting with_offsets to False
Expectation: Output is equal to the expected output
"""
for paras in test_paras:
check_bert_tokenizer_default(**paras)
@ -237,7 +239,9 @@ def test_bert_tokenizer_default():
def test_bert_tokenizer_with_offsets():
"""
Test WordpieceTokenizer when with_offsets=True
Feature: BertTokenizer
Description: Test BertTokenizer by setting with_offsets to True
Expectation: Output is equal to the expected output
"""
for paras in test_paras:
check_bert_tokenizer_with_offsets(**paras)
@ -245,7 +249,9 @@ def test_bert_tokenizer_with_offsets():
def test_bert_tokenizer_callable_invalid_input():
"""
Test WordpieceTokenizer in eager mode with invalid input
Feature: BertTokenizer
Description: Test BertTokenizer with invalid input
Expectation: Error is raised as expected
"""
data = {'张三': 18, '王五': 20}
vocab = text.Vocab.from_list(vocab_bert)

209
tests/ut/python/dataset/test_text_jieba_tokenizer.py
View File

@ -1,4 +1,4 @@
# Copyright 2020-2021 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -28,7 +28,9 @@ MP_FILE = "../data/dataset/jiebadict/jieba.dict.utf8"
def test_jieba_callable():
"""
Test jieba tokenizer op is callable
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer op with one tensor and multiple tensors
Expectation: Output is equal to the expected output for one tensor and error is raised for multiple tensors
"""
logger.info("test_jieba_callable")
jieba_op1 = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
@ -49,7 +51,11 @@ def test_jieba_callable():
def test_jieba_1():
"""Test jieba tokenizer with MP mode"""
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer op with MP mode
Expectation: Output is equal to the expected output
"""
data = ds.TextFileDataset(DATA_FILE)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -63,7 +69,11 @@ def test_jieba_1():
def test_jieba_1_1():
"""Test jieba tokenizer with HMM mode"""
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer op with HMM mode
Expectation: Output is equal to the expected output
"""
data = ds.TextFileDataset(DATA_FILE)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.HMM)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -76,7 +86,11 @@ def test_jieba_1_1():
def test_jieba_1_2():
"""Test jieba tokenizer with HMM MIX"""
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer op with HMM MIX
Expectation: Output is equal to the expected output
"""
data = ds.TextFileDataset(DATA_FILE)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MIX)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -89,9 +103,13 @@ def test_jieba_1_2():
def test_jieba_2():
"""Test add_word"""
DATA_FILE4 = "../data/dataset/testJiebaDataset/4.txt"
data = ds.TextFileDataset(DATA_FILE4)
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_word op
Expectation: Output is equal to the expected output
"""
data_file4 = "../data/dataset/testJiebaDataset/4.txt"
data = ds.TextFileDataset(data_file4)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
jieba_op.add_word("男默女泪")
expect = ['男默女泪', '市', '长江大桥']
@ -104,9 +122,13 @@ def test_jieba_2():
def test_jieba_2_1():
"""Test add_word with freq"""
DATA_FILE4 = "../data/dataset/testJiebaDataset/4.txt"
data = ds.TextFileDataset(DATA_FILE4)
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_word op with freq
Expectation: Output is equal to the expected output
"""
data_file4 = "../data/dataset/testJiebaDataset/4.txt"
data = ds.TextFileDataset(data_file4)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
jieba_op.add_word("男默女泪", 10)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -119,7 +141,11 @@ def test_jieba_2_1():
def test_jieba_2_2():
"""Test add_word with invalid None Input"""
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_word with invalid None input
Expectation: Error is raised as expected
"""
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
try:
jieba_op.add_word(None)
@ -128,9 +154,13 @@ def test_jieba_2_2():
def test_jieba_2_3():
"""Test add_word with freq, the value of freq affects the result of segmentation"""
DATA_FILE4 = "../data/dataset/testJiebaDataset/6.txt"
data = ds.TextFileDataset(DATA_FILE4)
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_word op with freq where the value of freq affects the result of segmentation
Expectation: Output is equal to the expected output
"""
data_file4 = "../data/dataset/testJiebaDataset/6.txt"
data = ds.TextFileDataset(data_file4)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
jieba_op.add_word("江大桥", 20000)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -143,12 +173,16 @@ def test_jieba_2_3():
def test_jieba_3():
"""Test add_dict with dict"""
DATA_FILE4 = "../data/dataset/testJiebaDataset/4.txt"
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_dict op with dict
Expectation: Output is equal to the expected output
"""
data_file4 = "../data/dataset/testJiebaDataset/4.txt"
user_dict = {
"男默女泪": 10
}
data = ds.TextFileDataset(DATA_FILE4)
data = ds.TextFileDataset(data_file4)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
jieba_op.add_dict(user_dict)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -161,13 +195,17 @@ def test_jieba_3():
def test_jieba_3_1():
"""Test add_dict with dict"""
DATA_FILE4 = "../data/dataset/testJiebaDataset/4.txt"
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_dict op with dict
Expectation: Output is equal to the expected output
"""
data_file4 = "../data/dataset/testJiebaDataset/4.txt"
user_dict = {
"男默女泪": 10,
"江大桥": 20000
}
data = ds.TextFileDataset(DATA_FILE4)
data = ds.TextFileDataset(data_file4)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
jieba_op.add_dict(user_dict)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -180,12 +218,17 @@ def test_jieba_3_1():
def test_jieba_4():
DATA_FILE4 = "../data/dataset/testJiebaDataset/3.txt"
DICT_FILE = "../data/dataset/testJiebaDataset/user_dict.txt"
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_dict op with valid file path
Expectation: Output is equal to the expected output
"""
data_file4 = "../data/dataset/testJiebaDataset/3.txt"
dict_file = "../data/dataset/testJiebaDataset/user_dict.txt"
data = ds.TextFileDataset(DATA_FILE4)
data = ds.TextFileDataset(data_file4)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
jieba_op.add_dict(DICT_FILE)
jieba_op.add_dict(dict_file)
data = data.map(operations=jieba_op, input_columns=["text"],
num_parallel_workers=1)
expect = ['今天天气', '太好了', '我们', '一起', '去', '外面', '玩吧']
@ -196,20 +239,28 @@ def test_jieba_4():
def test_jieba_4_1():
"""Test add dict with invalid file path"""
DICT_FILE = ""
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_dict op with invalid file path
Expectation: Error is raised as expected
"""
dict_file = ""
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
try:
jieba_op.add_dict(DICT_FILE)
jieba_op.add_dict(dict_file)
except ValueError:
pass
def test_jieba_5():
"""Test add dict with file path"""
DATA_FILE4 = "../data/dataset/testJiebaDataset/6.txt"
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_word op with num_parallel_workers=1
Expectation: Output is equal to the expected output
"""
data_file4 = "../data/dataset/testJiebaDataset/6.txt"
data = ds.TextFileDataset(DATA_FILE4)
data = ds.TextFileDataset(data_file4)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP)
jieba_op.add_word("江大桥", 20000)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -222,7 +273,11 @@ def test_jieba_5():
def test_jieba_with_offsets_1():
"""Test jieba tokenizer with MP mode"""
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer with MP mode and with_offsets=True
Expectation: Output is equal to the expected output
"""
data = ds.TextFileDataset(DATA_FILE)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -244,7 +299,11 @@ def test_jieba_with_offsets_1():
def test_jieba_with_offsets_1_1():
"""Test jieba tokenizer with HMM mode"""
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer with HMM mode and with_offsets=True
Expectation: Output is equal to the expected output
"""
data = ds.TextFileDataset(DATA_FILE)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.HMM, with_offsets=True)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -265,7 +324,11 @@ def test_jieba_with_offsets_1_1():
def test_jieba_with_offsets_1_2():
"""Test jieba tokenizer with HMM MIX"""
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer with HMM MIX mode and with_offsets=True
Expectation: Output is equal to the expected output
"""
data = ds.TextFileDataset(DATA_FILE)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MIX, with_offsets=True)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -286,9 +349,13 @@ def test_jieba_with_offsets_1_2():
def test_jieba_with_offsets_2():
"""Test add_word"""
DATA_FILE4 = "../data/dataset/testJiebaDataset/4.txt"
data = ds.TextFileDataset(DATA_FILE4)
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_word op with with_offsets=True
Expectation: Output is equal to the expected output
"""
data_file4 = "../data/dataset/testJiebaDataset/4.txt"
data = ds.TextFileDataset(data_file4)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
jieba_op.add_word("男默女泪")
expect = ['男默女泪', '市', '长江大桥']
@ -309,9 +376,13 @@ def test_jieba_with_offsets_2():
def test_jieba_with_offsets_2_1():
"""Test add_word with freq"""
DATA_FILE4 = "../data/dataset/testJiebaDataset/4.txt"
data = ds.TextFileDataset(DATA_FILE4)
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_word op with freq and with_offsets=True
Expectation: Output is equal to the expected output
"""
data_file4 = "../data/dataset/testJiebaDataset/4.txt"
data = ds.TextFileDataset(data_file4)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
jieba_op.add_word("男默女泪", 10)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -332,9 +403,13 @@ def test_jieba_with_offsets_2_1():
def test_jieba_with_offsets_2_2():
"""Test add_word with freq, the value of freq affects the result of segmentation"""
DATA_FILE4 = "../data/dataset/testJiebaDataset/6.txt"
data = ds.TextFileDataset(DATA_FILE4)
"""
Feature: JiebaTokenizer op
Description: Test add_word op with freq where freq affects the result of segmentation and with_offsets=True
Expectation: Output is equal to the expected output
"""
data_file4 = "../data/dataset/testJiebaDataset/6.txt"
data = ds.TextFileDataset(data_file4)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
jieba_op.add_word("江大桥", 20000)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -355,12 +430,16 @@ def test_jieba_with_offsets_2_2():
def test_jieba_with_offsets_3():
"""Test add_dict with dict"""
DATA_FILE4 = "../data/dataset/testJiebaDataset/4.txt"
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_dict op with dict and with_offsets=True
Expectation: Output is equal to the expected output
"""
data_file4 = "../data/dataset/testJiebaDataset/4.txt"
user_dict = {
"男默女泪": 10
}
data = ds.TextFileDataset(DATA_FILE4)
data = ds.TextFileDataset(data_file4)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
jieba_op.add_dict(user_dict)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -381,13 +460,17 @@ def test_jieba_with_offsets_3():
def test_jieba_with_offsets_3_1():
"""Test add_dict with dict"""
DATA_FILE4 = "../data/dataset/testJiebaDataset/4.txt"
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_dict op with dict and with_offsets=True
Expectation: Output is equal to the expected output
"""
data_file4 = "../data/dataset/testJiebaDataset/4.txt"
user_dict = {
"男默女泪": 10,
"江大桥": 20000
}
data = ds.TextFileDataset(DATA_FILE4)
data = ds.TextFileDataset(data_file4)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
jieba_op.add_dict(user_dict)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -408,12 +491,17 @@ def test_jieba_with_offsets_3_1():
def test_jieba_with_offsets_4():
DATA_FILE4 = "../data/dataset/testJiebaDataset/3.txt"
DICT_FILE = "../data/dataset/testJiebaDataset/user_dict.txt"
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_dict with valid file path and with_offsets=True
Expectation: Output is equal to the expected output
"""
data_file4 = "../data/dataset/testJiebaDataset/3.txt"
dict_file = "../data/dataset/testJiebaDataset/user_dict.txt"
data = ds.TextFileDataset(DATA_FILE4)
data = ds.TextFileDataset(data_file4)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
jieba_op.add_dict(DICT_FILE)
jieba_op.add_dict(dict_file)
data = data.map(operations=jieba_op, input_columns=["text"],
output_columns=["token", "offsets_start", "offsets_limit"],
column_order=["token", "offsets_start", "offsets_limit"],
@ -432,10 +520,14 @@ def test_jieba_with_offsets_4():
def test_jieba_with_offsets_5():
"""Test add dict with file path"""
DATA_FILE4 = "../data/dataset/testJiebaDataset/6.txt"
"""
Feature: JiebaTokenizer op
Description: Test JiebaTokenizer add_word op with valid input and with_offsets=True
Expectation: Output is equal to the expected output
"""
data_file4 = "../data/dataset/testJiebaDataset/6.txt"
data = ds.TextFileDataset(DATA_FILE4)
data = ds.TextFileDataset(data_file4)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
jieba_op.add_word("江大桥", 20000)
data = data.map(operations=jieba_op, input_columns=["text"],
@ -470,6 +562,11 @@ def pytoken_op(input_data):
def test_jieba_6():
"""
Feature: Pytoken_op
Description: Test pytoken_op on GeneratorDataset
Expectation: Output is equal to the expected output
"""
data = ds.GeneratorDataset(gen, column_names=["text"])
data = data.map(operations=pytoken_op, input_columns=["text"],
num_parallel_workers=1)

54
tests/ut/python/dataset/test_text_tokenizer.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -38,7 +38,9 @@ def split_by_unicode_char(input_strs):
def test_unicode_char_tokenizer_default():
"""
Test UnicodeCharTokenizer
Feature: UnicodeCharTokenizer op
Description: Test UnicodeCharTokenizer op with default parameters
Expectation: Output is equal to the expected output
"""
input_strs = ("Welcome to Beijing!", "北京欢迎您！", "我喜欢English!", " ")
dataset = ds.TextFileDataset(DATA_FILE, shuffle=False)
@ -54,7 +56,9 @@ def test_unicode_char_tokenizer_default():
def test_unicode_char_tokenizer_with_offsets():
"""
Test UnicodeCharTokenizer
Feature: UnicodeCharTokenizer op
Description: Test UnicodeCharTokenizer op with with_offsets=True
Expectation: Output is equal to the expected output
"""
input_strs = ("Welcome to Beijing!", "北京欢迎您！", "我喜欢English!", " ")
dataset = ds.TextFileDataset(DATA_FILE, shuffle=False)
@ -80,7 +84,9 @@ def test_unicode_char_tokenizer_with_offsets():
def test_whitespace_tokenizer_default():
"""
Test WhitespaceTokenizer
Feature: WhitespaceTokenizer op
Description: Test WhitespaceTokenizer op with default parameters
Expectation: Output is equal to the expected output
"""
whitespace_strs = [["Welcome", "to", "Beijing!"],
["北京欢迎您！"],
@ -99,7 +105,9 @@ def test_whitespace_tokenizer_default():
def test_whitespace_tokenizer_with_offsets():
"""
Test WhitespaceTokenizer
Feature: WhitespaceTokenizer op
Description: Test WhitespaceTokenizer op with with_offsets=True
Expectation: Output is equal to the expected output
"""
whitespace_strs = [["Welcome", "to", "Beijing!"],
["北京欢迎您！"],
@ -127,7 +135,9 @@ def test_whitespace_tokenizer_with_offsets():
def test_unicode_script_tokenizer_default():
"""
Test UnicodeScriptTokenizer when para keep_whitespace=False
Feature: UnicodeScriptTokenizer op
Description: Test UnicodeScriptTokenizer op with para keep_whitespace=False
Expectation: Output is equal to the expected output
"""
unicode_script_strs = [["Welcome", "to", "Beijing", "!"],
["北京欢迎您", "！"],
@ -147,7 +157,9 @@ def test_unicode_script_tokenizer_default():
def test_unicode_script_tokenizer_default2():
"""
Test UnicodeScriptTokenizer when para keep_whitespace=True
Feature: UnicodeScriptTokenizer op
Description: Test UnicodeScriptTokenizer op with para keep_whitespace=True
Expectation: Output is equal to the expected output
"""
unicode_script_strs2 = [["Welcome", " ", "to", " ", "Beijing", "!"],
["北京欢迎您", "！"],
@ -166,7 +178,9 @@ def test_unicode_script_tokenizer_default2():
def test_unicode_script_tokenizer_with_offsets():
"""
Test UnicodeScriptTokenizer when para keep_whitespace=False and with_offsets=True
Feature: UnicodeScriptTokenizer op
Description: Test UnicodeScriptTokenizer op with para keep_whitespace=False and with_offsets=True
Expectation: Output is equal to the expected output
"""
unicode_script_strs = [["Welcome", "to", "Beijing", "!"],
["北京欢迎您", "！"],
@ -193,7 +207,9 @@ def test_unicode_script_tokenizer_with_offsets():
def test_unicode_script_tokenizer_with_offsets2():
"""
Test UnicodeScriptTokenizer when para keep_whitespace=True and with_offsets=True
Feature: UnicodeScriptTokenizer op
Description: Test UnicodeScriptTokenizer op with para keep_whitespace=True and with_offsets=True
Expectation: Output is equal to the expected output
"""
unicode_script_strs2 = [["Welcome", " ", "to", " ", "Beijing", "!"],
["北京欢迎您", "！"],
@ -220,7 +236,9 @@ def test_unicode_script_tokenizer_with_offsets2():
def test_case_fold():
"""
Test CaseFold
Feature: CaseFold op
Description: Test CaseFold op basic usage
Expectation: Output is equal to the expected output
"""
expect_strs = ["welcome to beijing!", "北京欢迎您!", "我喜欢english!", " "]
dataset = ds.TextFileDataset(DATA_FILE, shuffle=False)
@ -236,7 +254,9 @@ def test_case_fold():
def test_normalize_utf8():
"""
Test NormalizeUTF8
Feature: NormalizeUTF8 op
Description: Test NormalizeUTF8 op basic usage
Expectation: Output is equal to the expected output
"""
def normalize(normalize_form):
@ -274,7 +294,9 @@ def test_normalize_utf8():
def test_regex_replace():
"""
Test RegexReplace
Feature: RegexReplace op
Description: Test RegexReplace op basic usage
Expectation: Output is equal to the expected output
"""
def regex_replace(first, last, expect_str, pattern, replace):
@ -301,7 +323,9 @@ def test_regex_replace():
def test_regex_tokenizer_default():
"""
Test RegexTokenizer
Feature: RegexTokenizer op
Description: Test RegexTokenizer op with default parameters
Expectation: Output is equal to the expected output
"""
def regex_tokenizer(first, last, expect_str, delim_pattern, keep_delim_pattern):
@ -332,7 +356,9 @@ def test_regex_tokenizer_default():
def test_regex_tokenizer_with_offsets():
"""
Test RegexTokenizer
Feature: RegexTokenizer op
Description: Test RegexTokenizer op with with_offsets=True
Expectation: Output is equal to the expected output
"""
def regex_tokenizer(first, last, expect_str, expected_offsets_start, expected_offsets_limit, delim_pattern,

10
tests/ut/python/dataset/test_text_wordpiece_tokenizer.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -142,7 +142,9 @@ def check_wordpiece_tokenizer_with_offsets(first, last, expect_str, expected_off
def test_wordpiece_tokenizer_default():
"""
Test WordpieceTokenizer
Feature: WordpieceTokenizer
Description: Test WordpieceTokenizer with default parameters
Expectation: Output is equal to the expected output
"""
for paras in test_paras:
check_wordpiece_tokenizer_default(**paras)
@ -150,7 +152,9 @@ def test_wordpiece_tokenizer_default():
def test_wordpiece_tokenizer_with_offsets():
"""
Test WordpieceTokenizer
Feature: WordpieceTokenizer
Description: Test WordpieceTokenizer by setting with_offsets to True
Expectation: Output is equal to the expected output
"""
for paras in test_paras:
check_wordpiece_tokenizer_with_offsets(**paras)

25
tests/ut/python/dataset/test_time_masking.py
View File

@ -1,4 +1,4 @@
# Copyright 2021 Huawei Technologies Co., Ltd
# Copyright 2021-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -54,7 +54,11 @@ def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True):
def test_func_time_masking_eager_random_input():
""" mindspore eager mode normal testcase:time_masking op"""
"""
Feature: TimeMasking op
Description: Test TimeMasking op in eager mode under normal test case
Expectation: Output's shape is the same as expected output's shape
"""
logger.info("test time_masking op")
spectrogram = next(gen((CHANNEL, FREQ, TIME)))[0]
out_put = audio.TimeMasking(False, 3, 1, 10)(spectrogram)
@ -62,7 +66,11 @@ def test_func_time_masking_eager_random_input():
def test_func_time_masking_eager_precision():
""" mindspore eager mode normal testcase:time_masking op"""
"""
Feature: TimeMasking op
Description: Test TimeMasking op in eager mode by comparing precision
Expectation: Output is the same as expected output
"""
logger.info("test time_masking op")
spectrogram = np.array([[[0.17274511, 0.85174704, 0.07162686, -0.45436913],
[-1.045921, -1.8204843, 0.62333095, -0.09532598],
@ -81,7 +89,11 @@ def test_func_time_masking_eager_precision():
def test_func_time_masking_pipeline():
""" mindspore pipeline mode normal testcase:time_masking op"""
"""
Feature: TimeMasking op
Description: Test TimeMasking op in pipeline mode under normal test case
Expectation: Output's shape is the same as expected output's shape
"""
logger.info("test time_masking op, pipeline")
generator = gen([CHANNEL, FREQ, TIME])
@ -96,6 +108,11 @@ def test_func_time_masking_pipeline():
def test_time_masking_invalid_input():
"""
Feature: TimeMasking op
Description: Test TimeMasking op with invalid input
Expectation: Correct error is raised as expected
"""
def test_invalid_param(test_name, iid_masks, time_mask_param, mask_start, error, error_msg):
logger.info("Test TimeMasking with wrong params: {0}".format(test_name))
with pytest.raises(error) as error_info:

18
tests/ut/python/dataset/test_time_stretch.py
View File

@ -1,4 +1,4 @@
# Copyright 2021 Huawei Technologies Co., Ltd
# Copyright 2021-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -53,7 +53,9 @@ def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True):
def test_time_stretch_pipeline():
"""
Test TimeStretch op. Pipeline.
Feature: TimeStretch op
Description: Test TimeStretch op in pipeline mode
Expectation: Output's shape is the same as expected output's shape
"""
logger.info("test TimeStretch op")
generator = gen([CHANNEL_NUM, FREQ, FRAME_NUM, COMPLEX])
@ -69,7 +71,9 @@ def test_time_stretch_pipeline():
def test_time_stretch_pipeline_invalid_param():
"""
Test TimeStretch op. Set invalid param. Pipeline.
Feature: TimeStretch op
Description: Test TimeStretch op in pipeline mode with invalid parameter
Expectation: Error is raised as expected
"""
logger.info("test TimeStretch op with invalid values")
generator = gen([CHANNEL_NUM, FREQ, FRAME_NUM, COMPLEX])
@ -86,7 +90,9 @@ def test_time_stretch_pipeline_invalid_param():
def test_time_stretch_eager():
"""
Test TimeStretch op. Set param. Eager.
Feature: TimeStretch op
Description: Test TimeStretch op in eager mode with customized parameter values
Expectation: Output's shape is the same as expected output's shape
"""
logger.info("test TimeStretch op with customized parameter values")
spectrogram = next(gen([CHANNEL_NUM, FREQ, FRAME_NUM, COMPLEX]))[0]
@ -96,7 +102,9 @@ def test_time_stretch_eager():
def test_percision_time_stretch_eager():
"""
Test TimeStretch op. Compare precision. Eager.
Feature: TimeStretch op
Description: Test TimeStretch op in eager mode by comparing precision
Expectation: Output is the same as expected output
"""
logger.info("test TimeStretch op with default values")
spectrogram = np.array([[[[1.0402449369430542, 0.3807601034641266],

12
tests/ut/python/dataset/test_to_pil.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -29,8 +29,9 @@ SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json"
def test_to_pil_01():
"""
Test ToPIL Op with md5 comparison: input is already PIL image
Expected to pass
Feature: ToPIL op
Description: Test ToPIL op with md5 comparison where input is already PIL image
Expectation: Passes the md5 check test
"""
logger.info("test_to_pil_01")
@ -52,8 +53,9 @@ def test_to_pil_01():
def test_to_pil_02():
"""
Test ToPIL Op with md5 comparison: input is not PIL image
Expected to pass
Feature: ToPIL op
Description: Test ToPIL op with md5 comparison where input is not a PIL image
Expectation: Passes the md5 check test
"""
logger.info("test_to_pil_02")

19
tests/ut/python/dataset/test_treble_biquad.py
View File

@ -1,4 +1,4 @@
# Copyright 2021 Huawei Technologies Co., Ltd
# Copyright 2021-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -31,7 +31,11 @@ def count_unequal_element(data_expected, data_me, rtol, atol):
def test_treble_biquad_eager():
""" mindspore eager mode normal testcase:treble_biquad op"""
"""
Feature: TrebleBiquad op
Description: Test TrebleBiquad op in eager mode under normal test case
Expectation: Output is equal to the expected output
"""
# Original waveform
waveform = np.array([[0.234, 1.873, 0.786], [-2.673, 0.886, 1.666]], dtype=np.float64)
# Expect waveform
@ -43,7 +47,11 @@ def test_treble_biquad_eager():
def test_treble_biquad_pipeline():
""" mindspore pipeline mode normal testcase:treble_biquad op"""
"""
Feature: TrebleBiquad op
Description: Test TrebleBiquad op in pipeline mode under normal test case
Expectation: Output is equal to the expected output
"""
# Original waveform
waveform = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float64)
# Expect waveform
@ -59,6 +67,11 @@ def test_treble_biquad_pipeline():
def test_treble_biquad_invalid_input():
"""
Feature: TrebleBiquad op
Description: Test TrebleBiquad op with invalid input
Expectation: Error is raised as expected
"""
def test_invalid_input(test_name, sample_rate, gain, central_freq, Q, error, error_msg):
logger.info("Test TrebleBiquad with bad input: {0}".format(test_name))
with pytest.raises(error) as error_info:

28
tests/ut/python/dataset/test_uniform_augment.py
View File

@ -28,7 +28,9 @@ DATA_DIR = "../data/dataset/testImageNetData/train/"
def test_uniform_augment_callable(num_ops=2):
"""
Test UniformAugment is callable
Feature: UniformAugment
Description: Test UniformAugment under normal test case
Expectation: Output's shape is the same as expected output's shape
"""
logger.info("test_uniform_augment_callable")
img = np.fromfile("../data/dataset/apple.jpg", dtype=np.uint8)
@ -47,7 +49,9 @@ def test_uniform_augment_callable(num_ops=2):
def test_uniform_augment(plot=False, num_ops=2):
"""
Test UniformAugment
Feature: UniformAugment
Description: Test UniformAugment using Python implementation
Expectation: Output is the same as expected output
"""
logger.info("Test UniformAugment")
@ -111,7 +115,9 @@ def test_uniform_augment(plot=False, num_ops=2):
def test_cpp_uniform_augment(plot=False, num_ops=2):
"""
Test UniformAugment
Feature: UniformAugment
Description: Test UniformAugment using Cpp implementation
Expectation: Output is the same as expected output
"""
logger.info("Test CPP UniformAugment")
@ -170,7 +176,9 @@ def test_cpp_uniform_augment(plot=False, num_ops=2):
def test_cpp_uniform_augment_exception_large_numops(num_ops=6):
"""
Test UniformAugment invalid large number of ops
Feature: UniformAugment
Description: Test UniformAugment using invalid large number of ops
Expectation: Exception is raised as expected
"""
logger.info("Test CPP UniformAugment invalid large num_ops exception")
@ -190,7 +198,9 @@ def test_cpp_uniform_augment_exception_large_numops(num_ops=6):
def test_cpp_uniform_augment_exception_nonpositive_numops(num_ops=0):
"""
Test UniformAugment invalid non-positive number of ops
Feature: UniformAugment
Description: Test UniformAugment using invalid non-positive num_ops
Expectation: Exception is raised as expected
"""
logger.info("Test CPP UniformAugment invalid non-positive num_ops exception")
@ -210,7 +220,9 @@ def test_cpp_uniform_augment_exception_nonpositive_numops(num_ops=0):
def test_cpp_uniform_augment_exception_float_numops(num_ops=2.5):
"""
Test UniformAugment invalid float number of ops
Feature: UniformAugment
Description: Test UniformAugment using invalid float num_ops
Expectation: Exception is raised as expected
"""
logger.info("Test CPP UniformAugment invalid float num_ops exception")
@ -230,7 +242,9 @@ def test_cpp_uniform_augment_exception_float_numops(num_ops=2.5):
def test_cpp_uniform_augment_random_crop_badinput(num_ops=1):
"""
Test UniformAugment with greater crop size
Feature: UniformAugment
Description: Test UniformAugment with greater crop size
Expectation: Exception is raised as expected
"""
logger.info("Test CPP UniformAugment with random_crop bad input")
batch_size = 2

7
tests/ut/python/dataset/test_unique_op.py
View File

@ -1,4 +1,4 @@
# Copyright 2020 Huawei Technologies Co., Ltd
# Copyright 2020-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -32,6 +32,11 @@ def compare(array, res, idx, cnt):
np.testing.assert_array_equal(cnt, d["z"])
def test_duplicate_basics():
"""
Feature: Unique op
Description: Test Unique op basic usage where there are duplicates
Expectation: Output is equal to the expected output
"""
compare([0, 1, 2, 1, 2, 3], np.array([0, 1, 2, 3]),
np.array([0, 1, 2, 1, 2, 3]), np.array([1, 2, 2, 1]))
compare([0.0, 1.0, 2.0, 1.0, 2.0, 3.0], np.array([0.0, 1.0, 2.0, 3.0]),

4
tests/ut/python/dataset/test_vectors.py
View File

@ -176,8 +176,8 @@ def test_vectors_from_file_eager():
def test_vectors_invalid_input():
"""
Feature: Vectors
Description: Test the validate function with invalid parameters.
Expectation:
Description: Test the validate function with invalid parameters
Expectation: Correct error is raised as expected
"""
def test_invalid_input(test_name, file_path, error, error_msg, max_vectors=None,
unk_init=None, lower_case_backup=False, token="ok"):

19
tests/ut/python/dataset/test_vol.py
View File

@ -1,4 +1,4 @@
# Copyright 2021 Huawei Technologies Co., Ltd
# Copyright 2021-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -45,7 +45,11 @@ def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True):
def test_func_vol_eager():
""" mindspore eager mode normal testcase:vol op"""
"""
Feature: Vol op
Description: Test Vol op in eager mode under normal test case
Expectation: Output is equal to the expected output
"""
logger.info("check vol op output")
ndarr_in = np.array([[0.3667, 0.5295, 0.2949, 0.4508, 0.6457, 0.3625, 0.4377, 0.3568],
@ -95,7 +99,11 @@ def test_func_vol_eager():
def test_func_vol_pipeline():
""" mindspore pipeline mode normal testcase:vol op"""
"""
Feature: Vol op
Description: Test Vol op in pipeline mode under normal test case
Expectation: Output is equal to the expected output
"""
logger.info("test vol op with gain_type='power'")
data = np.array([[[0.7012, 0.2500, 0.0108],
@ -140,6 +148,11 @@ def test_func_vol_pipeline():
def test_vol_invalid_input():
"""
Feature: Vol op
Description: Test Vol op with invalid input
Expectation: Error is raised as expected
"""
def test_invalid_input(test_name, gain, gain_type, error, error_msg):
logger.info("Test Vol with invalid input: {0}".format(test_name))
with pytest.raises(error) as error_info:

54
tests/ut/python/dataset/test_zip.py
View File

@ -1,4 +1,4 @@
# Copyright 2019 Huawei Technologies Co., Ltd
# Copyright 2019-2022 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -35,7 +35,9 @@ GENERATE_GOLDEN = False
def test_zip_01():
"""
Test zip: zip 2 datasets, #rows-data1 == #rows-data2, #cols-data1 < #cols-data2
Feature: Zip op
Description: Test zip op with 2 datasets where #rows-data1 == #rows-data2 and #cols-data1 < #cols-data2
Expectation: Output is equal to the expected output
"""
logger.info("test_zip_01")
ds.config.set_seed(1)
@ -49,7 +51,9 @@ def test_zip_01():
def test_zip_02():
"""
Test zip: zip 2 datasets, #rows-data1 < #rows-data2, #cols-data1 == #cols-data2
Feature: Zip op
Description: Test zip op with 2 datasets where #rows-data1 < #rows-data2 and #cols-data1 == #cols-data2
Expectation: Output is equal to the expected output
"""
logger.info("test_zip_02")
ds.config.set_seed(1)
@ -63,7 +67,9 @@ def test_zip_02():
def test_zip_03():
"""
Test zip: zip 2 datasets, #rows-data1 > #rows-data2, #cols-data1 > #cols-data2
Feature: Zip op
Description: Test zip op with 2 datasets where #rows-data1 > #rows-data2 and #cols-data1 > #cols-data2
Expectation: Output is equal to the expected output
"""
logger.info("test_zip_03")
ds.config.set_seed(1)
@ -77,7 +83,9 @@ def test_zip_03():
def test_zip_04():
"""
Test zip: zip >2 datasets
Feature: Zip op
Description: Test zip op with > 2 datasets
Expectation: Output is equal to the expected output
"""
logger.info("test_zip_04")
ds.config.set_seed(1)
@ -92,7 +100,9 @@ def test_zip_04():
def test_zip_05():
"""
Test zip: zip dataset with renamed columns
Feature: Zip op
Description: Test zip op with renamed columns
Expectation: Output is equal to the expected output
"""
logger.info("test_zip_05")
ds.config.set_seed(1)
@ -110,7 +120,9 @@ def test_zip_05():
def test_zip_06():
"""
Test zip: zip dataset with renamed columns and repeat zipped dataset
Feature: Zip op
Description: Test zip op with renamed columns and repeat zipped dataset
Expectation: Output is equal to the expected output
"""
logger.info("test_zip_06")
ds.config.set_seed(1)
@ -129,7 +141,9 @@ def test_zip_06():
def test_zip_exception_01():
"""
Test zip: zip same datasets
Feature: Zip op
Description: Test zip op with same datasets
Expectation: Exception is raised as expected
"""
logger.info("test_zip_exception_01")
data1 = ds.TFRecordDataset(DATA_DIR_1, SCHEMA_DIR_1)
@ -149,7 +163,9 @@ def test_zip_exception_01():
def test_zip_exception_02():
"""
Test zip: zip datasets with duplicate column name
Feature: Zip op
Description: Test zip op with duplicate column names
Expectation: Exception is raised as expected
"""
logger.info("test_zip_exception_02")
data1 = ds.TFRecordDataset(DATA_DIR_1, SCHEMA_DIR_1)
@ -170,7 +186,9 @@ def test_zip_exception_02():
def test_zip_exception_03():
"""
Test zip: zip with tuple of 1 dataset
Feature: Zip op
Description: Test zip op with tuple of 1 dataset
Expectation: Exception is raised as expected
"""
logger.info("test_zip_exception_03")
data1 = ds.TFRecordDataset(DATA_DIR_1, SCHEMA_DIR_1)
@ -191,7 +209,9 @@ def test_zip_exception_03():
def test_zip_exception_04():
"""
Test zip: zip with empty tuple of datasets
Feature: Zip op
Description: Test zip op with empty tuple of datasets
Expectation: Exception is raised as expected
"""
logger.info("test_zip_exception_04")
@ -211,7 +231,9 @@ def test_zip_exception_04():
def test_zip_exception_05():
"""
Test zip: zip with non-tuple of 2 datasets
Feature: Zip op
Description: Test zip op with non-tuple of 2 datasets
Expectation: Exception is raised as expected
"""
logger.info("test_zip_exception_05")
data1 = ds.TFRecordDataset(DATA_DIR_1, SCHEMA_DIR_1)
@ -232,7 +254,9 @@ def test_zip_exception_05():
def test_zip_exception_06():
"""
Test zip: zip with non-tuple of 1 dataset
Feature: Zip op
Description: Test zip op with non-tuple of 1 dataset
Expectation: Exception is raised as expected
"""
logger.info("test_zip_exception_06")
data1 = ds.TFRecordDataset(DATA_DIR_1, SCHEMA_DIR_1)
@ -252,7 +276,9 @@ def test_zip_exception_06():
def test_zip_exception_07():
"""
Test zip: zip with string as parameter
Feature: Zip op
Description: Test zip op with string as parameter
Expectation: Exception is raised as expected
"""
logger.info("test_zip_exception_07")