p60975123
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!21785 Fix Programming Specifications of Crowd Contributing Code 

Merge pull request !21785 from xiaotianci/fix_crowd_contributing
This commit is contained in:
i-robot 2021-08-17 07:53:50 +00:00 committed by Gitee
parent f919a6f78c aae886bcd7
commit debf4d8ece
78 changed files with 876 additions and 1021 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mindspore/ccsrc/minddata/dataset/api/audio.cc
View File

@ -47,7 +47,7 @@ std::shared_ptr<TensorOperation> AllpassBiquad::Parse() {
return std::make_shared<AllpassBiquadOperation>(data_->sample_rate_, data_->central_freq_, data_->Q_);
}
// AmplitudeToDB Operation.
// AmplitudeToDB Transform Operation.
struct AmplitudeToDB::Data {
Data(ScaleType stype, float ref_value, float amin, float top_db)
: stype_(stype), ref_value_(ref_value), amin_(amin), top_db_(top_db) {}
@ -175,7 +175,7 @@ std::shared_ptr<TensorOperation> TimeMasking::Parse() {
data_->mask_value_);
}
// TimeStretch Operation.
// TimeStretch Transform Operation.
struct TimeStretch::Data {
explicit Data(float hop_length, int n_freq, float fixed_rate)
: hop_length_(hop_length), n_freq_(n_freq), fixed_rate_(fixed_rate) {}

3
mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/audio/kernels/ir/bindings.cc
View File

@ -17,6 +17,8 @@
#include "minddata/dataset/api/python/pybind_conversion.h"
#include "minddata/dataset/api/python/pybind_register.h"
#include "minddata/dataset/include/dataset/transforms.h"
#include "minddata/dataset/audio/ir/kernels/allpass_biquad_ir.h"
#include "minddata/dataset/audio/ir/kernels/amplitude_to_db_ir.h"
#include "minddata/dataset/audio/ir/kernels/angle_ir.h"
@ -27,7 +29,6 @@
#include "minddata/dataset/audio/ir/kernels/frequency_masking_ir.h"
#include "minddata/dataset/audio/ir/kernels/time_masking_ir.h"
#include "minddata/dataset/audio/ir/kernels/time_stretch_ir.h"
#include "minddata/dataset/include/dataset/transforms.h"
namespace mindspore {
namespace dataset {

9
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/allpass_biquad_ir.cc
View File

@ -16,20 +16,20 @@
#include "minddata/dataset/audio/ir/kernels/allpass_biquad_ir.h"
#include "minddata/dataset/audio/kernels/allpass_biquad_op.h"
#include "minddata/dataset/audio/ir/validators.h"
#include "minddata/dataset/audio/kernels/allpass_biquad_op.h"
namespace mindspore {
namespace dataset {
namespace audio {
// AllpassBiquadOperation
AllpassBiquadOperation::AllpassBiquadOperation(int32_t sample_rate, float central_freq, float Q)
: sample_rate_(sample_rate), central_freq_(central_freq), Q_(Q) {}
Status AllpassBiquadOperation::ValidateParams() {
RETURN_IF_NOT_OK(CheckScalarNotZero("AllpassBiquad", "sample_rate", sample_rate_));
RETURN_IF_NOT_OK(CheckScalarNotZero("AllpassBiquad", "central_freq", central_freq_));
RETURN_IF_NOT_OK(ValidateScalarNotZero("AllpassBiquad", "sample_rate", sample_rate_));
RETURN_IF_NOT_OK(ValidateScalarNotZero("AllpassBiquad", "central_freq", central_freq_));
RETURN_IF_NOT_OK(ValidateScalar("AllpassBiquad", "Q", Q_, {0, 1.0}, true, false));
return Status::OK();
}
@ -38,6 +38,7 @@ std::shared_ptr<TensorOp> AllpassBiquadOperation::Build() {
std::shared_ptr<AllpassBiquadOp> tensor_op = std::make_shared<AllpassBiquadOp>(sample_rate_, central_freq_, Q_);
return tensor_op;
}
Status AllpassBiquadOperation::to_json(nlohmann::json *out_json) {
nlohmann::json args;
args["sample_rate"] = sample_rate_;

5
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/allpass_biquad_ir.h
View File

@ -20,6 +20,7 @@
#include <memory>
#include <string>
#include <vector>
#include "include/api/status.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/include/dataset/transforms.h"
@ -27,9 +28,8 @@
namespace mindspore {
namespace dataset {
namespace audio {
// Char arrays storing name of corresponding classes (in alphabetical order)
constexpr char kAllpassBiquadOperation[] = "AllpassBiquad";
class AllpassBiquadOperation : public TensorOperation {
@ -52,7 +52,6 @@ class AllpassBiquadOperation : public TensorOperation {
float Q_;
};
} // namespace audio
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_ALLPASS_BIQUAD_IR_H_

10
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/amplitude_to_db_ir.cc
View File

@ -15,15 +15,15 @@
*/
#include "minddata/dataset/audio/ir/kernels/amplitude_to_db_ir.h"
#include "minddata/dataset/audio/kernels/amplitude_to_db_op.h"
#include "minddata/dataset/audio/ir/validators.h"
#include "minddata/dataset/audio/kernels/amplitude_to_db_op.h"
namespace mindspore {
namespace dataset {
namespace audio {
// AmplitudeToDB
// AmplitudeToDBOperation
AmplitudeToDBOperation::AmplitudeToDBOperation(ScaleType stype, float ref_value, float amin, float top_db)
: stype_(stype), ref_value_(ref_value), amin_(amin), top_db_(top_db) {}
@ -32,9 +32,9 @@ AmplitudeToDBOperation::~AmplitudeToDBOperation() = default;
std::string AmplitudeToDBOperation::Name() const { return kAmplitudeToDBOperation; }
Status AmplitudeToDBOperation::ValidateParams() {
RETURN_IF_NOT_OK(CheckFloatScalarNonNegative("AmplitudeToDB", "top_db", top_db_));
RETURN_IF_NOT_OK(CheckFloatScalarPositive("AmplitudeToDB", "amin", amin_));
RETURN_IF_NOT_OK(CheckFloatScalarPositive("AmplitudeToDB", "ref_value", ref_value_));
RETURN_IF_NOT_OK(ValidateFloatScalarNonNegative("AmplitudeToDB", "top_db", top_db_));
RETURN_IF_NOT_OK(ValidateFloatScalarPositive("AmplitudeToDB", "amin", amin_));
RETURN_IF_NOT_OK(ValidateFloatScalarPositive("AmplitudeToDB", "ref_value", ref_value_));
return Status::OK();
}

3
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/angle_ir.cc
View File

@ -16,13 +16,12 @@
#include "minddata/dataset/audio/ir/kernels/angle_ir.h"
// Kernel Audio headers
#include "minddata/dataset/audio/kernels/angle_op.h"
namespace mindspore {
namespace dataset {
namespace audio {
// AngleOperation
AngleOperation::AngleOperation() {}

3
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/angle_ir.h
View File

@ -29,9 +29,8 @@
namespace mindspore {
namespace dataset {
namespace audio {
// Char arrays storing name of corresponding classes
constexpr char kAngleOperation[] = "Angle";
class AngleOperation : public TensorOperation {

5
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/band_biquad_ir.cc
View File

@ -16,9 +16,8 @@
#include "minddata/dataset/audio/ir/kernels/band_biquad_ir.h"
#include "minddata/dataset/audio/kernels/band_biquad_op.h"
#include "minddata/dataset/audio/ir/validators.h"
#include "minddata/dataset/audio/kernels/band_biquad_op.h"
namespace mindspore {
namespace dataset {
@ -30,7 +29,7 @@ BandBiquadOperation::BandBiquadOperation(int32_t sample_rate, float central_freq
Status BandBiquadOperation::ValidateParams() {
RETURN_IF_NOT_OK(ValidateScalar("BandBiquad", "Q", Q_, {0, 1.0}, true, false));
RETURN_IF_NOT_OK(CheckScalarNotZero("BandBIquad", "sample_rate", sample_rate_));
RETURN_IF_NOT_OK(ValidateScalarNotZero("BandBIquad", "sample_rate", sample_rate_));
return Status::OK();
}

2
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/band_biquad_ir.h
View File

@ -21,6 +21,7 @@
#include <string>
#include <utility>
#include <vector>
#include "include/api/status.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/include/dataset/transforms.h"
@ -30,7 +31,6 @@ namespace mindspore {
namespace dataset {
namespace audio {
// Char arrays storing name of corresponding classes (in alphabetical order)
constexpr char kBandBiquadOperation[] = "BandBiquad";
class BandBiquadOperation : public TensorOperation {

7
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandpass_biquad_ir.cc
View File

@ -16,13 +16,13 @@
#include "minddata/dataset/audio/ir/kernels/bandpass_biquad_ir.h"
#include "minddata/dataset/audio/kernels/bandpass_biquad_op.h"
#include "minddata/dataset/audio/ir/validators.h"
#include "minddata/dataset/audio/kernels/bandpass_biquad_op.h"
namespace mindspore {
namespace dataset {
namespace audio {
// BandpassBiquadOperation
BandpassBiquadOperation::BandpassBiquadOperation(int32_t sample_rate, float central_freq, float Q,
bool const_skirt_gain)
@ -30,9 +30,10 @@ BandpassBiquadOperation::BandpassBiquadOperation(int32_t sample_rate, float cent
Status BandpassBiquadOperation::ValidateParams() {
RETURN_IF_NOT_OK(ValidateScalar("BandpassBiquad", "Q", Q_, {0, 1.0}, true, false));
RETURN_IF_NOT_OK(CheckScalarNotZero("BandpassBiquad", "sample_rate", sample_rate_));
RETURN_IF_NOT_OK(ValidateScalarNotZero("BandpassBiquad", "sample_rate", sample_rate_));
return Status::OK();
}
std::shared_ptr<TensorOp> BandpassBiquadOperation::Build() {
std::shared_ptr<BandpassBiquadOp> tensor_op =
std::make_shared<BandpassBiquadOp>(sample_rate_, central_freq_, Q_, const_skirt_gain_);

4
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandpass_biquad_ir.h
View File

@ -21,6 +21,7 @@
#include <string>
#include <utility>
#include <vector>
#include "include/api/status.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/include/dataset/transforms.h"
@ -28,9 +29,8 @@
namespace mindspore {
namespace dataset {
namespace audio {
// Char arrays storing name of corresponding classes (in alphabetical order)
constexpr char kBandpassBiquadOperation[] = "BandpassBiquad";
class BandpassBiquadOperation : public TensorOperation {

6
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandreject_biquad_ir.cc
View File

@ -15,19 +15,21 @@
*/
#include "minddata/dataset/audio/ir/kernels/bandreject_biquad_ir.h"
#include "minddata/dataset/audio/kernels/bandreject_biquad_op.h"
#include "minddata/dataset/audio/ir/validators.h"
#include "minddata/dataset/audio/kernels/bandreject_biquad_op.h"
namespace mindspore {
namespace dataset {
namespace audio {
// BandrejectBiquadOperation
BandrejectBiquadOperation::BandrejectBiquadOperation(int32_t sample_rate, float central_freq, float Q)
: sample_rate_(sample_rate), central_freq_(central_freq), Q_(Q) {}
Status BandrejectBiquadOperation::ValidateParams() {
RETURN_IF_NOT_OK(ValidateScalar("BandrejectBiquad", "Q", Q_, {0, 1.0}, true, false));
RETURN_IF_NOT_OK(CheckScalarNotZero("BandrejectBiquad", "sample_rate", sample_rate_));
RETURN_IF_NOT_OK(ValidateScalarNotZero("BandrejectBiquad", "sample_rate", sample_rate_));
return Status::OK();
}

4
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandreject_biquad_ir.h
View File

@ -16,10 +16,12 @@
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_BANDREJECT_BIQUAD_IR_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_IR_KERNELS_BANDREJECT_BIQUAD_IR_H_
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "include/api/status.h"
#include "minddata/dataset/include/dataset/constants.h"
#include "minddata/dataset/include/dataset/transforms.h"
@ -27,10 +29,8 @@
namespace mindspore {
namespace dataset {
namespace audio {
// Char arrays storing name of corresponding classes (in alphabetical order)
constexpr char kBandrejectBiquadOperation[] = "BandrejectBiquad";
class BandrejectBiquadOperation : public TensorOperation {

6
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bass_biquad_ir.cc
View File

@ -16,9 +16,8 @@
#include "minddata/dataset/audio/ir/kernels/bass_biquad_ir.h"
#include "minddata/dataset/audio/kernels/bass_biquad_op.h"
#include "minddata/dataset/audio/ir/validators.h"
#include "minddata/dataset/audio/kernels/bass_biquad_op.h"
namespace mindspore {
namespace dataset {
@ -30,7 +29,7 @@ BassBiquadOperation::BassBiquadOperation(int32_t sample_rate, float gain, float
Status BassBiquadOperation::ValidateParams() {
RETURN_IF_NOT_OK(ValidateScalar("BassBiquad", "Q", Q_, {0, 1.0}, true, false));
RETURN_IF_NOT_OK(CheckScalarNotZero("BassBiquad", "sample_rate", sample_rate_));
RETURN_IF_NOT_OK(ValidateScalarNotZero("BassBiquad", "sample_rate", sample_rate_));
return Status::OK();
}
@ -38,6 +37,7 @@ std::shared_ptr<TensorOp> BassBiquadOperation::Build() {
std::shared_ptr<BassBiquadOp> tensor_op = std::make_shared<BassBiquadOp>(sample_rate_, gain_, central_freq_, Q_);
return tensor_op;
}
Status BassBiquadOperation::to_json(nlohmann::json *out_json) {
nlohmann::json args;
args["sample_rate"] = sample_rate_;

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bass_biquad_ir.h
View File

@ -31,7 +31,6 @@ namespace mindspore {
namespace dataset {
namespace audio {
// Char arrays storing name of corresponding classes (in alphabetical order)
constexpr char kBassBiquadOperation[] = "BassBiquad";
class BassBiquadOperation : public TensorOperation {

4
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/frequency_masking_ir.cc
View File

@ -33,8 +33,8 @@ FrequencyMaskingOperation::FrequencyMaskingOperation(bool iid_masks, int32_t fre
FrequencyMaskingOperation::~FrequencyMaskingOperation() = default;
Status FrequencyMaskingOperation::ValidateParams() {
RETURN_IF_NOT_OK(CheckIntScalarNonNegative("FrequencyMasking", "frequency_mask_param", frequency_mask_param_));
RETURN_IF_NOT_OK(CheckIntScalarNonNegative("FrequencyMasking", "mask_start", mask_start_));
RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("FrequencyMasking", "frequency_mask_param", frequency_mask_param_));
RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("FrequencyMasking", "mask_start", mask_start_));
return Status::OK();
}

8
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/time_masking_ir.cc
View File

@ -15,12 +15,12 @@
*/
#include "minddata/dataset/audio/ir/kernels/time_masking_ir.h"
#include "minddata/dataset/audio/kernels/time_masking_op.h"
#include "minddata/dataset/audio/ir/validators.h"
#include "minddata/dataset/audio/kernels/time_masking_op.h"
namespace mindspore {
namespace dataset {
namespace audio {
TimeMaskingOperation::TimeMaskingOperation(bool iid_masks, int64_t time_mask_param, int64_t mask_start,
@ -30,8 +30,8 @@ TimeMaskingOperation::TimeMaskingOperation(bool iid_masks, int64_t time_mask_par
TimeMaskingOperation::~TimeMaskingOperation() = default;
Status TimeMaskingOperation::ValidateParams() {
RETURN_IF_NOT_OK(CheckIntScalarNonNegative("TimeMasking", "time_mask_param", time_mask_param_));
RETURN_IF_NOT_OK(CheckIntScalarNonNegative("TimeMasking", "mask_start", mask_start_));
RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("TimeMasking", "time_mask_param", time_mask_param_));
RETURN_IF_NOT_OK(ValidateIntScalarNonNegative("TimeMasking", "mask_start", mask_start_));
return Status::OK();
}

13
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/time_stretch_ir.cc
View File

@ -14,14 +14,15 @@
* limitations under the License.
*/
#include "minddata/dataset/audio/ir/kernels/time_stretch_ir.h"
#include "minddata/dataset/audio/kernels/time_stretch_op.h"
#include "minddata/dataset/audio/ir/validators.h"
#include "minddata/dataset/audio/kernels/time_stretch_op.h"
namespace mindspore {
namespace dataset {
namespace audio {
// TimeStretch
// TimeStretchOperation
TimeStretchOperation::TimeStretchOperation(float hop_length, int n_freq, float fixed_rate)
: hop_length_(hop_length), n_freq_(n_freq), fixed_rate_(fixed_rate) {}
@ -31,10 +32,10 @@ std::string TimeStretchOperation::Name() const { return kTimeStretchOperation; }
Status TimeStretchOperation::ValidateParams() {
// param check
RETURN_IF_NOT_OK(CheckFloatScalarPositive("TimeStretch", "hop_length", hop_length_));
RETURN_IF_NOT_OK(CheckIntScalarPositive("TimeStretch", "n_freq", n_freq_));
RETURN_IF_NOT_OK(CheckFloatScalarNotNan("TimeStretch", "fixed_rate", fixed_rate_));
RETURN_IF_NOT_OK(CheckFloatScalarPositive("TimeStretch", "fixed_rate", fixed_rate_));
RETURN_IF_NOT_OK(ValidateFloatScalarPositive("TimeStretch", "hop_length", hop_length_));
RETURN_IF_NOT_OK(ValidateIntScalarPositive("TimeStretch", "n_freq", n_freq_));
RETURN_IF_NOT_OK(ValidateFloatScalarNotNan("TimeStretch", "fixed_rate", fixed_rate_));
RETURN_IF_NOT_OK(ValidateFloatScalarPositive("TimeStretch", "fixed_rate", fixed_rate_));
return Status::OK();
}

77
mindspore/ccsrc/minddata/dataset/audio/ir/validators.cc
View File

@ -17,87 +17,20 @@
namespace mindspore {
namespace dataset {
/* ####################################### Validator Functions ############################################ */
Status CheckFloatScalarPositive(const std::string &op_name, const std::string &scalar_name, float scalar) {
RETURN_IF_NOT_OK(CheckScalar(op_name, scalar_name, scalar, {0}, true));
Status ValidateIntScalarNonNegative(const std::string &op_name, const std::string &scalar_name, int32_t scalar) {
RETURN_IF_NOT_OK(ValidateScalar(op_name, scalar_name, scalar, {0}, false));
return Status::OK();
}
Status CheckFloatScalarNotNan(const std::string &op_name, const std::string &scalar_name, float scalar) {
Status ValidateFloatScalarNotNan(const std::string &op_name, const std::string &scalar_name, float scalar) {
if (std::isnan(scalar)) {
std::string err_msg = op_name + ":" + scalar_name + " should be specified, got: Nan.";
std::string err_msg = op_name + ": " + scalar_name + " should be specified, got: Nan";
MS_LOG(ERROR) << err_msg;
return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg);
}
return Status::OK();
}
Status CheckFloatScalarNonNegative(const std::string &op_name, const std::string &scalar_name, float scalar) {
RETURN_IF_NOT_OK(CheckScalar(op_name, scalar_name, scalar, {0}, false));
return Status::OK();
}
Status CheckIntScalarNonNegative(const std::string &op_name, const std::string &scalar_name, int32_t scalar) {
RETURN_IF_NOT_OK(CheckScalar(op_name, scalar_name, scalar, {0}, false));
return Status::OK();
}
Status CheckIntScalarPositive(const std::string &op_name, const std::string &scalar_name, int32_t scalar) {
RETURN_IF_NOT_OK(CheckScalar(op_name, scalar_name, scalar, {0}, true));
return Status::OK();
}
Status CheckStringScalarInList(const std::string &op_name, const std::string &scalar_name, const std::string &scalar,
const std::vector<std::string> &str_vec) {
auto ret = std::find(str_vec.begin(), str_vec.end(), scalar);
if (ret == str_vec.end()) {
std::string interval_description = "[";
for (int m = 0; m < str_vec.size(); m++) {
std::string word = str_vec[m];
interval_description = interval_description + word;
if (m != str_vec.size() - 1) interval_description = interval_description + ", ";
}
interval_description = interval_description + "]";
std::string err_msg = op_name + ": " + scalar_name + " must be one of " + interval_description + ", got: " + scalar;
MS_LOG(ERROR) << err_msg;
return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg);
}
return Status::OK();
}
template <typename T>
Status CheckScalar(const std::string &op_name, const std::string &scalar_name, const T scalar,
const std::vector<T> &range, bool left_open_interval, bool right_open_interval) {
if (range.empty() || range.size() > 2) {
std::string err_msg = "Range check expecting size 1 or 2, but got: " + std::to_string(range.size());
MS_LOG(ERROR) << err_msg;
return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg);
}
if ((left_open_interval && scalar <= range[0]) || (!left_open_interval && scalar < range[0])) {
std::string interval_description = left_open_interval ? " greater than " : " greater than or equal to ";
std::string err_msg = op_name + ":" + scalar_name + " must be" + interval_description + std::to_string(range[0]) +
", got: " + std::to_string(scalar);
MS_LOG(ERROR) << err_msg;
return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg);
}
if (range.size() == 2) {
if ((right_open_interval && scalar >= range[1]) || (!right_open_interval && scalar > range[1])) {
std::string left_bracket = left_open_interval ? "(" : "[";
std::string right_bracket = right_open_interval ? ")" : "]";
std::string err_msg = op_name + ":" + scalar_name + " is out of range " + left_bracket +
std::to_string(range[0]) + ", " + std::to_string(range[1]) + right_bracket +
", got: " + std::to_string(scalar);
MS_LOG(ERROR) << err_msg;
return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg);
}
}
return Status::OK();
}
template Status CheckScalar(const std::string &op_name, const std::string &scalar_name, const float scalar,
const std::vector<float> &range, bool left_open_interval, bool right_open_interval);
template Status CheckScalar(const std::string &op_name, const std::string &scalar_name, const int32_t scalar,
const std::vector<int32_t> &range, bool left_open_interval, bool right_open_interval);
} // namespace dataset
} // namespace mindspore

34
mindspore/ccsrc/minddata/dataset/audio/ir/validators.h
View File

@ -19,52 +19,32 @@
#include <string>
#include <vector>
#include "minddata/dataset/kernels/ir/validators.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/kernels/ir/tensor_operation.h"
#include "minddata/dataset/kernels/ir/validators.h"
#include "minddata/dataset/util/status.h"
namespace mindspore {
namespace dataset {
// Helper function to non-nan float scalar
Status CheckFloatScalarNotNan(const std::string &op_name, const std::string &scalar_name, float scalar);
// Helper function to positive float scalar
Status CheckFloatScalarPositive(const std::string &op_name, const std::string &scalar_name, float scalar);
// Helper function to positive int scalar
Status CheckIntScalarPositive(const std::string &op_name, const std::string &scalar_name, int32_t scalar);
Status ValidateIntScalarNonNegative(const std::string &op_name, const std::string &scalar_name, int32_t scalar);
// Helper function to non-nan float scalar
Status ValidateFloatScalarNotNan(const std::string &op_name, const std::string &scalar_name, float scalar);
template <typename T>
// Helper function to check scalar is not equal to zero
Status CheckScalarNotZero(const std::string &op_name, const std::string &scalar_name, const T scalar) {
Status ValidateScalarNotZero(const std::string &op_name, const std::string &scalar_name, const T scalar) {
if (scalar == 0) {
std::string err_msg = op_name + ":" + scalar_name + " can't be 0" + ", got: " + std::to_string(scalar);
std::string err_msg = op_name + ": " + scalar_name + " can't be zero, got: " + std::to_string(scalar);
MS_LOG(ERROR) << err_msg;
return Status(StatusCode::kMDSyntaxError, __LINE__, __FILE__, err_msg);
}
return Status::OK();
}
// Helper function to positive float scalar
Status CheckFloatScalarPositive(const std::string &op_name, const std::string &scalar_name, float scalar);
// Helper function to non-negative float scalar
Status CheckFloatScalarNonNegative(const std::string &op_name, const std::string &scalar_name, float scalar);
// Helper function to positive int scalar
Status CheckIntScalarNonNegative(const std::string &op_name, const std::string &scalar_name, int32_t scalar);
// Helper function to check string scalar
Status CheckStringScalarInList(const std::string &op_name, const std::string &scalar_name, const std::string &scalar,
const std::vector<std::string> &str_vec);
// Helper function to validate scalar
template <typename T>
Status CheckScalar(const std::string &op_name, const std::string &scalar_name, const T scalar,
const std::vector<T> &range, bool left_open_interval = false, bool right_open_interval = false);
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ADUIO_IR_VALIDATORS_H_

18
mindspore/ccsrc/minddata/dataset/audio/kernels/allpass_biquad_op.cc
View File

@ -20,14 +20,15 @@
namespace mindspore {
namespace dataset {
Status AllpassBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
TensorShape input_shape = input->shape();
CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "AllpassBiquad: input dimension should be greater than 0.");
CHECK_FAIL_RETURN_UNEXPECTED(input->type() == DataType(DataType::DE_FLOAT32) ||
input->type() == DataType(DataType::DE_FLOAT16) ||
input->type() == DataType(DataType::DE_FLOAT64),
"AllpassBiquad: input type should be float, but got " + input->type().ToString());
CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "AllpassBiquad: input tensor is not in shape of <..., time>.");
CHECK_FAIL_RETURN_UNEXPECTED(
input->type() == DataType(DataType::DE_FLOAT32) || input->type() == DataType(DataType::DE_FLOAT16) ||
input->type() == DataType(DataType::DE_FLOAT64),
"AllpassBiquad: input tensor type should be float, but got: " + input->type().ToString());
double w0 = 2 * PI * central_freq_ / sample_rate_;
double alpha = sin(w0) / 2 / Q_;
double b0 = 1 - alpha;
@ -36,15 +37,16 @@ Status AllpassBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::share
double a0 = b2;
double a1 = -2 * cos(w0);
double a2 = 1 - alpha;
if (input->type() == DataType(DataType::DE_FLOAT32))
if (input->type() == DataType(DataType::DE_FLOAT32)) {
return Biquad(input, output, static_cast<float>(b0), static_cast<float>(b1), static_cast<float>(b2),
static_cast<float>(a0), static_cast<float>(a1), static_cast<float>(a2));
else if (input->type() == DataType(DataType::DE_FLOAT64))
} else if (input->type() == DataType(DataType::DE_FLOAT64)) {
return Biquad(input, output, static_cast<double>(b0), static_cast<double>(b1), static_cast<double>(b2),
static_cast<double>(a0), static_cast<double>(a1), static_cast<double>(a2));
else
} else {
return Biquad(input, output, static_cast<float16>(b0), static_cast<float16>(b1), static_cast<float16>(b2),
static_cast<float16>(a0), static_cast<float16>(a1), static_cast<float16>(a2));
}
}
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/audio/kernels/allpass_biquad_op.h
View File

@ -26,6 +26,7 @@
namespace mindspore {
namespace dataset {
class AllpassBiquadOp : public TensorOp {
public:
AllpassBiquadOp(int32_t sample_rate, float central_freq, float Q)

13
mindspore/ccsrc/minddata/dataset/audio/kernels/amplitude_to_db_op.cc
View File

@ -13,9 +13,8 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <limits>
#include "minddata/dataset/audio/kernels/amplitude_to_db_op.h"
#include "minddata/dataset/audio/kernels/audio_utils.h"
#include "minddata/dataset/kernels/data/data_utils.h"
#include "minddata/dataset/util/status.h"
@ -26,7 +25,7 @@ namespace dataset {
Status AmplitudeToDBOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
if (input->shape().Rank() < 2) {
std::string err_msg = "AmplitudeToDB: input tensor shape should be <..., freq, time>";
std::string err_msg = "AmplitudeToDB: input tensor is not in shape of <..., freq, time>.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
@ -40,12 +39,12 @@ Status AmplitudeToDBOp::Compute(const std::shared_ptr<Tensor> &input, std::share
// typecast
CHECK_FAIL_RETURN_UNEXPECTED(input->type() != DataType::DE_STRING,
"AmplitudeToDB: input type should be float, but got string.");
"AmplitudeToDB: input tensor type should be float, but got: string.");
if (input->type() != DataType::DE_FLOAT64) {
CHECK_FAIL_RETURN_UNEXPECTED(TypeCast(input, &input_tensor, DataType(DataType::DE_FLOAT32)),
"AmplitudeToDB: input type should be float, but got " + input->type().ToString());
CHECK_FAIL_RETURN_UNEXPECTED(
TypeCast(input, &input_tensor, DataType(DataType::DE_FLOAT32)),
"AmplitudeToDB: input tensor type should be float, but got: " + input->type().ToString());
return AmplitudeToDB<float>(input_tensor, output, multiplier, amin, db_multiplier, top_db);
} else {
input_tensor = input;
return AmplitudeToDB<double>(input_tensor, output, multiplier, amin, db_multiplier, top_db);

1
mindspore/ccsrc/minddata/dataset/audio/kernels/amplitude_to_db_op.h
View File

@ -29,6 +29,7 @@
namespace mindspore {
namespace dataset {
class AmplitudeToDBOp : public TensorOp {
public:
AmplitudeToDBOp(ScaleType stype, float ref_value, float amin, float top_db)

6
mindspore/ccsrc/minddata/dataset/audio/kernels/angle_op.cc
View File

@ -25,8 +25,10 @@ namespace dataset {
Status AngleOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
// if If the last dimension is not 2, then it's not a complex number
CHECK_FAIL_RETURN_UNEXPECTED(input->shape()[-1] == 2, "Angle: The input is not several legal complex numbers");
CHECK_FAIL_RETURN_UNEXPECTED(input->type().IsNumeric(), "Angle: The input type should be numbers");
CHECK_FAIL_RETURN_UNEXPECTED(input->shape()[-1] == 2, "Angle: input tensor is not in shape of <..., complex=2>.");
CHECK_FAIL_RETURN_UNEXPECTED(
input->type().IsNumeric(),
"Angle: input tensor type should be int, float or double, but got: " + input->type().ToString());
if (input->type() == DataType(DataType::DE_FLOAT64)) {
return Angle<double>(input, output);
} else {

1
mindspore/ccsrc/minddata/dataset/audio/kernels/angle_op.h
View File

@ -26,6 +26,7 @@
namespace mindspore {
namespace dataset {
class AngleOp : public TensorOp {
public:
// Convert complex numbers to angles

79
mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc
View File

@ -18,58 +18,16 @@
#include "minddata/dataset/util/random.h"
#include "minddata/dataset/util/status.h"
namespace mindspore {
namespace dataset {
template <typename T>
Status AmplitudeToDB(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, T multiplier, T amin,
T db_multiplier, T top_db) {
TensorShape input_shape = input->shape();
TensorShape to_shape = input_shape.Rank() == 2
? TensorShape({1, 1, input_shape[-2], input_shape[-1]})
: TensorShape({input->Size() / (input_shape[-3] * input_shape[-2] * input_shape[-1]),
input_shape[-3], input_shape[-2], input_shape[-1]});
RETURN_IF_NOT_OK(input->Reshape(to_shape));
std::vector<T> max_val;
int step = to_shape[-3] * input_shape[-2] * input_shape[-1];
int cnt = 0;
T temp_max = std::numeric_limits<T>::lowest();
for (auto itr = input->begin<T>(); itr != input->end<T>(); itr++) {
// do clamp
*itr = *itr < amin ? log10(amin) * multiplier : log10(*itr) * multiplier;
*itr -= multiplier * db_multiplier;
// calculate max by axis
cnt++;
if ((*itr) > temp_max) temp_max = *itr;
if (cnt % step == 0) {
max_val.push_back(temp_max);
temp_max = std::numeric_limits<T>::lowest();
}
}
if (!std::isnan(top_db)) {
int ind = 0;
for (auto itr = input->begin<T>(); itr != input->end<T>(); itr++, ind++) {
float lower_bound = max_val[ind / step] - top_db;
*itr = std::max((*itr), static_cast<T>(lower_bound));
}
}
RETURN_IF_NOT_OK(input->Reshape(input_shape));
*output = input;
return Status::OK();
}
template Status AmplitudeToDB<float>(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output,
float multiplier, float amin, float db_multiplier, float top_db);
template Status AmplitudeToDB<double>(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output,
double multiplier, double amin, double db_multiplier, double top_db);
/// \brief Generate linearly spaced vector
/// \brief Generate linearly spaced vector.
/// \param[in] start - Value of the startpoint.
/// \param[in] end - Value of the endpoint.
/// \param[in] n - N points in the output tensor.
/// \param[out] output - Tensor has n points with linearly space. The spacing between the points is (end-start)/(n-1).
/// \return Status return code
/// \return Status return code.
template <typename T>
Status Linspace(std::shared_ptr<Tensor> *output, T start, T end, int n) {
if (start > end) {
@ -91,10 +49,10 @@ Status Linspace(std::shared_ptr<Tensor> *output, T start, T end, int n) {
return Status::OK();
}
/// \brief Calculate complex tensor angle
/// \brief Calculate complex tensor angle.
/// \param[in] input - Input tensor, must be complex, <channel, freq, time, complex=2>.
/// \param[out] output - Complex tensor angle.
/// \return Status return code
/// \return Status return code.
template <typename T>
Status ComplexAngle(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
// check complex
@ -123,10 +81,10 @@ Status ComplexAngle(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor
return Status::OK();
}
/// \brief Calculate complex tensor abs
/// \brief Calculate complex tensor abs.
/// \param[in] input - Input tensor, must be complex, <channel, freq, time, complex=2>.
/// \param[out] output - Complex tensor abs.
/// \return Status return code
/// \return Status return code.
template <typename T>
Status ComplexAbs(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
// check complex
@ -152,17 +110,17 @@ Status ComplexAbs(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor>
return Status::OK();
}
/// \brief Reconstruct complex tensor from norm and angle
/// \brief Reconstruct complex tensor from norm and angle.
/// \param[in] abs - The absolute value of the complex tensor.
/// \param[in] angle - The angle of the complex tensor.
/// \param[out] output - Complex tensor, <channel, freq, time, complex=2>.
/// \return Status return code
/// \return Status return code.
template <typename T>
Status Polar(const std::shared_ptr<Tensor> &abs, const std::shared_ptr<Tensor> &angle,
std::shared_ptr<Tensor> *output) {
// check shape
if (abs->shape() != angle->shape()) {
std::string err_msg = "Polar: input shape of abs and angle must be same.";
std::string err_msg = "Polar: input tensor shape of abs and angle must be the same.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
@ -185,12 +143,12 @@ Status Polar(const std::shared_ptr<Tensor> &abs, const std::shared_ptr<Tensor> &
return Status::OK();
}
/// \brief Pad complex tensor
/// \brief Pad complex tensor.
/// \param[in] input - The complex tensor.
/// \param[in] length - The length of padding.
/// \param[in] dim - The dim index for padding.
/// \param[out] output - Complex tensor, <channel, freq, time, complex=2>.
/// \return Status return code
/// \return Status return code.
template <typename T>
Status PadComplexTensor(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, int length, int dim) {
TensorShape input_shape = input->shape();
@ -218,13 +176,13 @@ Status PadComplexTensor(const std::shared_ptr<Tensor> &input, std::shared_ptr<Te
return Status::OK();
}
/// \brief Calculate phase
/// \brief Calculate phase.
/// \param[in] angle_0 - The angle.
/// \param[in] angle_1 - The angle.
/// \param[in] phase_advance - The phase advance.
/// \param[in] phase_time0 - The phase at time 0.
/// \param[out] output - Phase tensor.
/// \return Status return code
/// \return Status return code.
template <typename T>
Status Phase(const std::shared_ptr<Tensor> &angle_0, const std::shared_ptr<Tensor> &angle_1,
const std::shared_ptr<Tensor> &phase_advance, const std::shared_ptr<Tensor> &phase_time0,
@ -269,12 +227,12 @@ Status Phase(const std::shared_ptr<Tensor> &angle_0, const std::shared_ptr<Tenso
return Status::OK();
}
/// \brief Calculate magnitude
/// \brief Calculate magnitude.
/// \param[in] alphas - The alphas.
/// \param[in] abs_0 - The norm.
/// \param[in] abs_1 - The norm.
/// \param[out] output - Magnitude tensor.
/// \return Status return code
/// \return Status return code.
template <typename T>
Status Mag(const std::shared_ptr<Tensor> &abs_0, const std::shared_ptr<Tensor> &abs_1, std::shared_ptr<Tensor> *output,
const std::vector<T> &alphas) {
@ -377,9 +335,8 @@ Status TimeStretch(std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *outpu
RETURN_IF_NOT_OK(TimeStretch<double>(input, output, rate, phase_advance));
break;
default:
RETURN_STATUS_UNEXPECTED(
"TimeStretch: unsupported type, currently supported types include "
"[float, double].");
RETURN_STATUS_UNEXPECTED("TimeStretch: input tensor type should be float or double, but got: " +
input->type().ToString());
}
return Status::OK();
}

115
mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.h
View File

@ -29,21 +29,58 @@
#include "minddata/dataset/util/status.h"
constexpr double PI = 3.141592653589793;
namespace mindspore {
namespace dataset {
/// \brief Turn a tensor from the power/amplitude scale to the decibel scale.
/// \param input/output: Tensor of shape <...,freq,time>
/// \param multiplier: power - 10, amplitude - 20
/// \param amin: lower bound
/// \param db_multiplier: multiplier for decibels
/// \param top_db: the lower bound for decibels cut-off
/// \return Status code
/// \param input/output: Tensor of shape <..., freq, time>.
/// \param multiplier: power - 10, amplitude - 20.
/// \param amin: lower bound.
/// \param db_multiplier: multiplier for decibels.
/// \param top_db: the lower bound for decibels cut-off.
/// \return Status code.
template <typename T>
Status AmplitudeToDB(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, T multiplier, T amin,
T db_multiplier, T top_db);
T db_multiplier, T top_db) {
TensorShape input_shape = input->shape();
TensorShape to_shape = input_shape.Rank() == 2
? TensorShape({1, 1, input_shape[-2], input_shape[-1]})
: TensorShape({input->Size() / (input_shape[-3] * input_shape[-2] * input_shape[-1]),
input_shape[-3], input_shape[-2], input_shape[-1]});
RETURN_IF_NOT_OK(input->Reshape(to_shape));
/// \brief Calculate the angles of the complex numbers
/// \param input/output: Tensor of shape <...,time>
std::vector<T> max_val;
int step = to_shape[-3] * input_shape[-2] * input_shape[-1];
int cnt = 0;
T temp_max = std::numeric_limits<T>::lowest();
for (auto itr = input->begin<T>(); itr != input->end<T>(); itr++) {
// do clamp
*itr = *itr < amin ? log10(amin) * multiplier : log10(*itr) * multiplier;
*itr -= multiplier * db_multiplier;
// calculate max by axis
cnt++;
if ((*itr) > temp_max) temp_max = *itr;
if (cnt % step == 0) {
max_val.push_back(temp_max);
temp_max = std::numeric_limits<T>::lowest();
}
}
if (!std::isnan(top_db)) {
int ind = 0;
for (auto itr = input->begin<T>(); itr != input->end<T>(); itr++, ind++) {
float lower_bound = max_val[ind / step] - top_db;
*itr = std::max((*itr), static_cast<T>(lower_bound));
}
}
RETURN_IF_NOT_OK(input->Reshape(input_shape));
*output = input;
return Status::OK();
}
/// \brief Calculate the angles of the complex numbers.
/// \param input/output: Tensor of shape <..., time>.
template <typename T>
Status Angle(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
TensorShape shape = input->shape();
@ -68,14 +105,14 @@ Status Angle(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *outp
}
/// \brief Perform a biquad filter of input tensor.
/// \param input/output: Tensor of shape <...,time>
/// \param a0: denominator coefficient of current output y[n], typically 1
/// \param a1: denominator coefficient of current output y[n-1]
/// \param a2: denominator coefficient of current output y[n-2]
/// \param b0: numerator coefficient of current input, x[n]
/// \param b1: numerator coefficient of input one time step ago x[n-1]
/// \param b2: numerator coefficient of input two time steps ago x[n-2]
/// \return Status code
/// \param input/output: Tensor of shape <..., time>.
/// \param a0: denominator coefficient of current output y[n], typically 1.
/// \param a1: denominator coefficient of current output y[n-1].
/// \param a2: denominator coefficient of current output y[n-2].
/// \param b0: numerator coefficient of current input, x[n].
/// \param b1: numerator coefficient of input one time step ago x[n-1].
/// \param b2: numerator coefficient of input two time steps ago x[n-2].
/// \return Status code.
template <typename T>
Status Biquad(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, T b0, T b1, T b2, T a0, T a1,
T a2) {
@ -91,10 +128,10 @@ Status Biquad(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *out
}
/// \brief Perform an IIR filter by evaluating difference equation.
/// \param input/output: Tensor of shape <...,time>
/// \param input/output: Tensor of shape <..., time>
/// \param a_coeffs: denominator coefficients of difference equation of dimension of (n_order + 1).
/// \param b_coeffs: numerator coefficients of difference equation of dimension of (n_order + 1).
/// \param clamp: If True, clamp the output signal to be in the range [-1, 1] (Default: True)
/// \param clamp: If True, clamp the output signal to be in the range [-1, 1] (Default: True).
/// \return Status code
template <typename T>
Status LFilter(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, std::vector<T> a_coeffs,
@ -112,7 +149,7 @@ Status LFilter(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *ou
size_t channel_idx = 1;
size_t m_num_order = b_coeffs.size() - 1;
size_t m_den_order = a_coeffs.size() - 1;
// init A_coeffs and B_coeffs by div(a0)
// init A_coeffs and B_coeffs by div(a0)
for (size_t i = 1; i < a_coeffs.size(); i++) {
a_coeffs[i] /= a_coeffs[0];
}
@ -172,40 +209,40 @@ Status LFilter(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *ou
// unpack batch
Tensor::CreateFromVector(out_vect, input_shape, &out);
*output = out;
delete m_px;
delete m_py;
delete[] m_px;
delete[] m_py;
return Status::OK();
}
/// \brief Stretch STFT in time at a given rate, without changing the pitch.
/// \param[in] input - Tensor of shape <...,freq,time>.
/// \param[in] input - Tensor of shape <..., freq, time>.
/// \param[in] rate - Stretch factor.
/// \param[in] phase_advance - Expected phase advance in each bin.
/// \param[out] output - Tensor after stretch in time domain.
/// \return Status return code
/// \return Status return code.
Status TimeStretch(std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *output, float rate, float hop_length,
float n_freq);
/// \brief Apply a mask along axis.
/// \param input: Tensor of shape <...,freq,time>
/// \param output: Tensor of shape <...,freq,time>
/// \param mask_param: Number of columns to be masked will be uniformly sampled from [0, mask_param]
/// \param mask_value: Value to assign to the masked columns
/// \param axis: Axis to apply masking on (1 -> frequency, 2 -> time)
/// \param rnd: Number generator
/// \return Status code
/// \param input: Tensor of shape <..., freq, time>.
/// \param output: Tensor of shape <..., freq, time>.
/// \param mask_param: Number of columns to be masked will be uniformly sampled from [0, mask_param].
/// \param mask_value: Value to assign to the masked columns.
/// \param axis: Axis to apply masking on (1 -> frequency, 2 -> time).
/// \param rnd: Number generator.
/// \return Status code.
Status RandomMaskAlongAxis(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, int64_t mask_param,
double mask_value, int axis, std::mt19937 rnd);
/// \brief Apply a mask along axis. All examples will have the same mask interval.
/// \param input: Tensor of shape <...,freq,time>
/// \param output: Tensor of shape <...,freq,time>
/// \param mask_width: The width of the mask
/// \param mask_start: Starting position of the mask
/// Mask will be applied from indices [mask_start, mask_start + mask_width)
/// \param mask_value: Value to assign to the masked columns
/// \param axis: Axis to apply masking on (1 -> frequency, 2 -> time)
/// \return Status code
/// \param input: Tensor of shape <..., freq, time>.
/// \param output: Tensor of shape <..., freq, time>.
/// \param mask_width: The width of the mask.
/// \param mask_start: Starting position of the mask.
/// Mask will be applied from indices [mask_start, mask_start + mask_width).
/// \param mask_value: Value to assign to the masked columns.
/// \param axis: Axis to apply masking on (1 -> frequency, 2 -> time).
/// \return Status code.
Status MaskAlongAxis(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output, int64_t mask_width,
int64_t mask_start, double mask_value, int axis);
} // namespace dataset

11
mindspore/ccsrc/minddata/dataset/audio/kernels/band_biquad_op.cc
View File

@ -25,12 +25,12 @@ Status BandBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_p
IO_CHECK(input, output);
TensorShape input_shape = input->shape();
// check input tensor dimension, it should be greater than 0.
CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "BandBiquad: input dimension should be greater than 0.");
CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "BandBiquad: input tensor is not in shape of <..., time>.");
// check input type, it should be DE_FLOAT32 or DE_FLOAT16 or DE_FLOAT64
CHECK_FAIL_RETURN_UNEXPECTED(input->type() == DataType(DataType::DE_FLOAT32) ||
input->type() == DataType(DataType::DE_FLOAT16) ||
input->type() == DataType(DataType::DE_FLOAT64),
"BandBiquad: input type should be float, but got " + input->type().ToString());
"BandBiquad: input tensor type should be float, but got: " + input->type().ToString());
double w0 = 2 * PI * central_freq_ / sample_rate_;
double bw_Hz = central_freq_ / Q_;
double a0 = 1.;
@ -45,15 +45,16 @@ Status BandBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_p
}
double b1 = 0.;
double b2 = 0.;
if (input->type() == DataType(DataType::DE_FLOAT32))
if (input->type() == DataType(DataType::DE_FLOAT32)) {
return Biquad(input, output, static_cast<float>(b0), static_cast<float>(b1), static_cast<float>(b2),
static_cast<float>(a0), static_cast<float>(a1), static_cast<float>(a2));
else if (input->type() == DataType(DataType::DE_FLOAT64))
} else if (input->type() == DataType(DataType::DE_FLOAT64)) {
return Biquad(input, output, static_cast<double>(b0), static_cast<double>(b1), static_cast<double>(b2),
static_cast<double>(a0), static_cast<double>(a1), static_cast<double>(a2));
else
} else {
return Biquad(input, output, static_cast<float16>(b0), static_cast<float16>(b1), static_cast<float16>(b2),
static_cast<float16>(a0), static_cast<float16>(a1), static_cast<float16>(a2));
}
}
} // namespace dataset

3
mindspore/ccsrc/minddata/dataset/audio/kernels/band_biquad_op.h
View File

@ -17,8 +17,8 @@
#define MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_KERNELS_BAND_BIQUAD_OP_H_
#include <memory>
#include <vector>
#include <string>
#include <vector>
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/kernels/tensor_op.h"
@ -26,6 +26,7 @@
namespace mindspore {
namespace dataset {
class BandBiquadOp : public TensorOp {
public:
BandBiquadOp(int32_t sample_rate, float central_freq, float Q, bool noise)

17
mindspore/ccsrc/minddata/dataset/audio/kernels/bandpass_biquad_op.cc
View File

@ -24,12 +24,12 @@ namespace dataset {
Status BandpassBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
TensorShape input_shape = input->shape();
CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "BandpassBiquad: inpute dimension should be greater than 0.");
CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "BandpassBiquad: input tensor is not in shape of <..., time>.");
// check input type, it should be DE_FLOAT32 or DE_FLOAT16 or DE_FLOAT64
CHECK_FAIL_RETURN_UNEXPECTED(input->type() == DataType(DataType::DE_FLOAT32) ||
input->type() == DataType(DataType::DE_FLOAT16) ||
input->type() == DataType(DataType::DE_FLOAT64),
"BandpassBiquad: input type should be float, but got " + input->type().ToString());
CHECK_FAIL_RETURN_UNEXPECTED(
input->type() == DataType(DataType::DE_FLOAT32) || input->type() == DataType(DataType::DE_FLOAT16) ||
input->type() == DataType(DataType::DE_FLOAT64),
"BandpassBiquad: input tensor type should be float, but got: " + input->type().ToString());
float w0 = 2 * PI * central_freq_ / sample_rate_;
float alpha = sin(w0) / 2 / Q_;
float temp;
@ -46,15 +46,16 @@ Status BandpassBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shar
float a1 = (-2) * cos(w0);
float a2 = 1 - alpha;
if (input->type() == DataType(DataType::DE_FLOAT32))
if (input->type() == DataType(DataType::DE_FLOAT32)) {
return Biquad(input, output, static_cast<float>(b0), static_cast<float>(b1), static_cast<float>(b2),
static_cast<float>(a0), static_cast<float>(a1), static_cast<float>(a2));
else if (input->type() == DataType(DataType::DE_FLOAT64))
} else if (input->type() == DataType(DataType::DE_FLOAT64)) {
return Biquad(input, output, static_cast<double>(b0), static_cast<double>(b1), static_cast<double>(b2),
static_cast<double>(a0), static_cast<double>(a1), static_cast<double>(a2));
else
} else {
return Biquad(input, output, static_cast<float16>(b0), static_cast<float16>(b1), static_cast<float16>(b2),
static_cast<float16>(a0), static_cast<float16>(a1), static_cast<float16>(a2));
}
}
} // namespace dataset
} // namespace mindspore

3
mindspore/ccsrc/minddata/dataset/audio/kernels/bandpass_biquad_op.h
View File

@ -17,8 +17,8 @@
#define MINDSPORE_CCSRC_MINDDATA_DATASET_AUDIO_KERNELS_BANDPASS_BIQUAD_OP_H_
#include <memory>
#include <vector>
#include <string>
#include <vector>
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/kernels/tensor_op.h"
@ -26,6 +26,7 @@
namespace mindspore {
namespace dataset {
class BandpassBiquadOp : public TensorOp {
public:
BandpassBiquadOp(int32_t sample_rate, float central_freq, float Q, bool const_skirt_gain)

19
mindspore/ccsrc/minddata/dataset/audio/kernels/bandreject_biquad_op.cc
View File

@ -20,15 +20,17 @@
namespace mindspore {
namespace dataset {
Status BandrejectBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
// check input type and input shape
TensorShape input_shape = input->shape();
CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "BandrejectBiquad: input dimension should be greater than 0.");
CHECK_FAIL_RETURN_UNEXPECTED(input->type() == DataType(DataType::DE_FLOAT32) ||
input->type() == DataType(DataType::DE_FLOAT16) ||
input->type() == DataType(DataType::DE_FLOAT64),
"BandrejectBiquad: input type should be float, but got " + input->type().ToString());
CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0,
"BandrejectBiquad: input tensor is not in shape of <..., time>.");
CHECK_FAIL_RETURN_UNEXPECTED(
input->type() == DataType(DataType::DE_FLOAT32) || input->type() == DataType(DataType::DE_FLOAT16) ||
input->type() == DataType(DataType::DE_FLOAT64),
"BandrejectBiquad: input tensor type should be float, but got: " + input->type().ToString());
double w0 = 2 * PI * central_freq_ / sample_rate_;
double alpha = sin(w0) / 2 / Q_;
double b0 = 1;
@ -37,15 +39,16 @@ Status BandrejectBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::sh
double a0 = 1 + alpha;
double a1 = b1;
double a2 = 1 - alpha;
if (input->type() == DataType(DataType::DE_FLOAT32))
if (input->type() == DataType(DataType::DE_FLOAT32)) {
return Biquad(input, output, static_cast<float>(b0), static_cast<float>(b1), static_cast<float>(b2),
static_cast<float>(a0), static_cast<float>(a1), static_cast<float>(a2));
else if (input->type() == DataType(DataType::DE_FLOAT64))
} else if (input->type() == DataType(DataType::DE_FLOAT64)) {
return Biquad(input, output, static_cast<double>(b0), static_cast<double>(b1), static_cast<double>(b2),
static_cast<double>(a0), static_cast<double>(a1), static_cast<double>(a2));
else
} else {
return Biquad(input, output, static_cast<float16>(b0), static_cast<float16>(b1), static_cast<float16>(b2),
static_cast<float16>(a0), static_cast<float16>(a1), static_cast<float16>(a2));
}
}
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/audio/kernels/bandreject_biquad_op.h
View File

@ -26,6 +26,7 @@
namespace mindspore {
namespace dataset {
class BandrejectBiquadOp : public TensorOp {
public:
BandrejectBiquadOp(int32_t sample_rate, float central_freq, float Q)

11
mindspore/ccsrc/minddata/dataset/audio/kernels/bass_biquad_op.cc
View File

@ -24,12 +24,12 @@ namespace dataset {
Status BassBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
TensorShape input_shape = input->shape();
CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "BassBiquad: input dimension should be greater than 0.");
CHECK_FAIL_RETURN_UNEXPECTED(input_shape.Size() > 0, "BassBiquad: input tensor is not in shape of <..., time>.");
// check input type, it should be DE_FLOAT32 or DE_FLOAT16 or DE_FLOAT64
CHECK_FAIL_RETURN_UNEXPECTED(input->type() == DataType(DataType::DE_FLOAT32) ||
input->type() == DataType(DataType::DE_FLOAT16) ||
input->type() == DataType(DataType::DE_FLOAT64),
"BassBiquad: input type should be float, but got " + input->type().ToString());
"BassBiquad: input tensor type should be float, but got: " + input->type().ToString());
double w0 = 2 * PI * central_freq_ / sample_rate_;
double alpha = sin(w0) / 2 / Q_;
@ -45,17 +45,18 @@ Status BassBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_p
double a0 = (A + 1) + temp2 + temp1;
double a1 = -2 * ((A - 1) + temp3);
double a2 = (A + 1) + temp2 - temp1;
if (input->type() == DataType(DataType::DE_FLOAT32))
if (input->type() == DataType(DataType::DE_FLOAT32)) {
return Biquad(input, output, static_cast<float>(b0 / a0), static_cast<float>(b1 / a0), static_cast<float>(b2 / a0),
static_cast<float>(1.0), static_cast<float>(a1 / a0), static_cast<float>(a2 / a0));
else if (input->type() == DataType(DataType::DE_FLOAT64))
} else if (input->type() == DataType(DataType::DE_FLOAT64)) {
return Biquad(input, output, static_cast<double>(b0 / a0), static_cast<double>(b1 / a0),
static_cast<double>(b2 / a0), static_cast<double>(1.0), static_cast<double>(a1 / a0),
static_cast<double>(a2 / a0));
else
} else {
return Biquad(input, output, static_cast<float16>(b0 / a0), static_cast<float16>(b1 / a0),
static_cast<float16>(b2 / a0), static_cast<float16>(1.0), static_cast<float16>(a1 / a0),
static_cast<float16>(a2 / a0));
}
}
} // namespace dataset
} // namespace mindspore

3
mindspore/ccsrc/minddata/dataset/audio/kernels/bass_biquad_op.h
View File

@ -27,6 +27,7 @@
namespace mindspore {
namespace dataset {
class BassBiquadOp : public TensorOp {
public:
BassBiquadOp(int32_t sample_rate, float gain, float central_freq, float Q)
@ -35,7 +36,7 @@ class BassBiquadOp : public TensorOp {
~BassBiquadOp() override = default;
void Print(std::ostream &out) const override {
out << Name() << ": sample_rate: " << sample_rate_ << ", gain:" << gain_ << ", central_freq: " << central_freq_
out << Name() << ": sample_rate: " << sample_rate_ << ", gain: " << gain_ << ", central_freq: " << central_freq_
<< ", Q: " << Q_ << std::endl;
}

2
mindspore/ccsrc/minddata/dataset/audio/kernels/frequency_masking_op.cc
View File

@ -47,7 +47,7 @@ Status FrequencyMaskingOp::Compute(const std::shared_ptr<Tensor> &input, std::sh
std::shared_ptr<Tensor> input_tensor;
// typecast
CHECK_FAIL_RETURN_UNEXPECTED(input->type() != DataType::DE_STRING,
"FrequencyMasking: input tensor type should be float, but got string.");
"FrequencyMasking: input tensor type should be float, but got: string.");
if (input->type() != DataType::DE_FLOAT64) {
RETURN_IF_NOT_OK(TypeCast(input, &input_tensor, DataType(DataType::DE_FLOAT32)));
} else {

4
mindspore/ccsrc/minddata/dataset/audio/kernels/time_masking_op.cc
View File

@ -33,7 +33,7 @@ TimeMaskingOp::TimeMaskingOp(bool iid_masks, int64_t time_mask_param, int64_t ma
Status TimeMaskingOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
// input <..., freq, time>
CHECK_FAIL_RETURN_UNEXPECTED(input->Rank() >= 2, "TimeMasking: input dimension must be greater than 2.");
CHECK_FAIL_RETURN_UNEXPECTED(input->Rank() >= 2, "TimeMasking: input tensor is not in shape of <..., freq, time>.");
TensorShape input_shape = input->shape();
CHECK_FAIL_RETURN_UNEXPECTED(input_shape[-1] >= time_mask_param_,
"TimeMasking: time_mask_param should be less than the length of time dimension.");
@ -41,7 +41,7 @@ Status TimeMaskingOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_
std::shared_ptr<Tensor> input_tensor;
// typecast
CHECK_FAIL_RETURN_UNEXPECTED(input->type() != DataType::DE_STRING,
"TimeMasking: input type should be float, but got string.");
"TimeMasking: input tensor type should be float, but got: string.");
if (input->type() != DataType::DE_FLOAT64) {
RETURN_IF_NOT_OK(TypeCast(input, &input_tensor, DataType(DataType::DE_FLOAT32)));
} else {

13
mindspore/ccsrc/minddata/dataset/audio/kernels/time_stretch_op.cc
View File

@ -33,15 +33,8 @@ Status TimeStretchOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_
IO_CHECK(input, output);
// check shape
if (input->shape().Rank() < 3) {
std::string err_msg = "TimeStretch: input tensor shape is not <..., freq, num_frame, complex=2>.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
// check complex
if (!input->IsComplex()) {
std::string err_msg = "TimeStretch: input tensor is not in shape of <..., 2>.";
if (input->shape().Rank() < 3 || !input->IsComplex()) {
std::string err_msg = "TimeStretch: input tensor is not in shape of <..., freq, num_frame, complex=2>.";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_SYNTAX_ERROR(err_msg);
}
@ -51,7 +44,7 @@ Status TimeStretchOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_
float hop_length = std::isnan(hop_length_) ? (n_freq_ - 1) : hop_length_;
// typecast
CHECK_FAIL_RETURN_UNEXPECTED(input->type() != DataType::DE_STRING,
"TimeStretch: input tensor type should be [int, float, double], but got string.");
"TimeStretch: input tensor type should be int, float or double, but got: string.");
if (input->type() != DataType::DE_FLOAT64) {
RETURN_IF_NOT_OK(TypeCast(input, &input_tensor, DataType(DataType::DE_FLOAT32)));
} else {

4
mindspore/ccsrc/minddata/dataset/audio/kernels/time_stretch_op.h
View File

@ -27,6 +27,7 @@
namespace mindspore {
namespace dataset {
class TimeStretchOp : public TensorOp {
public:
/// Default value
@ -43,9 +44,6 @@ class TimeStretchOp : public TensorOp {
std::string Name() const override { return kTimeStretchOp; }
/// \param[in] inputs
/// \param[out] outputs
/// \return Status code
Status OutputShape(const std::vector<TensorShape> &inputs, std::vector<TensorShape> &outputs) override;
private:

24
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/flickr_op.cc
View File

@ -17,10 +17,10 @@
#include <algorithm>
#include <fstream>
#include <iomanip>
#include <set>
#include <utility>
#include "debug/common.h"
#include "minddata/dataset/core/config_manager.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/engine/datasetops/source/sampler/sequential_sampler.h"
@ -94,7 +94,13 @@ void FlickrOp::Print(std::ostream &out, bool show_all) const {
}
Status FlickrOp::ParseFlickrData() {
std::ifstream file_handle(file_path_);
auto real_file_path = Common::GetRealPath(file_path_);
if (!real_file_path.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file_path_;
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file_path_);
}
std::ifstream file_handle(real_file_path.value());
if (!file_handle.is_open()) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open Flickr annotation file: " + file_path_);
}
@ -129,7 +135,11 @@ Status FlickrOp::ParseFlickrData() {
}
bool valid = false;
RETURN_IF_NOT_OK(CheckImageType(image_file_path, &valid));
Status type_check = CheckImageType(image_file_path, &valid);
if (type_check.IsError()) {
file_handle.close();
RETURN_IF_NOT_OK(type_check);
}
if (!valid) {
continue;
}
@ -153,10 +163,16 @@ Status FlickrOp::ParseFlickrData() {
// Optimization: Could take in a tensor
// This function does not return status because we want to just skip bad input, not crash
Status FlickrOp::CheckImageType(const std::string &file_name, bool *valid) {
auto real_file_name = Common::GetRealPath(file_name);
if (!real_file_name.has_value()) {
MS_LOG(ERROR) << "Get real path failed, path=" << file_name;
RETURN_STATUS_UNEXPECTED("Get real path failed, path=" + file_name);
}
std::ifstream file_handle;
constexpr int read_num = 3;
*valid = false;
file_handle.open(file_name, std::ios::binary | std::ios::in);
file_handle.open(real_file_name.value(), std::ios::binary | std::ios::in);
if (!file_handle.is_open()) {
RETURN_STATUS_UNEXPECTED("Invalid file, failed to open image file: " + file_name);
}

13
mindspore/ccsrc/minddata/dataset/include/dataset/audio.h
View File

@ -35,6 +35,7 @@ class TensorOperation;
// Transform operations for performing computer audio.
namespace audio {
/// \brief Compute the angle of complex tensor input.
class Angle final : public TensorTransform {
public:
@ -98,10 +99,10 @@ class AllpassBiquad final : public TensorTransform {
class AmplitudeToDB final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] stype ['kPower', 'kMagnitude']
/// \param[in] ref_value Calculate db_multiplier
/// \param[in] amin Clamp the input waveform
/// \param[in] top_db Decibels cut-off value
/// \param[in] stype ['kPower', 'kMagnitude'].
/// \param[in] ref_value Calculate db_multiplier.
/// \param[in] amin Clamp the input waveform.
/// \param[in] top_db Decibels cut-off value.
explicit AmplitudeToDB(ScaleType stype = ScaleType::kPower, float ref_value = 1.0, float amin = 1e-10,
float top_db = 80.0);
@ -124,9 +125,9 @@ class BandpassBiquad final : public TensorTransform {
/// \brief Constructor.
/// \param[in] sample_rate Sampling rate of the waveform, e.g. 44100 (Hz).
/// \param[in] central_freq Central frequency (in Hz).
/// \param[in] Q Quality factor, https://en.wikipedia.org/wiki/Q_factor (Default: 0.707).
/// \param[in] Q Quality factor, https://en.wikipedia.org/wiki/Q_factor (Default: 0.707).
/// \param[in] const_skirt_gain, If ``True``, uses a constant skirt gain (peak gain = Q). If ``False``, uses a
/// constant 0dB peak gain. (Default: False).
/// constant 0dB peak gain (Default: False).
explicit BandpassBiquad(int32_t sample_rate, float central_freq, float Q = 0.707, bool const_skirt_gain = false);
/// \brief Destructor.

41
mindspore/ccsrc/minddata/dataset/include/dataset/vision.h
View File

@ -36,26 +36,6 @@ class TensorOperation;
// Transform operations for performing computer vision.
namespace vision {
/// \brief Apply automatic contrast on the input image.
class AutoContrast final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] cutoff Percent of pixels to cut off from the histogram, the valid range of cutoff value is 0 to 50.
/// \param[in] ignore Pixel values to ignore.
explicit AutoContrast(float cutoff = 0.0, std::vector<uint32_t> ignore = {});
/// \brief Destructor.
~AutoContrast() = default;
protected:
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief AdjustGamma TensorTransform.
/// \notes Apply gamma correction on input image.
@ -80,6 +60,27 @@ class AdjustGamma final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief Apply automatic contrast on the input image.
class AutoContrast final : public TensorTransform {
public:
/// \brief Constructor.
/// \param[in] cutoff Percent of pixels to cut off from the histogram, the valid range of cutoff value is 0 to 50.
/// \param[in] ignore Pixel values to ignore.
explicit AutoContrast(float cutoff = 0.0, std::vector<uint32_t> ignore = {});
/// \brief Destructor.
~AutoContrast() = default;
protected:
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
private:
struct Data;
std::shared_ptr<Data> data_;
};
/// \brief BoundingBoxAugment TensorTransform.
/// \note Apply a given image transform on a random selection of bounding box regions of a given image.
class BoundingBoxAugment final : public TensorTransform {

66
mindspore/ccsrc/minddata/dataset/include/dataset/vision_lite.h
View File

@ -89,39 +89,6 @@ class CenterCrop final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RGB2BGR TensorTransform.
/// \notes Convert the format of input image from RGB to BGR.
class RGB2BGR final : public TensorTransform {
public:
/// \brief Constructor.
RGB2BGR() = default;
/// \brief Destructor.
~RGB2BGR() = default;
protected:
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief RGB2GRAY TensorTransform.
/// \note Convert RGB image or color image to grayscale image.
/// \brief Convert a RGB image or color image to a grayscale one.
class RGB2GRAY final : public TensorTransform {
public:
/// \brief Constructor.
RGB2GRAY() = default;
/// \brief Destructor.
~RGB2GRAY() = default;
protected:
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief Crop an image based on location and crop size.
class Crop final : public TensorTransform {
public:
@ -308,6 +275,39 @@ class ResizePreserveAR final : public TensorTransform {
std::shared_ptr<Data> data_;
};
/// \brief RGB2BGR TensorTransform.
/// \notes Convert the format of input image from RGB to BGR.
class RGB2BGR final : public TensorTransform {
public:
/// \brief Constructor.
RGB2BGR() = default;
/// \brief Destructor.
~RGB2BGR() = default;
protected:
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief RGB2GRAY TensorTransform.
/// \note Convert RGB image or color image to grayscale image.
/// \brief Convert a RGB image or color image to a grayscale one.
class RGB2GRAY final : public TensorTransform {
public:
/// \brief Constructor.
RGB2GRAY() = default;
/// \brief Destructor.
~RGB2GRAY() = default;
protected:
/// \brief The function to convert a TensorTransform object into a TensorOperation object.
/// \return Shared pointer to TensorOperation object.
std::shared_ptr<TensorOperation> Parse() override;
};
/// \brief Rotate the input image according to parameters.
class Rotate final : public TensorTransform {
public:

6
mindspore/ccsrc/minddata/dataset/kernels/image/adjust_gamma_op.cc
View File

@ -15,21 +15,21 @@
*/
#include "minddata/dataset/kernels/image/adjust_gamma_op.h"
#include <memory>
#include "minddata/dataset/kernels/data/data_utils.h"
#include "minddata/dataset/kernels/image/image_utils.h"
namespace mindspore {
namespace dataset {
const float AdjustGammaOp::kGain = 1.0;
constexpr float AdjustGammaOp::kGain = 1.0;
Status AdjustGammaOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
// typecast
CHECK_FAIL_RETURN_UNEXPECTED(input->type() != DataType::DE_STRING,
"AdjustGamma: input tensor type should be [int, float, double], but got string.");
"AdjustGamma: input tensor type should be int, float or double, but got: string.");
if (input->type().IsFloat()) {
std::shared_ptr<Tensor> input_tensor;

4
mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc
View File

@ -877,7 +877,7 @@ Status AdjustGamma(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor>
try {
int num_channels = 1;
if (input->Rank() < 2) {
RETURN_STATUS_UNEXPECTED("AdjustGamma: image shape is not <...,H,W,C> or <H,W>.");
RETURN_STATUS_UNEXPECTED("AdjustGamma: input tensor is not in shape of <...,H,W,C> or <H,W>.");
}
if (input->Rank() > 2) {
num_channels = input->shape()[-1];
@ -1261,7 +1261,7 @@ Status RgbToBgr(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *o
RETURN_STATUS_UNEXPECTED("RgbToBgr: load image failed.");
}
if (input_cv->Rank() != 3 || input_cv->shape()[2] != 3) {
RETURN_STATUS_UNEXPECTED("RgbToBgr: image shape is not <H,W,C> or channel is not 3.");
RETURN_STATUS_UNEXPECTED("RgbToBgr: input tensor is not in shape of <H,W,C> or channel is not 3.");
}
cv::Mat image = input_cv->mat().clone();

2
mindspore/ccsrc/minddata/dataset/kernels/image/rgb_to_bgr_op.h
View File

@ -17,8 +17,8 @@
#define MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_RGB_TO_BGR_OP_H_
#include <memory>
#include <vector>
#include <string>
#include <vector>
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/kernels/tensor_op.h"

2
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/adjust_gamma_ir.cc
View File

@ -13,8 +13,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <algorithm>
#include "minddata/dataset/kernels/ir/vision/adjust_gamma_ir.h"
#ifndef ENABLE_ANDROID

4
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/rgb_to_bgr_ir.cc
View File

@ -13,14 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <algorithm>
#include "minddata/dataset/kernels/ir/vision/rgb_to_bgr_ir.h"
#include "minddata/dataset/kernels/image/rgb_to_bgr_op.h"
#include "minddata/dataset/kernels/ir/validators.h"
namespace mindspore {
namespace dataset {

2
mindspore/dataset/audio/__init__.py
View File

@ -11,7 +11,9 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
This module is to support audio augmentations.
"""
from . import transforms
from . import utils

198
mindspore/dataset/audio/transforms.py
View File

@ -11,14 +11,17 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
The module audio.transforms is inherited from _c_dataengine.
and is implemented based on C++. It's a high performance module to
process audio. Users can apply suitable augmentations on audio data
to improve their training models.
The module audio.transforms is inherited from _c_dataengine and is
implemented based on C++. It's a high performance module to process
audio. Users can apply suitable augmentations on audio data to improve
their training models.
"""
import mindspore._c_dataengine as cde
import numpy as np
import mindspore._c_dataengine as cde
from ..transforms.c_transforms import TensorOperation
from .utils import ScaleType
from .validators import check_allpass_biquad, check_amplitude_to_db, check_band_biquad, check_bandpass_biquad, \
@ -33,38 +36,31 @@ class AudioTensorOperation(TensorOperation):
def __call__(self, *input_tensor_list):
for tensor in input_tensor_list:
if not isinstance(tensor, (np.ndarray,)):
raise TypeError(
"Input should be NumPy audio, got {}.".format(type(tensor)))
raise TypeError("Input should be NumPy audio, got {}.".format(type(tensor)))
return super().__call__(*input_tensor_list)
def parse(self):
raise NotImplementedError(
"AudioTensorOperation has to implement parse() method.")
raise NotImplementedError("AudioTensorOperation has to implement parse() method.")
class AllpassBiquad(AudioTensorOperation):
"""
Design two-pole all-pass filter for audio waveform of dimension of `(..., time)`
Design two-pole all-pass filter for audio waveform of dimension of (..., time).
Args:
sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz),
the value must be greater than 0 .
central_freq (float): central frequency (in Hz),
the value must be greater than 0 .
Q(float, optional): Quality factor,https://en.wikipedia.org/wiki/Q_factor,
Range: (0, 1] (Default=0.707).
Args:
sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz), the value must be greater than 0.
central_freq (float): central frequency (in Hz), the value must be greater than 0.
Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707).
Examples:
>>> import mindspore.dataset.audio.transforms as audio
>>> import numpy as np
>>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03],[9.246826171875e-03, 1.0894775390625e-02]])
>>> allpasspass_biquad_op = audio.AllpassBiquad(44100, 200.0)
>>> waveform_filtered = allpass_biquad_op(waveform)
References:
https://www.w3.org/2011/audio/audio-eq-cookbook.html#APF
Examples:
>>> import numpy as np
>>>
>>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]])
>>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"])
>>> transforms = [audio.AllpassBiquad(44100, 200.0)]
>>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"])
"""
@check_allpass_biquad
def __init__(self, sample_rate, central_freq, Q=0.707):
self.sample_rate = sample_rate
@ -84,23 +80,22 @@ class AmplitudeToDB(AudioTensorOperation):
Converts the input tensor from amplitude/power scale to decibel scale.
Args:
stype (ScaleType, optional): Scale of the input tensor. (Default="ScaleType.POWER").
It can be any of [ScaleType.MAGNITUDE, ScaleType.POWER].
stype (ScaleType, optional): Scale of the input tensor (default=ScaleType.POWER).
It can be one of ScaleType.MAGNITUDE or ScaleType.POWER.
ref_value (float, optional): Param for generate db_multiplier.
amin (float, optional): Lower bound to clamp the input waveform.
top_db (float, optional): Minimum cut-off decibels. The range of values is non-negative. Commonly set at 80.
(Default=80.0)
top_db (float, optional): Minimum cut-off decibels. The range of values is non-negative.
Commonly set at 80 (default=80.0).
Examples:
>>> channel = 1
>>> n_fft = 400
>>> n_frame = 30
>>> specrogram = np.random.random([channel, n_fft//2+1, n_frame])
>>> numpy_slices_dataset = ds.NumpySlicesDataset(data=specrogram, column_names=["audio"])
>>> import numpy as np
>>>
>>> waveform = np.random.random([1, 400//2+1, 30])
>>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"])
>>> transforms = [audio.AmplitudeToDB(stype=ScaleType.POWER)]
>>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"])
"""
@ check_amplitude_to_db
@check_amplitude_to_db
def __init__(self, stype=ScaleType.POWER, ref_value=1.0, amin=1e-10, top_db=80.0):
self.stype = stype
self.ref_value = ref_value
@ -115,15 +110,14 @@ class Angle(AudioTensorOperation):
"""
Calculate the angle of the complex number sequence of shape (..., 2).
The first dimension represents the real part while the second represents the imaginary.
Args:
Examples:
>>> import mindspore.dataset.audio.transforms as audio
>>> import numpy as np
>>> input_complex = np.array([[1.43, 5.434], [23.54, 89.38]])
>>> angle_op = audio.Angle()
>>> angles = angle_op(input_complex)
>>>
>>> waveform = np.array([[1.43, 5.434], [23.54, 89.38]])
>>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"])
>>> transforms = [audio.Angle()]
>>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"])
"""
def parse(self):
@ -132,24 +126,24 @@ class Angle(AudioTensorOperation):
class BandBiquad(AudioTensorOperation):
"""
Design two-pole band filter for audio waveform of dimension of `(..., time)`
Design two-pole band filter for audio waveform of dimension of (..., time).
Args:
sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz), the value can't be zero.
central_freq (float): central frequency (in Hz),
Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, Range: (0, 1] (Default=0.707).
noise (bool, optional) : If ``True``, uses the alternate mode for un-pitched audio (e.g. percussion).
If ``False``, uses mode oriented to pitched audio, i.e. voice, singing,
or instrumental music (Default: ``False``).
sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz), the value can't be zero.
central_freq (float): Central frequency (in Hz).
Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707).
noise (bool, optional) : If True, uses the alternate mode for un-pitched audio (e.g. percussion).
If False, uses mode oriented to pitched audio, i.e. voice, singing, or instrumental music (default=False).
Examples:
>>> import mindspore.dataset.audio.transforms as audio
>>> import numpy as np
>>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03],[9.246826171875e-03, 1.0894775390625e-02]])
>>> band_biquad_op = audio.BandBiquad(44100, 200.0)
>>> waveform_filtered = band_biquad_op(waveform)
>>>
>>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]])
>>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"])
>>> transforms = [audio.BandBiquad(44100, 200.0)]
>>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"])
"""
@check_band_biquad
def __init__(self, sample_rate, central_freq, Q=0.707, noise=False):
self.sample_rate = sample_rate
@ -161,25 +155,26 @@ class BandBiquad(AudioTensorOperation):
return cde.BandBiquadOperation(self.sample_rate, self.central_freq, self.Q, self.noise)
class BandpassBiquad(TensorOperation):
class BandpassBiquad(AudioTensorOperation):
"""
Design two-pole band-pass filter. Similar to SoX implementation.
Design two-pole band-pass filter. Similar to SoX implementation.
Args:
sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz)
central_freq (float): central frequency (in Hz)
Q (float, optional): https://en.wikipedia.org/wiki/Q_factor Range: (0,1] (Default=0.707).
const_skirt_gain (bool, optional) : If ``True``, uses a constant skirt gain (peak gain = Q).
If ``False``, uses a constant 0dB peak gain. (Default: ``False``)
sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz).
central_freq (float): Central frequency (in Hz).
Q (float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0,1] (default=0.707).
const_skirt_gain (bool, optional) : If True, uses a constant skirt gain (peak gain = Q).
If False, uses a constant 0dB peak gain (default=False).
Examples:
>>> import mindspore.dataset.audio.transforms as audio
>>> import numpy as np
>>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03],[9.246826171875e-03, 1.0894775390625e-02]])
>>> bandpass_biquad_op = audio.BandpassBiquad(44100, 200.0)
>>> waveform_filtered = bandpass_biquad_op(waveform)
>>>
>>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]])
>>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"])
>>> transforms = [audio.BandpassBiquad(44100, 200.0)]
>>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"])
"""
@check_bandpass_biquad
def __init__(self, sample_rate, central_freq, Q=0.707, const_skirt_gain=False):
self.sample_rate = sample_rate
@ -193,23 +188,20 @@ class BandpassBiquad(TensorOperation):
class BandrejectBiquad(AudioTensorOperation):
"""
Design two-pole band filter for audio waveform of dimension of `(..., time)`
Design two-pole band filter for audio waveform of dimension of (..., time).
Args:
sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz),
the value must be greater than 0 .
central_freq (float): central frequency (in Hz),
the value must be greater than 0 .
Q(float, optional): Quality factor,https://en.wikipedia.org/wiki/Q_factor,
Range: (0, 1] (Default=0.707).
sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz), the value must be greater than 0.
central_freq (float): central frequency (in Hz), the value must be greater than 0.
Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707).
Examples:
>>> import mindspore.dataset.audio.transforms as audio
>>> import numpy as np
>>>
>>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03],[9.246826171875e-03, 1.0894775390625e-02]])
>>> band_biquad_op = audio.BandBiquad(44100, 200.0)
>>> waveform_filtered = band_biquad_op(waveform)
>>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"])
>>> transforms = [audio.BandrejectBiquad(44100, 200.0)]
>>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"])
"""
@check_bandreject_biquad
@ -224,22 +216,23 @@ class BandrejectBiquad(AudioTensorOperation):
class BassBiquad(AudioTensorOperation):
"""
Design a bass tone-control effect for audio waveform of dimension of `(..., time)`
Design a bass tone-control effect for audio waveform of dimension of (..., time).
Args:
sample_rate (int): sampling rate of the waveform, e.g. 44100 (Hz)
gain (float): desired gain at the boost (or attenuation) in dB.
central_freq (float): central frequency (in Hz)(Default=100.0).
Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, Range: (0, 1] (Default=0.707).
sample_rate (int): Sampling rate of the waveform, e.g. 44100 (Hz).
gain (float): Desired gain at the boost (or attenuation) in dB.
central_freq (float): Central frequency (in Hz) (default=100.0).
Q(float, optional): Quality factor, https://en.wikipedia.org/wiki/Q_factor, range: (0, 1] (default=0.707).
Examples:
>>> import mindspore.dataset.audio.transforms as audio
>>> import numpy as np
>>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03],[9.246826171875e-03, 1.0894775390625e-02]])
>>> bass_biquad_op = audio.BassBiquad(44100, 100.0)
>>> waveform_filtered = bass_biquad_op(waveform)
>>>
>>> waveform = np.array([[2.716064453125e-03, 6.34765625e-03], [9.246826171875e-03, 1.0894775390625e-02]])
>>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"])
>>> transforms = [audio.BassBiquad(44100, 100.0)]
>>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"])
"""
@check_bass_biquad
def __init__(self, sample_rate, gain, central_freq=100.0, Q=0.707):
self.sample_rate = sample_rate
@ -263,14 +256,12 @@ class FrequencyMasking(AudioTensorOperation):
mask_value (double): Mask value (default=0.0).
Examples:
>>> def gen():
... random.seed(0)
... data = numpy.random.random([1, 3, 2])
... yield (numpy.array(data, dtype=numpy.float32),)
>>> dataset = ds.GeneratorDataset(source=gen,
... column_names=["multi_dim_data"])
>>> dataset = dataset.map(operations=FrequencyMasking(frequency_mask_param=1),
... input_columns=["multi_dim_data"])
>>> import numpy as np
>>>
>>> waveform = np.random.random([1, 3, 2])
>>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"])
>>> transforms = [audio.FrequencyMasking(frequency_mask_param=1)]
>>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"])
"""
@check_masking
def __init__(self, iid_masks=False, frequency_mask_param=0, mask_start=0, mask_value=0.0):
@ -296,15 +287,14 @@ class TimeMasking(AudioTensorOperation):
mask_value (double): Mask value (default=0.0).
Examples:
>>> def gen():
... random.seed(0)
... data = numpy.random.random([1, 3, 2])
... yield (numpy.array(data, dtype=numpy.float32),)
>>> dataset = ds.GeneratorDataset(source=gen,
... column_names=["multi_dim_data"])
>>> dataset = dataset.map(operations=TimeMasking(time_mask_param=1),
... input_columns=["multi_dim_data"])
>>> import numpy as np
>>>
>>> waveform = np.random.random([1, 3, 2])
>>> numpy_slices_dataset = ds.NumpySlicesDataset(data=waveform, column_names=["audio"])
>>> transforms = [audio.TimeMasking(time_mask_param=1)]
>>> numpy_slices_dataset = numpy_slices_dataset.map(operations=transforms, input_columns=["audio"])
"""
@check_masking
def __init__(self, iid_masks=False, time_mask_param=0, mask_start=0, mask_value=0.0):
self.iid_masks = iid_masks

2
mindspore/dataset/audio/utils.py
View File

@ -11,9 +11,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
enum for audio ops
"""
from enum import Enum

86
mindspore/dataset/audio/validators.py
View File

@ -15,15 +15,17 @@
"""
Validators for TensorOps.
"""
from functools import wraps
from mindspore.dataset.core.validator_helpers import check_not_zero, check_int32, check_float32, check_value, \
check_value_normalize_std, check_value_ratio, DOUBLE_MAX_INTEGER, FLOAT_MAX_INTEGER, INT64_MAX, parse_user_args, \
type_check
from mindspore.dataset.core.validator_helpers import check_float32, check_int32_not_zero, \
check_non_negative_float32, check_non_negative_float64, check_pos_float32, check_pos_int64, check_value, \
parse_user_args, type_check
from .utils import ScaleType
def check_amplitude_to_db(method):
"""Wrapper method to check the parameters of amplitude_to_db."""
"""Wrapper method to check the parameters of AmplitudeToDB."""
@wraps(method)
def new_method(self, *args, **kwargs):
@ -35,30 +37,30 @@ def check_amplitude_to_db(method):
# type check ref_value
type_check(ref_value, (int, float), "ref_value")
# value check ref_value
if not ref_value is None:
check_value_ratio(ref_value, (0, FLOAT_MAX_INTEGER), "ref_value")
if ref_value is not None:
check_pos_float32(ref_value, "ref_value")
# type check amin
type_check(amin, (int, float), "amin")
# value check amin
if not amin is None:
check_value_ratio(amin, (0, FLOAT_MAX_INTEGER), "amin")
if amin is not None:
check_pos_float32(amin, "amin")
# type check top_db
type_check(top_db, (int, float), "top_db")
# value check top_db
if not top_db is None:
check_value_ratio(top_db, (0, FLOAT_MAX_INTEGER), "top_db")
if top_db is not None:
check_pos_float32(top_db, "top_db")
return method(self, *args, **kwargs)
return new_method
def check_biquad_sample_rate(sample_rate):
"""Wrapper method to check the parameters of sample_rate."""
type_check(sample_rate, (int,), "sample_rate")
check_int32(sample_rate, "sample_rate")
check_not_zero(sample_rate, "sample_rate")
check_int32_not_zero(sample_rate, "sample_rate")
def check_biquad_central_freq(central_freq):
@ -70,7 +72,7 @@ def check_biquad_central_freq(central_freq):
def check_biquad_Q(Q):
"""Wrapper method to check the parameters of Q."""
type_check(Q, (float, int), "Q")
check_value_normalize_std(Q, [0, 1], "Q")
check_value(Q, [0, 1], "Q", True)
def check_biquad_noise(noise):
@ -106,7 +108,7 @@ def check_band_biquad(method):
def check_allpass_biquad(method):
"""Wrapper method to check the parameters of CutMixBatch."""
"""Wrapper method to check the parameters of AllpassBiquad."""
@wraps(method)
def new_method(self, *args, **kwargs):
@ -123,7 +125,7 @@ def check_allpass_biquad(method):
def check_bandpass_biquad(method):
"""Wrapper method to check the parameters of BandpassBiquad."""
@ wraps(method)
@wraps(method)
def new_method(self, *args, **kwargs):
[sample_rate, central_freq, Q, const_skirt_gain], _ = parse_user_args(
method, *args, **kwargs)
@ -152,7 +154,7 @@ def check_bandreject_biquad(method):
def check_bass_biquad(method):
"""Wrapper method to check the parameters of CutMixBatch."""
"""Wrapper method to check the parameters of BassBiquad."""
@wraps(method)
def new_method(self, *args, **kwargs):
@ -167,8 +169,30 @@ def check_bass_biquad(method):
return new_method
def check_time_stretch(method):
"""Wrapper method to check the parameters of TimeStretch."""
@wraps(method)
def new_method(self, *args, **kwargs):
[hop_length, n_freq, fixed_rate], _ = parse_user_args(method, *args, **kwargs)
if hop_length is not None:
type_check(hop_length, (int,), "hop_length")
check_pos_int64(hop_length, "hop_length")
type_check(n_freq, (int,), "n_freq")
check_pos_int64(n_freq, "n_freq")
if fixed_rate is not None:
type_check(fixed_rate, (int, float), "fixed_rate")
check_pos_float32(fixed_rate, "fixed_rate")
return method(self, *args, **kwargs)
return new_method
def check_masking(method):
"""Wrapper method to check the parameters of time_masking and frequency_masking"""
"""Wrapper method to check the parameters of time_masking and FrequencyMasking"""
@wraps(method)
def new_method(self, *args, **kwargs):
@ -176,33 +200,11 @@ def check_masking(method):
method, *args, **kwargs)
type_check(iid_masks, (bool,), "iid_masks")
type_check(mask_param, (int,), "mask_param")
check_value(mask_param, (0, FLOAT_MAX_INTEGER), "mask_param")
check_non_negative_float32(mask_param, "mask_param")
type_check(mask_start, (int,), "mask_start")
check_value(mask_start, (0, FLOAT_MAX_INTEGER), "mask_start")
check_non_negative_float32(mask_start, "mask_start")
type_check(mask_value, (int, float), "mask_value")
check_value(mask_value, (0, DOUBLE_MAX_INTEGER), "mask_value")
return method(self, *args, **kwargs)
return new_method
def check_time_stretch(method):
"""Wrapper method to check the parameters of time_stretch."""
@wraps(method)
def new_method(self, *args, **kwargs):
[hop_length, n_freq, fixed_rate], _ = parse_user_args(method, *args, **kwargs)
# type check
type_check(hop_length, (int, type(None)), "hop_length")
type_check(n_freq, (int,), "n_freq")
type_check(fixed_rate, (int, float, type(None)), "fixed_rate")
# value check
if hop_length is not None:
check_value(hop_length, (1, INT64_MAX), "hop_length")
check_value(n_freq, (1, INT64_MAX), "n_freq")
if fixed_rate is not None:
check_value_ratio(fixed_rate, (0, FLOAT_MAX_INTEGER), "fixed_rate")
check_non_negative_float64(mask_value, "mask_value")
return method(self, *args, **kwargs)
return new_method

98
mindspore/dataset/core/validator_helpers.py
View File

@ -92,20 +92,38 @@ def pad_arg_name(arg_name):
return arg_name
def check_value(value, valid_range, arg_name=""):
def check_value(value, valid_range, arg_name="", left_open_interval=False, right_open_interval=False):
"""
Validates a value is within a desired range [inclusive, inclusive].
Validates a value is within a desired range with left and right interval open or close.
:param value: the value to be validated
:param valid_range: the desired range
:param arg_name: arg_name: arg_name: name of the variable to be validated
:param value: the value to be validated.
:param valid_range: the desired range.
:param arg_name: name of the variable to be validated.
:param left_open_interval: True for left interval open and False for close.
:param right_open_interval: True for right interval open and False for close.
:return: Exception: when the validation fails, nothing otherwise.
"""
arg_name = pad_arg_name(arg_name)
if value < valid_range[0] or value > valid_range[1]:
raise ValueError(
"Input {0}is not within the required interval of [{1}, {2}].".format(arg_name, valid_range[0],
valid_range[1]))
if not left_open_interval and not right_open_interval:
if value < valid_range[0] or value > valid_range[1]:
raise ValueError(
"Input {0}is not within the required interval of [{1}, {2}].".format(arg_name, valid_range[0],
valid_range[1]))
elif left_open_interval and not right_open_interval:
if value <= valid_range[0] or value > valid_range[1]:
raise ValueError(
"Input {0}is not within the required interval of ({1}, {2}].".format(arg_name, valid_range[0],
valid_range[1]))
elif not left_open_interval and right_open_interval:
if value < valid_range[0] or value >= valid_range[1]:
raise ValueError(
"Input {0}is not within the required interval of [{1}, {2}).".format(arg_name, valid_range[0],
valid_range[1]))
else:
if value <= valid_range[0] or value >= valid_range[1]:
raise ValueError(
"Input {0}is not within the required interval of ({1}, {2}).".format(arg_name, valid_range[0],
valid_range[1]))
def check_value_cutoff(value, valid_range, arg_name=""):
@ -117,11 +135,7 @@ def check_value_cutoff(value, valid_range, arg_name=""):
:param arg_name: arg_name: arg_name: name of the variable to be validated
:return: Exception: when the validation fails, nothing otherwise.
"""
arg_name = pad_arg_name(arg_name)
if value < valid_range[0] or value >= valid_range[1]:
raise ValueError(
"Input {0}is not within the required interval of [{1}, {2}).".format(arg_name, valid_range[0],
valid_range[1]))
check_value(value, valid_range, arg_name, False, True)
def check_value_ratio(value, valid_range, arg_name=""):
@ -133,11 +147,7 @@ def check_value_ratio(value, valid_range, arg_name=""):
:param arg_name: arg_name: name of the variable to be validated
:return: Exception: when the validation fails, nothing otherwise.
"""
arg_name = pad_arg_name(arg_name)
if value <= valid_range[0] or value > valid_range[1]:
raise ValueError(
"Input {0}is not within the required interval of ({1}, {2}].".format(arg_name, valid_range[0],
valid_range[1]))
check_value(value, valid_range, arg_name, True, False)
def check_value_normalize_std(value, valid_range, arg_name=""):
@ -149,11 +159,7 @@ def check_value_normalize_std(value, valid_range, arg_name=""):
:param arg_name: arg_name: name of the variable to be validated
:return: Exception: when the validation fails, nothing otherwise.
"""
arg_name = pad_arg_name(arg_name)
if value <= valid_range[0] or value > valid_range[1]:
raise ValueError(
"Input {0}is not within the required interval of ({1}, {2}].".format(arg_name, valid_range[0],
valid_range[1]))
check_value(value, valid_range, arg_name, True, False)
def check_range(values, valid_range, arg_name=""):
@ -185,10 +191,12 @@ def check_positive(value, arg_name=""):
raise ValueError("Input {0}must be greater than 0.".format(arg_name))
def check_not_zero(value, arg_name=""):
def check_int32_not_zero(value, arg_name=""):
arg_name = pad_arg_name(arg_name)
if value == 0:
raise ValueError("Input {0}can not be 0.".format(arg_name))
type_check(value, (int,), arg_name)
if value < INT32_MIN or value > INT32_MAX or value == 0:
raise ValueError(
"Input {0}is not within the required interval of [-2147483648, 0) and (0, 2147483647].".format(arg_name))
def check_odd(value, arg_name=""):
@ -211,6 +219,13 @@ def check_2tuple(value, arg_name=""):
def check_int32(value, arg_name=""):
"""
Validates the value of a variable is within the range of int32.
:param value: the value of the variable
:param arg_name: name of the variable to be validated
:return: Exception: when the validation fails, nothing otherwise.
"""
type_check(value, (int,), arg_name)
check_value(value, [INT32_MIN, INT32_MAX], arg_name)
@ -284,7 +299,7 @@ def check_pos_int64(value, arg_name=""):
:return: Exception: when the validation fails, nothing otherwise.
"""
type_check(value, (int,), arg_name)
check_value(value, [UINT64_MIN, INT64_MAX])
check_value(value, [POS_INT_MIN, INT64_MAX])
def check_float32(value, arg_name=""):
@ -317,7 +332,7 @@ def check_pos_float32(value, arg_name=""):
:param arg_name: name of the variable to be validated
:return: Exception: when the validation fails, nothing otherwise.
"""
check_value(value, [UINT32_MIN, FLOAT_MAX_INTEGER], arg_name)
check_value(value, [UINT32_MIN, FLOAT_MAX_INTEGER], arg_name, True)
def check_pos_float64(value, arg_name=""):
@ -328,7 +343,29 @@ def check_pos_float64(value, arg_name=""):
:param arg_name: name of the variable to be validated
:return: Exception: when the validation fails, nothing otherwise.
"""
check_value(value, [UINT64_MIN, DOUBLE_MAX_INTEGER], arg_name)
check_value(value, [UINT64_MIN, DOUBLE_MAX_INTEGER], arg_name, True)
def check_non_negative_float32(value, arg_name=""):
"""
Validates the value of a variable is within the range of non negative float32.
:param value: the value of the variable
:param arg_name: name of the variable to be validated
:return: Exception: when the validation fails, nothing otherwise.
"""
check_value(value, [UINT32_MIN, FLOAT_MAX_INTEGER], arg_name)
def check_non_negative_float64(value, arg_name=""):
"""
Validates the value of a variable is within the range of non negative float64.
:param value: the value of the variable
:param arg_name: name of the variable to be validated
:return: Exception: when the validation fails, nothing otherwise.
"""
check_value(value, [UINT32_MIN, DOUBLE_MAX_INTEGER], arg_name)
def check_valid_detype(type_):
@ -680,4 +717,3 @@ def check_c_tensor_op(param, param_name):
def replace_none(value, default):
""" replaces None with a default value."""
return value if value is not None else default

174
mindspore/dataset/engine/datasets.py
View File

@ -5773,62 +5773,74 @@ class FlickrDataset(MappableDataset):
>>> # In FLICKR dataset, each dictionary has keys "image" and "annotation"
About Flickr8k dataset:
| The Flickr8k dataset consists of 8092 colour images. There are 40460 annotations in the Flickr8k.token.txt,
each image has 5 annotations.
| You can unzip the dataset files into the following directory structure and read by MindSpore's API.
| .
| └── Flickr8k
| ├── Flickr8k_Dataset
| | ├── 1000268201_693b08cb0e.jpg
| | ├── 1001773457_577c3a7d70.jpg
| | ├── ...
| └── Flickr8k.token.txt
The Flickr8k dataset consists of 8092 colour images. There are 40460 annotations in the Flickr8k.token.txt,
each image has 5 annotations.
.. code-block::
You can unzip the dataset files into the following directory structure and read by MindSpore's API.
@article{DBLP:journals/jair/HodoshYH13,
author = {Micah Hodosh and Peter Young and Julia Hockenmaier},
title = {Framing Image Description as a Ranking Task: Data, Models and Evaluation Metrics},
journal = {J. Artif. Intell. Res.},
volume = {47},
pages = {853--899},
year = {2013},
url = {https://doi.org/10.1613/jair.3994},
doi = {10.1613/jair.3994},
timestamp = {Mon, 21 Jan 2019 15:01:17 +0100},
biburl = {https://dblp.org/rec/journals/jair/HodoshYH13.bib},
bibsource = {dblp computer science bibliography, https://dblp.org}
}
.. code-block::
.
└── Flickr8k
├── Flickr8k_Dataset
│ ├── 1000268201_693b08cb0e.jpg
│ ├── 1001773457_577c3a7d70.jpg
│ ├── ...
└── Flickr8k.token.txt
Citation:
.. code-block::
@article{DBLP:journals/jair/HodoshYH13,
author = {Micah Hodosh and Peter Young and Julia Hockenmaier},
title = {Framing Image Description as a Ranking Task: Data, Models and Evaluation Metrics},
journal = {J. Artif. Intell. Res.},
volume = {47},
pages = {853--899},
year = {2013},
url = {https://doi.org/10.1613/jair.3994},
doi = {10.1613/jair.3994},
timestamp = {Mon, 21 Jan 2019 15:01:17 +0100},
biburl = {https://dblp.org/rec/journals/jair/HodoshYH13.bib},
bibsource = {dblp computer science bibliography, https://dblp.org}
}
About Flickr30k dataset:
| The Flickr30k dataset consists of 31783 colour images. There are 158915 annotations in
the results_20130124.token, each image has 5 annotations.
| You can unzip the dataset files into the following directory structure and read by MindSpore's API.
| .
| └── Flickr30k
| ├── flickr30k-images
| | ├── 1000092795.jpg
| | ├── 10002456.jpg
| | ├── ...
| └── results_20130124.token
The Flickr30k dataset consists of 31783 colour images. There are 158915 annotations in
the results_20130124.token, each image has 5 annotations.
.. code-block::
You can unzip the dataset files into the following directory structure and read by MindSpore's API.
@article{DBLP:journals/tacl/YoungLHH14,
author = {Peter Young and Alice Lai and Micah Hodosh and Julia Hockenmaier},
title = {From image descriptions to visual denotations: New similarity metrics
for semantic inference over event descriptions},
journal = {Trans. Assoc. Comput. Linguistics},
volume = {2},
pages = {67--78},
year = {2014},
url = {https://tacl2013.cs.columbia.edu/ojs/index.php/tacl/article/view/229},
timestamp = {Wed, 17 Feb 2021 21:55:25 +0100},
biburl = {https://dblp.org/rec/journals/tacl/YoungLHH14.bib},
bibsource = {dblp computer science bibliography, https://dblp.org}
}
Citation:
.. code-block::
.
└── Flickr30k
├── flickr30k-images
│ ├── 1000092795.jpg
│ ├── 10002456.jpg
│ ├── ...
└── results_20130124.token
.. code-block::
@article{DBLP:journals/tacl/YoungLHH14,
author = {Peter Young and Alice Lai and Micah Hodosh and Julia Hockenmaier},
title = {From image descriptions to visual denotations: New similarity metrics
for semantic inference over event descriptions},
journal = {Trans. Assoc. Comput. Linguistics},
volume = {2},
pages = {67--78},
year = {2014},
url = {https://tacl2013.cs.columbia.edu/ojs/index.php/tacl/article/view/229},
timestamp = {Wed, 17 Feb 2021 21:55:25 +0100},
biburl = {https://dblp.org/rec/journals/tacl/YoungLHH14.bib},
bibsource = {dblp computer science bibliography, https://dblp.org}
}
"""
@check_flickr_dataset
@ -5928,39 +5940,43 @@ class SBDataset(GeneratorDataset):
>>>
>>> # In Semantic Boundaries Dataset, each dictionary has keys "image" and "task"
About Semantic Boundaries Dataset.
| The Semantic Boundaries Dataset consists of 11355 colour images. There are 8498 images' name in the train.txt,
2857 images' name in the val.txt and 5623 images' name in the train_noval.txt. The category cls/
contains the Segmentation and Boundaries results of category-level, the category inst/ catains the
Segmentation and Boundaries results of instance-level.
About Semantic Boundaries Dataset:
| You can unzip the dataset files into the following structure and read by MindSpore's API,
| .
| └── benchmark_RELEASE
| └── dataset
| ├── img
| | ├── 2008_000002.jpg
| | ├── 2008_000003.jpg
| | ├── ...
| ├── cls
| | ├── 2008_000002.mat
| | ├── 2008_000003.mat
| | ├── ...
| ├── inst
| | ├── 2008_000002.mat
| | ├── 2008_000003.mat
| | ├── ...
| ├── train.txt
| └── val.txt
The Semantic Boundaries Dataset consists of 11355 colour images. There are 8498 images' name in the train.txt,
2857 images' name in the val.txt and 5623 images' name in the train_noval.txt. The category cls/
contains the Segmentation and Boundaries results of category-level, the category inst/ catains the
Segmentation and Boundaries results of instance-level.
.. code-block::
You can unzip the dataset files into the following structure and read by MindSpore's API:
@InProceedings{BharathICCV2011,
author = "Bharath Hariharan and Pablo Arbelaez and Lubomir Bourdev and
Subhransu Maji and Jitendra Malik",
title = "Semantic Contours from Inverse Detectors",
booktitle = "International Conference on Computer Vision (ICCV)",
year = "2011",
.. code-block::
.
└── benchmark_RELEASE
├── dataset
├── img
│ ├── 2008_000002.jpg
│ ├── 2008_000003.jpg
│ ├── ...
├── cls
│ ├── 2008_000002.mat
│ ├── 2008_000003.mat
│ ├── ...
├── inst
│ ├── 2008_000002.mat
│ ├── 2008_000003.mat
│ ├── ...
├── train.txt
└── val.txt
.. code-block::
@InProceedings{BharathICCV2011,
author = "Bharath Hariharan and Pablo Arbelaez and Lubomir Bourdev and
Subhransu Maji and Jitendra Malik",
title = "Semantic Contours from Inverse Detectors",
booktitle = "International Conference on Computer Vision (ICCV)",
year = "2011",
"""
@check_sb_dataset
@ -5972,7 +5988,7 @@ class SBDataset(GeneratorDataset):
num_shards=num_shards, shard_id=shard_id)
class _SBDataset():
class _SBDataset:
"""
Dealing with the data file with .mat extension, and return one row in tuple (image, task) each time.
"""

3
mindspore/dataset/vision/c_transforms.py
View File

@ -109,7 +109,7 @@ def parse_padding(padding):
class AdjustGamma(ImageTensorOperation):
r"""
Apply gamma correction on input image. Input image is expected to be in [..., H, W, C] or [H, W, C] format.
Apply gamma correction on input image. Input image is expected to be in [..., H, W, C] or [H, W] format.
.. math::
I_{\text{out}} = 255 \times \text{gain} \times \left(\frac{I_{\text{in}}}{255}\right)^{\gamma}
@ -1511,6 +1511,7 @@ class RgbToBgr(ImageTensorOperation):
Examples:
>>> from mindspore.dataset.vision import Inter
>>>
>>> decode_op = c_vision.Decode()
>>> rgb2bgr_op = c_vision.RgbToBgr()
>>> transforms_list = [decode_op, rgb2bgr_op]

1
mindspore/dataset/vision/utils.py
View File

@ -43,6 +43,7 @@ class ImageBatchFormat(IntEnum):
NHWC = 0
NCHW = 1
class SliceMode(IntEnum):
PAD = 0
DROP = 1

4
mindspore/dataset/vision/validators.py
View File

@ -22,7 +22,7 @@ from mindspore._c_dataengine import TensorOp, TensorOperation
from mindspore.dataset.core.validator_helpers import check_value, check_uint8, FLOAT_MIN_INTEGER, FLOAT_MAX_INTEGER, \
check_pos_float32, check_float32, check_2tuple, check_range, check_positive, INT32_MAX, INT32_MIN, \
parse_user_args, type_check, type_check_list, check_c_tensor_op, UINT8_MAX, check_value_normalize_std, \
check_value_cutoff, check_value_ratio, check_odd
check_value_cutoff, check_value_ratio, check_odd, check_non_negative_float32
from .utils import Inter, Border, ImageBatchFormat, SliceMode
@ -143,7 +143,7 @@ def check_degrees(degrees):
"""Check if the degrees is legal."""
type_check(degrees, (int, float, list, tuple), "degrees")
if isinstance(degrees, (int, float)):
check_pos_float32(degrees, "degrees")
check_non_negative_float32(degrees, "degrees")
elif isinstance(degrees, (list, tuple)):
if len(degrees) == 2:
type_check_list(degrees, (int, float), "degrees")

1
tests/ut/cpp/dataset/CMakeLists.txt
View File

@ -16,7 +16,6 @@ SET(DE_UT_SRCS
c_api_audio_r_to_z_test.cc
c_api_cache_test.cc
c_api_dataset_album_test.cc
c_api_audio_a_to_q_test.cc
c_api_dataset_cifar_test.cc
c_api_dataset_clue_test.cc
c_api_dataset_coco_test.cc

55
tests/ut/cpp/dataset/c_api_audio_a_to_q_test.cc
View File

@ -13,13 +13,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "common/common.h"
#include "include/api/types.h"
#include "utils/log_adapter.h"
#include "minddata/dataset/include/dataset/audio.h"
#include "minddata/dataset/include/dataset/datasets.h"
#include "minddata/dataset/include/dataset/execute.h"
#include "minddata/dataset/include/dataset/transforms.h"
using namespace mindspore::dataset;
@ -33,7 +33,7 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
};
TEST_F(MindDataTestPipeline, TestAmplitudeToDBPipeline) {
MS_LOG(INFO) << "Basic Function Test";
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAmplitudeToDBPipeline.";
// Original waveform
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200}));
@ -71,7 +71,7 @@ TEST_F(MindDataTestPipeline, TestAmplitudeToDBPipeline) {
}
TEST_F(MindDataTestPipeline, TestAmplitudeToDBWrongArgs) {
MS_LOG(INFO) << "Basic Function Test";
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAmplitudeToDBWrongArgs.";
// Original waveform
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200}));
@ -91,8 +91,8 @@ TEST_F(MindDataTestPipeline, TestAmplitudeToDBWrongArgs) {
EXPECT_EQ(iter, nullptr);
}
TEST_F(MindDataTestPipeline, Level0_TestBandBiquad001) {
MS_LOG(INFO) << "Basic Function Test";
TEST_F(MindDataTestPipeline, TestBandBiquadBasic) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBandBiquadBasic.";
// Original waveform
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200}));
@ -130,8 +130,8 @@ TEST_F(MindDataTestPipeline, Level0_TestBandBiquad001) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, Level0_TestBandBiquad002) {
MS_LOG(INFO) << "Wrong Arg.";
TEST_F(MindDataTestPipeline, TestBandBiquadParamCheck) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBandBiquadParamCheck.";
std::shared_ptr<SchemaObj> schema = Schema();
// Original waveform
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 2}));
@ -159,8 +159,8 @@ TEST_F(MindDataTestPipeline, Level0_TestBandBiquad002) {
EXPECT_EQ(iter02, nullptr);
}
TEST_F(MindDataTestPipeline, Level0_TestAllpassBiquad001) {
MS_LOG(INFO) << "Basic Function Test";
TEST_F(MindDataTestPipeline, TestAllpassBiquadBasic) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAllpassBiquadBasic.";
// Original waveform
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200}));
@ -198,8 +198,8 @@ TEST_F(MindDataTestPipeline, Level0_TestAllpassBiquad001) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, Level0_TestAllpassBiquad002) {
MS_LOG(INFO) << "Wrong Arg.";
TEST_F(MindDataTestPipeline, TestAllpassBiquadParamCheck) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAllpassBiquadParamCheck.";
std::shared_ptr<SchemaObj> schema = Schema();
// Original waveform
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 2}));
@ -227,8 +227,8 @@ TEST_F(MindDataTestPipeline, Level0_TestAllpassBiquad002) {
EXPECT_EQ(iter02, nullptr);
}
TEST_F(MindDataTestPipeline, Level0_TestBandpassBiquad001) {
MS_LOG(INFO) << "Basic Function Test";
TEST_F(MindDataTestPipeline, TestBandpassBiquadBasic) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBandpassBiquadBasic.";
// Original waveform
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200}));
@ -266,8 +266,8 @@ TEST_F(MindDataTestPipeline, Level0_TestBandpassBiquad001) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, Level0_TestBandpassBiquad002) {
MS_LOG(INFO) << "Wrong Arg.";
TEST_F(MindDataTestPipeline, TestBandpassBiquadParamCheck) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBandpassBiquadParamCheck.";
std::shared_ptr<SchemaObj> schema = Schema();
// Original waveform
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 2}));
@ -295,8 +295,8 @@ TEST_F(MindDataTestPipeline, Level0_TestBandpassBiquad002) {
EXPECT_EQ(iter02, nullptr);
}
TEST_F(MindDataTestPipeline, Level0_TestBandrejectBiquad001) {
MS_LOG(INFO) << "Basic Function Test";
TEST_F(MindDataTestPipeline, TestBandrejectBiquadBasic) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBandrejectBiquadBasic.";
// Original waveform
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200}));
@ -334,8 +334,8 @@ TEST_F(MindDataTestPipeline, Level0_TestBandrejectBiquad001) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, Level0_TestBandrejectBiquad002) {
MS_LOG(INFO) << "Wrong Arg.";
TEST_F(MindDataTestPipeline, TestBandrejectBiquadParamCheck) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBandrejectBiquadParamCheck.";
std::shared_ptr<SchemaObj> schema = Schema();
// Original waveform
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 2}));
@ -363,8 +363,8 @@ TEST_F(MindDataTestPipeline, Level0_TestBandrejectBiquad002) {
EXPECT_EQ(iter02, nullptr);
}
TEST_F(MindDataTestPipeline, Level0_TestBassBiquad001) {
MS_LOG(INFO) << "Basic Function Test";
TEST_F(MindDataTestPipeline, TestBassBiquadBasic) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBassBiquadBasic.";
// Original waveform
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200}));
@ -402,8 +402,8 @@ TEST_F(MindDataTestPipeline, Level0_TestBassBiquad001) {
iter->Stop();
}
TEST_F(MindDataTestPipeline, Level0_TestBassBiquad002) {
MS_LOG(INFO) << "Wrong Arg.";
TEST_F(MindDataTestPipeline, TestBassBiquadParamCheck) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestBassBiquadParamCheck.";
std::shared_ptr<SchemaObj> schema = Schema();
// Original waveform
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 2}));
@ -432,7 +432,7 @@ TEST_F(MindDataTestPipeline, Level0_TestBassBiquad002) {
}
TEST_F(MindDataTestPipeline, TestAnglePipeline) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAnglePipeline";
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAnglePipeline.";
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("complex", mindspore::DataType::kNumberTypeFloat32, {2, 2}));
@ -470,7 +470,7 @@ TEST_F(MindDataTestPipeline, TestAnglePipeline) {
}
TEST_F(MindDataTestPipeline, TestAnglePipelineError) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAnglePipelineError";
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAnglePipelineError.";
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("complex", mindspore::DataType::kNumberTypeFloat32, {3, 2, 1}));
@ -491,7 +491,7 @@ TEST_F(MindDataTestPipeline, TestAnglePipelineError) {
}
TEST_F(MindDataTestPipeline, TestFrequencyMaskingPipeline) {
MS_LOG(INFO) << "Doing TestFrequencyMasking Pipeline.";
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFrequencyMaskingPipeline.";
// Original waveform
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {200, 200}));
@ -506,7 +506,6 @@ TEST_F(MindDataTestPipeline, TestFrequencyMaskingPipeline) {
ds = ds->Map({frequencymasking});
EXPECT_NE(ds, nullptr);
// Filtered waveform by bandbiquad
std::shared_ptr<Iterator> iter = ds->CreateIterator();
EXPECT_NE(ds, nullptr);
@ -530,7 +529,7 @@ TEST_F(MindDataTestPipeline, TestFrequencyMaskingPipeline) {
}
TEST_F(MindDataTestPipeline, TestFrequencyMaskingWrongArgs) {
MS_LOG(INFO) << "Doing TestFrequencyMasking with wrong args.";
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestFrequencyMaskingWrongArgs.";
// Original waveform
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {20, 20}));

11
tests/ut/cpp/dataset/c_api_audio_r_to_z_test.cc
View File

@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "common/common.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/include/dataset/datasets.h"
@ -28,7 +29,7 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
};
TEST_F(MindDataTestPipeline, TestTimeMaskingPipeline) {
MS_LOG(INFO) << "Doing TestTimeMasking Pipeline.";
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTimeMaskingPipeline.";
// Original waveform
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 200}));
@ -67,7 +68,7 @@ TEST_F(MindDataTestPipeline, TestTimeMaskingPipeline) {
}
TEST_F(MindDataTestPipeline, TestTimeMaskingWrongArgs) {
MS_LOG(INFO) << "Doing TestTimeMasking with wrong args.";
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTimeMaskingWrongArgs.";
// Original waveform
std::shared_ptr<SchemaObj> schema = Schema();
ASSERT_OK(schema->add_column("inputData", mindspore::DataType::kNumberTypeFloat32, {2, 20}));
@ -89,7 +90,7 @@ TEST_F(MindDataTestPipeline, TestTimeMaskingWrongArgs) {
}
TEST_F(MindDataTestPipeline, TestTimeStretchPipeline) {
MS_LOG(INFO) << "Doing test TimeStretchOp with custom param value. Pipeline.";
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTimeStretchPipeline.";
// op param
int freq = 1025;
int hop_length = 512;
@ -115,7 +116,7 @@ TEST_F(MindDataTestPipeline, TestTimeStretchPipeline) {
std::unordered_map<std::string, mindspore::MSTensor> row;
ASSERT_OK(iter->GetNextRow(&row));
std::vector<int64_t> expected = {2, freq, int(std::ceil(400 / rate)), 2};
std::vector<int64_t> expected = {2, freq, static_cast<int64_t>(std::ceil(400 / rate)), 2};
int i = 0;
while (row.size() != 0) {
@ -131,7 +132,7 @@ TEST_F(MindDataTestPipeline, TestTimeStretchPipeline) {
}
TEST_F(MindDataTestPipeline, TestTimeStretchPipelineWrongArgs) {
MS_LOG(INFO) << "Doing test TimeStretchOp with wrong param value. Pipeline.";
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestTimeStretchPipelineWrongArgs.";
// op param
int freq = 1025;
int hop_length = 512;

13
tests/ut/cpp/dataset/c_api_vision_a_to_q_test.cc
View File

@ -27,9 +27,8 @@ class MindDataTestPipeline : public UT::DatasetOpTesting {
// Tests for vision C++ API A to Q TensorTransform Operations (in alphabetical order)
TEST_F(MindDataTestPipeline, TestAdjustGammaSuccess1) {
// pipeline 3-channel
MS_LOG(INFO) << "Pipeline Test.";
TEST_F(MindDataTestPipeline, TestAdjustGamma3Channel) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAdjustGamma3Channel.";
std::string MindDataPath = "data/dataset";
std::string folder_path = MindDataPath + "/testImageNetData/train/";
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));
@ -65,9 +64,8 @@ TEST_F(MindDataTestPipeline, TestAdjustGammaSuccess1) {
iter2->Stop();
}
TEST_F(MindDataTestPipeline, TestAdjustGammaSuccess2) {
// pipeline 1-channel
MS_LOG(INFO) << "Pipeline Test.";
TEST_F(MindDataTestPipeline, TestAdjustGamma1Channel) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAdjustGamma1Channel.";
std::string MindDataPath = "data/dataset";
std::string folder_path = MindDataPath + "/testImageNetData/train/";
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));
@ -105,8 +103,7 @@ TEST_F(MindDataTestPipeline, TestAdjustGammaSuccess2) {
}
TEST_F(MindDataTestPipeline, TestAdjustGammaParamCheck) {
// pipeline 3-channel
MS_LOG(INFO) << "Pipeline Test.";
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestAdjustGammaParamCheck.";
std::string MindDataPath = "data/dataset";
std::string folder_path = MindDataPath + "/testImageNetData/train/";
std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));

38
tests/ut/cpp/dataset/c_api_vision_r_to_z_test.cc
View File

@ -312,3 +312,41 @@ TEST_F(MindDataTestPipeline, TestRotatePass) {
// Manually terminate the pipeline
iter->Stop();
}
TEST_F(MindDataTestPipeline, TestRGB2BGR) {
MS_LOG(INFO) << "Doing MindDataTestPipeline-TestRGB2BGR.";
// create two imagenet dataset
std::string MindDataPath = "data/dataset";
std::string folder_path = MindDataPath + "/testImageNetData/train/";
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));
EXPECT_NE(ds2, nullptr);
auto rgb2bgr_op = vision::RGB2BGR();
ds1 = ds1->Map({rgb2bgr_op});
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Iterator> iter1 = ds1->CreateIterator();
EXPECT_NE(iter1, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row1;
iter1->GetNextRow(&row1);
std::shared_ptr<Iterator> iter2 = ds2->CreateIterator();
EXPECT_NE(iter2, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row2;
iter2->GetNextRow(&row2);
uint64_t i = 0;
while (row1.size() != 0) {
i++;
auto image =row1["image"];
iter1->GetNextRow(&row1);
iter2->GetNextRow(&row2);
}
EXPECT_EQ(i, 2);
iter1->Stop();
iter2->Stop();
}

89
tests/ut/cpp/dataset/execute_test.cc
View File

@ -35,7 +35,7 @@ class MindDataTestExecute : public UT::DatasetOpTesting {
};
TEST_F(MindDataTestExecute, TestAllpassBiquadWithEager) {
MS_LOG(INFO) << "Basic Function Test With Eager.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestAllpassBiquadWithEager.";
// Original waveform
std::vector<float> labels = {
2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02,
@ -54,7 +54,7 @@ TEST_F(MindDataTestExecute, TestAllpassBiquadWithEager) {
}
TEST_F(MindDataTestExecute, TestAllpassBiquadWithWrongArg) {
MS_LOG(INFO) << "Wrong Arg.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestAllpassBiquadWithWrongArg.";
std::vector<double> labels = {
2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02,
1.138305664062500000e-02, 1.156616210937500000e-02, 1.394653320312500000e-02, 1.550292968750000000e-02,
@ -72,9 +72,8 @@ TEST_F(MindDataTestExecute, TestAllpassBiquadWithWrongArg) {
EXPECT_FALSE(s01.IsOk());
}
TEST_F(MindDataTestExecute, TestAdjustGammaEager1) {
// 3-channel eager
MS_LOG(INFO) << "3-channel image test";
TEST_F(MindDataTestExecute, TestAdjustGammaEager3Channel) {
MS_LOG(INFO) << "Doing MindDataTestExecute-TestAdjustGammaEager3Channel.";
// Read images
auto image = ReadFileToTensor("data/dataset/apple.jpg");
@ -87,9 +86,8 @@ TEST_F(MindDataTestExecute, TestAdjustGammaEager1) {
EXPECT_EQ(rc, Status::OK());
}
TEST_F(MindDataTestExecute, TestAdjustGammaEager2) {
// 1-channel eager
MS_LOG(INFO) << "1-channel image test";
TEST_F(MindDataTestExecute, TestAdjustGammaEager1Channel) {
MS_LOG(INFO) << "Doing MindDataTestExecute-TestAdjustGammaEager1Channel.";
auto m1 = ReadFileToTensor("data/dataset/apple.jpg");
// Transform params
auto decode = vision::Decode();
@ -102,7 +100,7 @@ TEST_F(MindDataTestExecute, TestAdjustGammaEager2) {
}
TEST_F(MindDataTestExecute, TestAmplitudeToDB) {
MS_LOG(INFO) << "Basic Function Test With Eager.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestAmplitudeToDB.";
// Original waveform
std::vector<float> labels = {
2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02,
@ -122,7 +120,7 @@ TEST_F(MindDataTestExecute, TestAmplitudeToDB) {
}
TEST_F(MindDataTestExecute, TestAmplitudeToDBWrongArgs) {
MS_LOG(INFO) << "Wrong Arg.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestAmplitudeToDBWrongArgs.";
// Original waveform
std::vector<float> labels = {
2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02,
@ -142,7 +140,7 @@ TEST_F(MindDataTestExecute, TestAmplitudeToDBWrongArgs) {
}
TEST_F(MindDataTestExecute, TestAmplitudeToDBWrongInput) {
MS_LOG(INFO) << "Wrong Input.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestAmplitudeToDBWrongInput.";
// Original waveform
std::vector<float> labels = {
2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02,
@ -161,7 +159,7 @@ TEST_F(MindDataTestExecute, TestAmplitudeToDBWrongInput) {
}
TEST_F(MindDataTestExecute, TestComposeTransforms) {
MS_LOG(INFO) << "Doing TestComposeTransforms.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestComposeTransforms.";
// Read images
auto image = ReadFileToTensor("data/dataset/apple.jpg");
@ -198,7 +196,7 @@ TEST_F(MindDataTestExecute, TestCrop) {
}
TEST_F(MindDataTestExecute, TestFrequencyMasking) {
MS_LOG(INFO) << "Doing TestFrequencyMasking.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestFrequencyMasking.";
std::shared_ptr<Tensor> input_tensor_;
TensorShape s = TensorShape({6, 2});
ASSERT_OK(Tensor::CreateFromVector(
@ -211,7 +209,7 @@ TEST_F(MindDataTestExecute, TestFrequencyMasking) {
}
TEST_F(MindDataTestExecute, TestTimeMasking) {
MS_LOG(INFO) << "Doing TestTimeMasking.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestTimeMasking.";
std::shared_ptr<Tensor> input_tensor_;
TensorShape s = TensorShape({2, 6});
ASSERT_OK(Tensor::CreateFromVector(
@ -224,7 +222,7 @@ TEST_F(MindDataTestExecute, TestTimeMasking) {
}
TEST_F(MindDataTestExecute, TestTimeStretchEager) {
MS_LOG(INFO) << "Doing test TimeStretchOp with custom param value. Eager.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestTimeStretchEager.";
std::shared_ptr<Tensor> input_tensor_;
// op param
int freq = 4;
@ -248,8 +246,8 @@ TEST_F(MindDataTestExecute, TestTimeStretchEager) {
EXPECT_TRUE(status.IsOk());
}
TEST_F(MindDataTestExecute, TestTimeStretchParamCheck1) {
MS_LOG(INFO) << "Doing MindDataTestTimeStretch-TestTimeStretchParamCheck with invalid parameters.";
TEST_F(MindDataTestExecute, TestTimeStretchParamCheck) {
MS_LOG(INFO) << "Doing MindDataTestTimeStretch-TestTimeStretchParamCheck.";
// Create an input
std::shared_ptr<Tensor> input_tensor_;
std::shared_ptr<Tensor> output_tensor;
@ -259,26 +257,15 @@ TEST_F(MindDataTestExecute, TestTimeStretchParamCheck1) {
1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f}),
s, &input_tensor_));
auto input_ms = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(input_tensor_));
std::shared_ptr<TensorTransform> timestretch = std::make_shared<audio::TimeStretch>(4, 512, -2);
mindspore::dataset::Execute Transform({timestretch});
Status status = Transform(input_ms, &input_ms);
EXPECT_FALSE(status.IsOk());
}
TEST_F(MindDataTestExecute, TestTimeStretchParamCheck2) {
MS_LOG(INFO) << "Doing MindDataTestTimeStretch-TestTimeStretchParamCheck with invalid parameters.";
// Create an input
std::shared_ptr<Tensor> input_tensor_;
std::shared_ptr<Tensor> output_tensor;
TensorShape s = TensorShape({1, 4, 3, 2});
ASSERT_OK(Tensor::CreateFromVector(
std::vector<float>({1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f,
1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 6.0f, 5.0f, 4.0f, 3.0f, 2.0f, 1.0f}),
s, &input_tensor_));
auto input_ms = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(input_tensor_));
std::shared_ptr<TensorTransform> timestretch = std::make_shared<audio::TimeStretch>(4, -512, 2);
mindspore::dataset::Execute Transform({timestretch});
Status status = Transform(input_ms, &input_ms);
std::shared_ptr<TensorTransform> time_stretch1 = std::make_shared<audio::TimeStretch>(4, 512, -2);
mindspore::dataset::Execute Transform1({time_stretch1});
Status status = Transform1(input_ms, &input_ms);
EXPECT_FALSE(status.IsOk());
std::shared_ptr<TensorTransform> time_stretch2 = std::make_shared<audio::TimeStretch>(4, -512, 2);
mindspore::dataset::Execute Transform2({time_stretch2});
status = Transform2(input_ms, &input_ms);
EXPECT_FALSE(status.IsOk());
}
@ -511,7 +498,7 @@ TEST_F(MindDataTestExecute, TestResizeWithBBox) {
}
TEST_F(MindDataTestExecute, TestBandBiquadWithEager) {
MS_LOG(INFO) << "Basic Function Test With Eager.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestBandBiquadWithEager.";
// Original waveform
std::vector<float> labels = {
2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02,
@ -530,7 +517,7 @@ TEST_F(MindDataTestExecute, TestBandBiquadWithEager) {
}
TEST_F(MindDataTestExecute, TestBandBiquadWithWrongArg) {
MS_LOG(INFO) << "Wrong Arg.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestBandBiquadWithWrongArg.";
std::vector<double> labels = {
2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02,
1.138305664062500000e-02, 1.156616210937500000e-02, 1.394653320312500000e-02, 1.550292968750000000e-02,
@ -549,7 +536,7 @@ TEST_F(MindDataTestExecute, TestBandBiquadWithWrongArg) {
}
TEST_F(MindDataTestExecute, TestBandpassBiquadWithEager) {
MS_LOG(INFO) << "Basic Function Test With Eager.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestBandpassBiquadWithEager.";
// Original waveform
std::vector<float> labels = {
2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02,
@ -568,7 +555,7 @@ TEST_F(MindDataTestExecute, TestBandpassBiquadWithEager) {
}
TEST_F(MindDataTestExecute, TestBandpassBiquadWithWrongArg) {
MS_LOG(INFO) << "Wrong Arg.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestBandpassBiquadWithWrongArg.";
std::vector<double> labels = {
2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02,
1.138305664062500000e-02, 1.156616210937500000e-02, 1.394653320312500000e-02, 1.550292968750000000e-02,
@ -587,7 +574,7 @@ TEST_F(MindDataTestExecute, TestBandpassBiquadWithWrongArg) {
}
TEST_F(MindDataTestExecute, TestBandrejectBiquadWithEager) {
MS_LOG(INFO) << "Basic Function Test With Eager.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestBandrejectBiquadWithEager.";
// Original waveform
std::vector<float> labels = {
2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02,
@ -606,7 +593,7 @@ TEST_F(MindDataTestExecute, TestBandrejectBiquadWithEager) {
}
TEST_F(MindDataTestExecute, TestBandrejectBiquadWithWrongArg) {
MS_LOG(INFO) << "Wrong Arg.";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestBandrejectBiquadWithWrongArg.";
std::vector<double> labels = {
2.716064453125000000e-03, 6.347656250000000000e-03, 9.246826171875000000e-03, 1.089477539062500000e-02,
1.138305664062500000e-02, 1.156616210937500000e-02, 1.394653320312500000e-02, 1.550292968750000000e-02,
@ -625,7 +612,7 @@ TEST_F(MindDataTestExecute, TestBandrejectBiquadWithWrongArg) {
}
TEST_F(MindDataTestExecute, TestAngleEager) {
MS_LOG(INFO) << "Doing MindDataTestExecute-TestAngleEager";
MS_LOG(INFO) << "Doing MindDataTestExecute-TestAngleEager.";
std::vector<double> origin = {1.143, 1.3123, 2.632, 2.554, -1.213, 1.3, 0.456, 3.563};
TensorShape input_shape({4, 2});
std::shared_ptr<Tensor> de_tensor;
@ -638,3 +625,19 @@ TEST_F(MindDataTestExecute, TestAngleEager) {
ASSERT_TRUE(s.IsOk());
}
TEST_F(MindDataTestExecute, TestRGB2BGREager) {
MS_LOG(INFO) << "Doing MindDataTestExecute-TestRGB2BGREager.";
// Read images
auto image = ReadFileToTensor("data/dataset/apple.jpg");
// Transform params
auto decode = vision::Decode();
auto rgb2bgr_op = vision::RGB2BGR();
auto transform = Execute({decode, rgb2bgr_op});
Status rc = transform(image, &image);
EXPECT_EQ(rc, Status::OK());
}

100
tests/ut/cpp/dataset/rgb_to_bgr_test_op.cc
View File

@ -1,100 +0,0 @@
/**
* Copyright 2021 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include <opencv2/imgcodecs.hpp>
#include <opencv2/opencv.hpp>
#include "common/common.h"
#include "common/cvop_common.h"
#include "include/dataset/datasets.h"
#include "include/dataset/transforms.h"
#include "include/dataset/vision.h"
#include "include/dataset/execute.h"
#include "minddata/dataset/kernels/image/image_utils.h"
#include "minddata/dataset/kernels/image/rgb_to_bgr_op.h"
#include "minddata/dataset/core/cv_tensor.h"
#include "utils/log_adapter.h"
using namespace std;
using namespace mindspore::dataset;
using mindspore::dataset::CVTensor;
using mindspore::dataset::BorderType;
using mindspore::dataset::Tensor;
using mindspore::LogStream;
using mindspore::ExceptionType::NoExceptionType;
using mindspore::MsLogLevel::INFO;
class MindDataTestRgbToBgrOp : public UT::DatasetOpTesting {
protected:
};
TEST_F(MindDataTestRgbToBgrOp, TestOp1) {
// Eager
MS_LOG(INFO) << "Doing MindDataTestGaussianBlur-TestGaussianBlurEager.";
// Read images
auto image = ReadFileToTensor("data/dataset/apple.jpg");
// Transform params
auto decode = vision::Decode();
auto rgb2bgr_op = vision::RGB2BGR();
auto transform = Execute({decode, rgb2bgr_op});
Status rc = transform(image, &image);
EXPECT_EQ(rc, Status::OK());
}
TEST_F(MindDataTestRgbToBgrOp, TestOp2) {
// pipeline
MS_LOG(INFO) << "Basic Function Test.";
// create two imagenet dataset
std::string MindDataPath = "data/dataset";
std::string folder_path = MindDataPath + "/testImageNetData/train/";
std::shared_ptr<Dataset> ds1 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Dataset> ds2 = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));
EXPECT_NE(ds2, nullptr);
auto rgb2bgr_op = vision::RGB2BGR();
ds1 = ds1->Map({rgb2bgr_op});
EXPECT_NE(ds1, nullptr);
std::shared_ptr<Iterator> iter1 = ds1->CreateIterator();
EXPECT_NE(iter1, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row1;
iter1->GetNextRow(&row1);
std::shared_ptr<Iterator> iter2 = ds2->CreateIterator();
EXPECT_NE(iter2, nullptr);
std::unordered_map<std::string, mindspore::MSTensor> row2;
iter2->GetNextRow(&row2);
uint64_t i = 0;
while (row1.size() != 0) {
i++;
auto image =row1["image"];
iter1->GetNextRow(&row1);
iter2->GetNextRow(&row2);
}
EXPECT_EQ(i, 2);
iter1->Stop();
iter2->Stop();
}

32
tests/ut/python/dataset/test_adjustgamma.py
View File

@ -31,8 +31,6 @@ MNIST_DATA_DIR = "../data/dataset/testMnistData"
DATA_DIR_2 = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json"
GENERATE_GOLDEN = False
def generate_numpy_random_rgb(shape):
"""
@ -90,26 +88,20 @@ def test_adjust_gamma_invalid_gamma_param_c():
logger.info("Test AdjustGamma C Op with invalid ignore parameter")
try:
data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
data_set = data_set.map(operations=[C.Decode(),
C.Resize((224, 224)),
lambda img: np.array(img[:, :, 0])],
data_set = data_set.map(operations=[C.Decode(), C.Resize((224, 224)), lambda img: np.array(img[:, :, 0])],
input_columns=["image"])
# invalid gamma
data_set = data_set.map(operations=C.AdjustGamma(gamma=-10.0,
gain=1.0),
data_set = data_set.map(operations=C.AdjustGamma(gamma=-10.0, gain=1.0),
input_columns="image")
except ValueError as error:
logger.info("Got an exception in AdjustGamma: {}".format(str(error)))
assert "Input is not within the required interval of " in str(error)
try:
data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
data_set = data_set.map(operations=[C.Decode(),
C.Resize((224, 224)),
lambda img: np.array(img[:, :, 0])],
data_set = data_set.map(operations=[C.Decode(), C.Resize((224, 224)), lambda img: np.array(img[:, :, 0])],
input_columns=["image"])
# invalid gamma
data_set = data_set.map(operations=C.AdjustGamma(gamma=[1, 2],
gain=1.0),
data_set = data_set.map(operations=C.AdjustGamma(gamma=[1, 2], gain=1.0),
input_columns="image")
except TypeError as error:
logger.info("Got an exception in AdjustGamma: {}".format(str(error)))
@ -129,8 +121,7 @@ def test_adjust_gamma_invalid_gamma_param_py():
F.AdjustGamma(gamma=-10.0),
F.ToTensor()
])
data_set = data_set.map(operations=[trans],
input_columns=["image"])
data_set = data_set.map(operations=[trans], input_columns=["image"])
except ValueError as error:
logger.info("Got an exception in AdjustGamma: {}".format(str(error)))
assert "Input is not within the required interval of " in str(error)
@ -142,8 +133,7 @@ def test_adjust_gamma_invalid_gamma_param_py():
F.AdjustGamma(gamma=[1, 2]),
F.ToTensor()
])
data_set = data_set.map(operations=[trans],
input_columns=["image"])
data_set = data_set.map(operations=[trans], input_columns=["image"])
except TypeError as error:
logger.info("Got an exception in AdjustGamma: {}".format(str(error)))
assert "is not of type [<class 'float'>, <class 'int'>], but got" in str(error)
@ -156,13 +146,10 @@ def test_adjust_gamma_invalid_gain_param_c():
logger.info("Test AdjustGamma C Op with invalid gain parameter")
try:
data_set = ds.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
data_set = data_set.map(operations=[C.Decode(),
C.Resize((224, 224)),
lambda img: np.array(img[:, :, 0])],
data_set = data_set.map(operations=[C.Decode(), C.Resize((224, 224)), lambda img: np.array(img[:, :, 0])],
input_columns=["image"])
# invalid gain
data_set = data_set.map(operations=C.AdjustGamma(gamma=10.0,
gain=[1, 10]),
data_set = data_set.map(operations=C.AdjustGamma(gamma=10.0, gain=[1, 10]),
input_columns="image")
except TypeError as error:
logger.info("Got an exception in AdjustGamma: {}".format(str(error)))
@ -182,8 +169,7 @@ def test_adjust_gamma_invalid_gain_param_py():
F.AdjustGamma(gamma=10.0, gain=[1, 10]),
F.ToTensor()
])
data_set = data_set.map(operations=[trans],
input_columns=["image"])
data_set = data_set.map(operations=[trans], input_columns=["image"])
except TypeError as error:
logger.info("Got an exception in AdjustGamma: {}".format(str(error)))
assert "is not of type [<class 'float'>, <class 'int'>], but got " in str(error)

48
tests/ut/python/dataset/test_allpass_biquad.py
View File

@ -19,16 +19,14 @@ import mindspore.dataset.audio.transforms as audio
from mindspore import log as logger
def _count_unequal_element(data_expected, data_me, rtol, atol):
def count_unequal_element(data_expected, data_me, rtol, atol):
assert data_expected.shape == data_me.shape
total_count = len(data_expected.flatten())
error = np.abs(data_expected - data_me)
greater = np.greater(error, atol + np.abs(data_expected) * rtol)
loss_count = np.count_nonzero(greater)
assert (loss_count / total_count) < rtol, \
"\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \
format(data_expected[greater], data_me[greater], error[greater])
assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format(
data_expected[greater], data_me[greater], error[greater])
def test_func_allpass_biquad_eager():
@ -37,12 +35,11 @@ def test_func_allpass_biquad_eager():
# Original waveform
waveform = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float64)
# Expect waveform
expect_waveform = np.array([[0.96049707, 1.0, 1.0],
[1.0, 1.0, 1.0]], dtype=np.float64)
expect_waveform = np.array([[0.96049707, 1.0, 1.0], [1.0, 1.0, 1.0]], dtype=np.float64)
allpass_biquad_op = audio.AllpassBiquad(44100, 200.0, 0.707)
# Filtered waveform by allpassbiquad
output = allpass_biquad_op(waveform)
_count_unequal_element(expect_waveform, output, 0.0001, 0.0001)
count_unequal_element(expect_waveform, output, 0.0001, 0.0001)
def test_func_allpass_biquad_pipeline():
@ -51,56 +48,57 @@ def test_func_allpass_biquad_pipeline():
# Original waveform
waveform = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float64)
# Expect waveform
expect_waveform = np.array([[0.96049707, 1.0, 1.0],
[1.0, 1.0, 1.0]], dtype=np.float64)
expect_waveform = np.array([[0.96049707, 1.0, 1.0], [1.0, 1.0, 1.0]], dtype=np.float64)
label = np.random.sample((2, 1))
data = (waveform, label)
dataset = ds.NumpySlicesDataset(data, ["channel", "sample"], shuffle=False)
allpass_biquad_op = audio.AllpassBiquad(44100, 200.0)
# Filtered waveform by allpassbiquad
dataset = dataset.map(
input_columns=["channel"], operations=allpass_biquad_op, num_parallel_workers=8)
dataset = dataset.map(input_columns=["channel"], operations=allpass_biquad_op, num_parallel_workers=8)
i = 0
for _ in dataset.create_dict_iterator(output_numpy=True):
_count_unequal_element(expect_waveform[i, :],
_['channel'], 0.0001, 0.0001)
for item in dataset.create_dict_iterator(output_numpy=True):
count_unequal_element(expect_waveform[i, :], item['channel'], 0.0001, 0.0001)
i += 1
def test_invalid_input_all():
waveform = np.random.rand(2, 1000)
def test_invalid_input(test_name, sample_rate, central_freq, Q, error, error_msg):
logger.info("Test Allpassallpassiquad with bad input: {0}".format(test_name))
with pytest.raises(error) as error_info:
audio.AllpassBiquad(sample_rate, central_freq, Q)(waveform)
assert error_msg in str(error_info.value)
test_invalid_input("invalid sample_rate parameter type as a float", 44100.5, 200, 0.707, TypeError,
"Argument sample_rate with value 44100.5 is not of type [<class 'int'>],"
+" but got <class 'float'>.")
+ " but got <class 'float'>.")
test_invalid_input("invalid sample_rate parameter type as a String", "44100", 200, 0.707, TypeError,
"Argument sample_rate with value 44100 is not of type [<class 'int'>],"+
"Argument sample_rate with value 44100 is not of type [<class 'int'>]," +
" but got <class 'str'>.")
test_invalid_input("invalid contral_freq parameter type as a String", 44100, "200", 0.707, TypeError,
"Argument central_freq with value 200 is not of type [<class 'float'>, <class 'int'>],"
+" but got <class 'str'>.")
+ " but got <class 'str'>.")
test_invalid_input("invalid Q parameter type as a String", 44100, 200, "0.707", TypeError,
"Argument Q with value 0.707 is not of type [<class 'float'>, <class 'int'>],"
+" but got <class 'str'>.")
+ " but got <class 'str'>.")
test_invalid_input("invalid sample_rate parameter value", 441324343243242342345300, 200, 0.707, ValueError,
"Input sample_rate is not within the required interval of [-2147483648, 2147483647].")
"Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].")
test_invalid_input("invalid contral_freq parameter value", 44100, 32434324324234321, 0.707, ValueError,
"Input central_freq is not within the required interval of [-16777216, 16777216].")
test_invalid_input("invalid sample_rate parameter value", None, 200, 0.707, TypeError,
"Argument sample_rate with value None is not of type [<class 'int'>],"
+" but got <class 'NoneType'>.")
+ " but got <class 'NoneType'>.")
test_invalid_input("invalid central_rate parameter value", 44100, None, 0.707, TypeError,
"Argument central_freq with value None is not of type [<class 'float'>, <class 'int'>],"
+" but got <class 'NoneType'>.")
+ " but got <class 'NoneType'>.")
test_invalid_input("invalid sample_rate parameter value", 0, 200, 0.707, ValueError,
"Input sample_rate can not be 0.")
"Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].")
test_invalid_input("invalid Q parameter value", 44100, 200, 1.707, ValueError,
"Input Q is not within the required interval of (0, 1].")
if __name__ == '__main__':
test_eager_allpassbiquad_mindspore_001()
test_pipeline_allpass_biquad_001()
test_func_allpass_biquad_eager()
test_func_allpass_biquad_pipeline()
test_invalid_input_all()

19
tests/ut/python/dataset/test_amplitude_to_db.py
View File

@ -23,7 +23,6 @@ import mindspore.dataset.audio.transforms as c_audio
from mindspore import log as logger
from mindspore.dataset.audio.utils import ScaleType
CHANNEL = 1
FREQ = 20
TIME = 15
@ -32,19 +31,18 @@ TIME = 15
def gen(shape):
np.random.seed(0)
data = np.random.random(shape)
yield(np.array(data, dtype=np.float32),)
yield (np.array(data, dtype=np.float32),)
def _count_unequal_element(data_expected, data_me, rtol, atol):
def count_unequal_element(data_expected, data_me, rtol, atol):
""" Precision calculation func """
assert data_expected.shape == data_me.shape
total_count = len(data_expected.flatten())
error = np.abs(data_expected - data_me)
greater = np.greater(error, atol + np.abs(data_expected) * rtol)
loss_count = np.count_nonzero(greater)
assert (loss_count / total_count) < rtol, \
"\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \
format(data_expected[greater], data_me[greater], error[greater])
assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format(
data_expected[greater], data_me[greater], error[greater])
def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True):
@ -52,9 +50,7 @@ def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True):
if np.any(np.isnan(data_expected)):
assert np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan)
elif not np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan):
_count_unequal_element(data_expected, data_me, rtol, atol)
else:
assert True
count_unequal_element(data_expected, data_me, rtol, atol)
def test_func_amplitude_to_db_eager():
@ -91,9 +87,7 @@ def test_func_amplitude_to_db_pipeline():
data1 = ds.GeneratorDataset(source=generator, column_names=["multi_dimensional_data"])
transforms = [
c_audio.AmplitudeToDB()
]
transforms = [c_audio.AmplitudeToDB()]
data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
@ -102,7 +96,6 @@ def test_func_amplitude_to_db_pipeline():
def test_amplitude_to_db_invalid_input():
def test_invalid_input(test_name, stype, ref_value, amin, top_db, error, error_msg):
logger.info("Test AmplitudeToDB with bad input: {0}".format(test_name))
with pytest.raises(error) as error_info:

23
tests/ut/python/dataset/test_angle.py
View File

@ -19,28 +19,28 @@ import pytest
import mindspore.dataset as ds
import mindspore.dataset.audio.transforms as a_c_trans
def _count_unequal_element(data_expected, data_me, rtol, atol):
def count_unequal_element(data_expected, data_me, rtol, atol):
assert data_expected.shape == data_me.shape
total_count = len(data_expected.flatten())
error = np.abs(data_expected - data_me)
greater = np.greater(error, atol + np.abs(data_expected) * rtol)
loss_count = np.count_nonzero(greater)
assert (loss_count / total_count) < rtol, \
"\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \
format(data_expected[greater], data_me[greater], error[greater])
assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format(
data_expected[greater], data_me[greater], error[greater])
def test_func_angle_001():
"""
Eager Test
"""
arr = np.array([[73.04, -13.00], [57.49, 13.20], [-57.64, 6.51], [-52.25, 30.67], [-30.11, -18.34], \
arr = np.array([[73.04, -13.00], [57.49, 13.20], [-57.64, 6.51], [-52.25, 30.67], [-30.11, -18.34],
[-63.32, 99.33], [95.82, -24.76]], dtype=np.double)
expected = np.array([-0.17614017, 0.22569334, 3.02912684, 2.6107975, -2.59450886, 2.13831337, -0.25286988], \
expected = np.array([-0.17614017, 0.22569334, 3.02912684, 2.6107975, -2.59450886, 2.13831337, -0.25286988],
dtype=np.double)
angle_op = a_c_trans.Angle()
output = angle_op(arr)
_count_unequal_element(expected, output, 0.0001, 0.0001)
count_unequal_element(expected, output, 0.0001, 0.0001)
def test_func_angle_002():
@ -48,9 +48,9 @@ def test_func_angle_002():
Pipeline Test
"""
np.random.seed(6)
arr = np.array([[[84.25, -85.92], [-92.23, 23.06], [-7.33, -44.17], [-62.95, -14.73]], \
arr = np.array([[[84.25, -85.92], [-92.23, 23.06], [-7.33, -44.17], [-62.95, -14.73]],
[[93.09, 38.18], [-81.94, 71.34], [71.33, -39.00], [95.25, -32.94]]], dtype=np.double)
expected = np.array([[-0.79521156, 2.89658848, -1.73524737, -2.91173309], \
expected = np.array([[-0.79521156, 2.89658848, -1.73524737, -2.91173309],
[0.3892177, 2.42523905, -0.50034807, -0.33295219]], dtype=np.double)
label = np.random.sample((2, 4, 1))
data = (arr, label)
@ -58,7 +58,8 @@ def test_func_angle_002():
angle_op = a_c_trans.Angle()
dataset = dataset.map(operations=angle_op, input_columns=["col1"])
for item1, item2 in zip(dataset.create_dict_iterator(output_numpy=True), expected):
_count_unequal_element(item2, item1['col1'], 0.0001, 0.0001)
count_unequal_element(item2, item1['col1'], 0.0001, 0.0001)
def test_func_angle_003():
"""
@ -72,7 +73,7 @@ def test_func_angle_003():
angle_op = a_c_trans.Angle()
dataset = dataset.map(operations=angle_op, input_columns=["col1"])
num_itr = 0
with pytest.raises(RuntimeError, match="The input type should be numbers"):
with pytest.raises(RuntimeError, match="input tensor type should be int, float or double"):
for _ in dataset.create_dict_iterator(output_numpy=True):
num_itr += 1

20
tests/ut/python/dataset/test_band_biquad.py
View File

@ -19,16 +19,14 @@ import mindspore.dataset.audio.transforms as audio
from mindspore import log as logger
def _count_unequal_element(data_expected, data_me, rtol, atol):
def count_unequal_element(data_expected, data_me, rtol, atol):
assert data_expected.shape == data_me.shape
total_count = len(data_expected.flatten())
error = np.abs(data_expected - data_me)
greater = np.greater(error, atol + np.abs(data_expected) * rtol)
loss_count = np.count_nonzero(greater)
assert (loss_count / total_count) < rtol, \
"\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \
format(data_expected[greater], data_me[greater], error[greater])
assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format(
data_expected[greater], data_me[greater], error[greater])
def test_func_band_biquad_eager():
@ -42,7 +40,7 @@ def test_func_band_biquad_eager():
band_biquad_op = audio.BandBiquad(44100, 200.0, 0.707, False)
# Filtered waveform by bandbiquad
output = band_biquad_op(waveform)
_count_unequal_element(expect_waveform, output, 0.0001, 0.0001)
count_unequal_element(expect_waveform, output, 0.0001, 0.0001)
def test_func_band_biquad_pipeline():
@ -61,9 +59,9 @@ def test_func_band_biquad_pipeline():
dataset = dataset.map(
input_columns=["channel"], operations=band_biquad_op, num_parallel_workers=8)
i = 0
for _ in dataset.create_dict_iterator(output_numpy=True):
_count_unequal_element(expect_waveform[i, :],
_['channel'], 0.0001, 0.0001)
for item in dataset.create_dict_iterator(output_numpy=True):
count_unequal_element(expect_waveform[i, :],
item['channel'], 0.0001, 0.0001)
i += 1
@ -83,7 +81,7 @@ def test_band_biquad_invalid_input():
"Argument central_freq with value 200 is not of type [<class 'float'>, <class 'int'>],"
" but got <class 'str'>.")
test_invalid_input("invalid sample_rate parameter value", 0, 200, 0.707, True, ValueError,
"Input sample_rate can not be 0.")
"Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].")
test_invalid_input("invalid contral_freq parameter value", 44100, 32434324324234321, 0.707, True, ValueError,
"Input central_freq is not within the required interval of [-16777216, 16777216].")
test_invalid_input("invalid Q parameter type as a String", 44100, 200, "0.707", True, TypeError,
@ -94,7 +92,7 @@ def test_band_biquad_invalid_input():
test_invalid_input("invalid Q parameter value", 44100, 200, 0, True, ValueError,
"Input Q is not within the required interval of (0, 1].")
test_invalid_input("invalid sample_rate parameter value", 441324343243242342345300, 200, 0.707, True, ValueError,
"Input sample_rate is not within the required interval of [-2147483648, 2147483647].")
"Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].")
test_invalid_input("invalid sample_rate parameter value", None, 200, 0.707, True, TypeError,
"Argument sample_rate with value None is not of type [<class 'int'>],"
" but got <class 'NoneType'>.")

25
tests/ut/python/dataset/test_bandpass_biquad.py
View File

@ -19,16 +19,14 @@ import mindspore.dataset.audio.transforms as audio
from mindspore import log as logger
def _count_unequal_element(data_expected, data_me, rtol, atol):
def count_unequal_element(data_expected, data_me, rtol, atol):
assert data_expected.shape == data_me.shape
total_count = len(data_expected.flatten())
error = np.abs(data_expected - data_me)
greater = np.greater(error, atol + np.abs(data_expected) * rtol)
loss_count = np.count_nonzero(greater)
assert (loss_count / total_count) < rtol, \
"\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \
format(data_expected[greater], data_me[greater], error[greater])
assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format(
data_expected[greater], data_me[greater], error[greater])
def test_func_bandpass_biquad_eager():
@ -42,7 +40,7 @@ def test_func_bandpass_biquad_eager():
bandpass_biquad_op = audio.BandpassBiquad(44000, 200.0, 0.707, False)
# Filtered waveform by bandpassbiquad
output = bandpass_biquad_op(waveform)
_count_unequal_element(expect_waveform, output, 0.0001, 0.0001)
count_unequal_element(expect_waveform, output, 0.0001, 0.0001)
def test_func_bandpass_biquad_pipeline():
@ -58,12 +56,10 @@ def test_func_bandpass_biquad_pipeline():
dataset = ds.NumpySlicesDataset(data, ["channel", "sample"], shuffle=False)
bandpass_biquad_op = audio.BandpassBiquad(44000, 200.0)
# Filtered waveform by bandpassbiquad
dataset = dataset.map(
input_columns=["channel"], operations=bandpass_biquad_op, num_parallel_workers=8)
dataset = dataset.map(input_columns=["channel"], operations=bandpass_biquad_op, num_parallel_workers=8)
i = 0
for _ in dataset.create_dict_iterator(output_numpy=True):
_count_unequal_element(expect_waveform[i, :],
_['channel'], 0.0001, 0.0001)
for item in dataset.create_dict_iterator(output_numpy=True):
count_unequal_element(expect_waveform[i, :], item['channel'], 0.0001, 0.0001)
i += 1
@ -72,8 +68,7 @@ def test_bandpass_biquad_invalid_input():
logger.info(
"Test BandpassBiquad with bad input: {0}".format(test_name))
with pytest.raises(error) as error_info:
audio.BandpassBiquad(
sample_rate, central_freq, Q, const_skirt_gain)
audio.BandpassBiquad(sample_rate, central_freq, Q, const_skirt_gain)
assert error_msg in str(error_info.value)
test_invalid_input("invalid sample_rate parameter type as a float", 44100.5, 200, 0.707, True, TypeError,
@ -85,7 +80,7 @@ def test_bandpass_biquad_invalid_input():
"Argument central_freq with value 200 is not of type [<class 'float'>, <class 'int'>],"
" but got <class 'str'>.")
test_invalid_input("invalid sample_rate parameter value", 0, 200, 0.707, True, ValueError,
"Input sample_rate can not be 0.")
"Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].")
test_invalid_input("invalid contral_freq parameter value", 44100, 32434324324234321, 0.707, True, ValueError,
"Input central_freq is not within the required interval of [-16777216, 16777216].")
test_invalid_input("invalid Q parameter type as a String", 44100, 200, "0.707", True, TypeError,
@ -96,7 +91,7 @@ def test_bandpass_biquad_invalid_input():
test_invalid_input("invalid Q parameter value", 44100, 200, 0, True, ValueError,
"Input Q is not within the required interval of (0, 1].")
test_invalid_input("invalid sample_rate parameter value", 441324343243242342345300, 200, 0.707, True, ValueError,
"Input sample_rate is not within the required interval of [-2147483648, 2147483647].")
"Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].")
test_invalid_input("invalid sample_rate parameter value", None, 200, 0.707, True, TypeError,
"Argument sample_rate with value None is not of type [<class 'int'>],"
" but got <class 'NoneType'>.")

27
tests/ut/python/dataset/test_bandreject_biquad.py
View File

@ -19,16 +19,14 @@ import mindspore.dataset.audio.transforms as audio
from mindspore import log as logger
def _count_unequal_element(data_expected, data_me, rtol, atol):
def count_unequal_element(data_expected, data_me, rtol, atol):
assert data_expected.shape == data_me.shape
total_count = len(data_expected.flatten())
error = np.abs(data_expected - data_me)
greater = np.greater(error, atol + np.abs(data_expected) * rtol)
loss_count = np.count_nonzero(greater)
assert (loss_count / total_count) < rtol, \
"\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \
format(data_expected[greater], data_me[greater], error[greater])
assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format(
data_expected[greater], data_me[greater], error[greater])
def test_func_bandreject_biquad_eager():
@ -43,7 +41,7 @@ def test_func_bandreject_biquad_eager():
bandreject_biquad_op = audio.BandrejectBiquad(44100, 200.0, 0.707)
# Filtered waveform by bandrejectbiquad
output = bandreject_biquad_op(waveform)
_count_unequal_element(expect_waveform, output, 0.0001, 0.0001)
count_unequal_element(expect_waveform, output, 0.0001, 0.0001)
def test_func_bandreject_biquad_pipeline():
@ -63,9 +61,9 @@ def test_func_bandreject_biquad_pipeline():
dataset = dataset.map(
input_columns=["channel"], operations=bandreject_biquad_op, num_parallel_workers=8)
i = 0
for _ in dataset.create_dict_iterator(output_numpy=True):
_count_unequal_element(expect_waveform[i, :],
_['channel'], 0.0001, 0.0001)
for item in dataset.create_dict_iterator(output_numpy=True):
count_unequal_element(expect_waveform[i, :],
item['channel'], 0.0001, 0.0001)
i += 1
@ -76,6 +74,7 @@ def test_bandreject_biquad_invalid_input():
with pytest.raises(error) as error_info:
audio.BandrejectBiquad(sample_rate, central_freq, Q)
assert error_msg in str(error_info.value)
test_invalid_input("invalid sample_rate parameter type as a float", 44100.5, 200, 0.707, TypeError,
"Argument sample_rate with value 44100.5 is not of type [<class 'int'>],"
" but got <class 'float'>.")
@ -85,7 +84,7 @@ def test_bandreject_biquad_invalid_input():
"Argument central_freq with value 200 is not of type [<class 'float'>, <class 'int'>],"
" but got <class 'str'>.")
test_invalid_input("invalid sample_rate parameter value", 0, 200, 0.707, ValueError,
"Input sample_rate can not be 0.")
"Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].")
test_invalid_input("invalid contral_freq parameter value", 44100, 32434324324234321, 0.707, ValueError,
"Input central_freq is not within the required interval of [-16777216, 16777216].")
test_invalid_input("invalid Q parameter type as a String", 44100, 200, "0.707", TypeError,
@ -96,7 +95,7 @@ def test_bandreject_biquad_invalid_input():
test_invalid_input("invalid Q parameter value", 44100, 200, 0, ValueError,
"Input Q is not within the required interval of (0, 1].")
test_invalid_input("invalid sample_rate parameter value", 441324343243242342345300, 200, 0.707, ValueError,
"Input sample_rate is not within the required interval of [-2147483648, 2147483647].")
"Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].")
test_invalid_input("invalid sample_rate parameter value", None, 200, 0.707, TypeError,
"Argument sample_rate with value None is not of type [<class 'int'>],"
" but got <class 'NoneType'>.")
@ -106,6 +105,6 @@ def test_bandreject_biquad_invalid_input():
if __name__ == "__main__":
test_func_band_biquad_eager()
test_func_band_biquad_pipeline()
test_band_biquad_invalid_input()
test_func_bandreject_biquad_eager()
test_func_bandreject_biquad_pipeline()
test_bandreject_biquad_invalid_input()

22
tests/ut/python/dataset/test_bass_biquad.py
View File

@ -19,16 +19,14 @@ import mindspore.dataset.audio.transforms as audio
from mindspore import log as logger
def _count_unequal_element(data_expected, data_me, rtol, atol):
def count_unequal_element(data_expected, data_me, rtol, atol):
assert data_expected.shape == data_me.shape
total_count = len(data_expected.flatten())
error = np.abs(data_expected - data_me)
greater = np.greater(error, atol + np.abs(data_expected) * rtol)
loss_count = np.count_nonzero(greater)
assert (loss_count / total_count) < rtol, \
"\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \
format(data_expected[greater], data_me[greater], error[greater])
assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format(
data_expected[greater], data_me[greater], error[greater])
def test_func_bass_biquad_eager():
@ -42,7 +40,7 @@ def test_func_bass_biquad_eager():
bass_biquad_op = audio.BassBiquad(44100, 50.0, 100.0, 0.707)
# Filtered waveform by bassbiquad
output = bass_biquad_op(waveform)
_count_unequal_element(expect_waveform, output, 0.0001, 0.0001)
count_unequal_element(expect_waveform, output, 0.0001, 0.0001)
def test_func_bass_biquad_pipeline():
@ -61,9 +59,9 @@ def test_func_bass_biquad_pipeline():
dataset = dataset.map(
input_columns=["channel"], operations=bass_biquad_op, num_parallel_workers=8)
i = 0
for _ in dataset.create_dict_iterator(output_numpy=True):
_count_unequal_element(expect_waveform[i, :],
_['channel'], 0.0001, 0.0001)
for item in dataset.create_dict_iterator(output_numpy=True):
count_unequal_element(expect_waveform[i, :],
item['channel'], 0.0001, 0.0001)
i += 1
@ -73,6 +71,7 @@ def test_invalid_invalid_input():
with pytest.raises(error) as error_info:
audio.BassBiquad(sample_rate, gain, central_freq, Q)
assert error_msg in str(error_info.value)
test_invalid_input("invalid sample_rate parameter type as a float", 44100.5, 50.0, 200, 0.707, TypeError,
"Argument sample_rate with value 44100.5 is not of type [<class 'int'>],"
" but got <class 'float'>.")
@ -90,7 +89,7 @@ def test_invalid_invalid_input():
" but got <class 'str'>.")
test_invalid_input("invalid sample_rate parameter value", 441324343243242342345300, 50.0, 200, 0.707, ValueError,
"Input sample_rate is not within the required interval of [-2147483648, 2147483647].")
"Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].")
test_invalid_input("invalid gain parameter value", 44100, 32434324324234321, 200, 0.707, ValueError,
"Input gain is not within the required interval of [-16777216, 16777216].")
test_invalid_input("invalid contral_freq parameter value", 44100, 50, 32434324324234321, 0.707, ValueError,
@ -107,10 +106,11 @@ def test_invalid_invalid_input():
" but got <class 'NoneType'>.")
test_invalid_input("invalid sample_rate parameter value", 0, 50.0, 200, 0.707, ValueError,
"Input sample_rate can not be 0.")
"Input sample_rate is not within the required interval of [-2147483648, 0) and (0, 2147483647].")
test_invalid_input("invalid Q parameter value", 44100, 50.0, 200, 1.707, ValueError,
"Input Q is not within the required interval of (0, 1].")
if __name__ == '__main__':
test_func_bass_biquad_eager()
test_func_bass_biquad_pipeline()

2
tests/ut/python/dataset/test_datasets_sbd.py
View File

@ -22,7 +22,6 @@ import mindspore.dataset as ds
from mindspore import log as logger
import mindspore.dataset.vision.c_transforms as c_vision
DATASET_DIR = "../data/dataset/testSBData/sbd"
@ -193,6 +192,7 @@ def test_sbd_usage():
"""
Validate SBDataset image readings
"""
def test_config(usage):
try:
data = ds.SBDataset(DATASET_DIR, task='Segmentation', usage=usage)

34
tests/ut/python/dataset/test_frequency_masking.py
View File

@ -20,10 +20,9 @@ import numpy as np
import pytest
import mindspore.dataset as ds
import mindspore.dataset.audio.transforms as atf
import mindspore.dataset.audio.transforms as audio
from mindspore import log as logger
CHANNEL = 2
FREQ = 30
TIME = 30
@ -32,19 +31,18 @@ TIME = 30
def gen(shape):
np.random.seed(0)
data = np.random.random(shape)
yield(np.array(data, dtype=np.float32),)
yield (np.array(data, dtype=np.float32),)
def _count_unequal_element(data_expected, data_me, rtol, atol):
def count_unequal_element(data_expected, data_me, rtol, atol):
""" Precision calculation func """
assert data_expected.shape == data_me.shape
total_count = len(data_expected.flatten())
error = np.abs(data_expected - data_me)
greater = np.greater(error, atol + np.abs(data_expected) * rtol)
loss_count = np.count_nonzero(greater)
assert (loss_count / total_count) < rtol, \
"\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \
format(data_expected[greater], data_me[greater], error[greater])
assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format(
data_expected[greater], data_me[greater], error[greater])
def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True):
@ -52,16 +50,14 @@ def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True):
if np.any(np.isnan(data_expected)):
assert np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan)
elif not np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan):
_count_unequal_element(data_expected, data_me, rtol, atol)
else:
assert True
count_unequal_element(data_expected, data_me, rtol, atol)
def test_func_frequency_masking_eager_random_input():
""" mindspore eager mode normal testcase:frequency_masking op"""
logger.info("test frequency_masking op")
spectrogram = next(gen((CHANNEL, FREQ, TIME)))[0]
out_put = atf.FrequencyMasking(False, 3, 1, 10)(spectrogram)
out_put = audio.FrequencyMasking(False, 3, 1, 10)(spectrogram)
assert out_put.shape == (CHANNEL, FREQ, TIME)
@ -74,7 +70,7 @@ def test_func_frequency_masking_eager_precision():
[[-1.205032, 0.18922766, -0.5277673, -1.3090396],
[1.8914849, -0.97001046, -0.23726775, 0.00525892],
[-1.0271876, 0.33526883, 1.7413973, 0.12313101]]]).astype(np.float32)
out_ms = atf.FrequencyMasking(False, 2, 0, 0)(spectrogram)
out_ms = audio.FrequencyMasking(False, 2, 0, 0)(spectrogram)
out_benchmark = np.array([[[0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0],
[1.8175547, -0.25779432, -0.58152324, -0.00221091]],
@ -89,14 +85,10 @@ def test_func_frequency_masking_pipeline():
logger.info("test frequency_masking op, pipeline")
generator = gen([CHANNEL, FREQ, TIME])
data1 = ds.GeneratorDataset(source=generator, column_names=[
"multi_dimensional_data"])
data1 = ds.GeneratorDataset(source=generator, column_names=["multi_dimensional_data"])
transforms = [
atf.FrequencyMasking(True, 8)
]
data1 = data1.map(operations=transforms, input_columns=[
"multi_dimensional_data"])
transforms = [audio.FrequencyMasking(True, 8)]
data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
out_put = item["multi_dimensional_data"]
@ -107,14 +99,14 @@ def test_frequency_masking_invalid_input():
def test_invalid_param(test_name, iid_masks, frequency_mask_param, mask_start, error, error_msg):
logger.info("Test FrequencyMasking with wrong params: {0}".format(test_name))
with pytest.raises(error) as error_info:
atf.FrequencyMasking(iid_masks, frequency_mask_param, mask_start)
audio.FrequencyMasking(iid_masks, frequency_mask_param, mask_start)
assert error_msg in str(error_info.value)
def test_invalid_input(test_name, iid_masks, frequency_mask_param, mask_start, error, error_msg):
logger.info("Test FrequencyMasking with wrong params: {0}".format(test_name))
with pytest.raises(error) as error_info:
spectrogram = next(gen((CHANNEL, FREQ, TIME)))[0]
_ = atf.FrequencyMasking(iid_masks, frequency_mask_param, mask_start)(spectrogram)
_ = audio.FrequencyMasking(iid_masks, frequency_mask_param, mask_start)(spectrogram)
assert error_msg in str(error_info.value)
test_invalid_param("invalid mask_start", True, 2, -10, ValueError,

2
tests/ut/python/dataset/test_rgb_bgr.py
View File

@ -24,8 +24,6 @@ import mindspore.dataset.vision.c_transforms as vision
import mindspore.dataset.vision.py_transforms as py_vision
import mindspore.dataset.vision.py_transforms_util as util
GENERATE_GOLDEN = False
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json"

34
tests/ut/python/dataset/test_time_masking.py
View File

@ -20,10 +20,9 @@ import numpy as np
import pytest
import mindspore.dataset as ds
import mindspore.dataset.audio.transforms as atf
import mindspore.dataset.audio.transforms as audio
from mindspore import log as logger
CHANNEL = 2
FREQ = 20
TIME = 30
@ -32,19 +31,18 @@ TIME = 30
def gen(shape):
np.random.seed(0)
data = np.random.random(shape)
yield(np.array(data, dtype=np.float32),)
yield (np.array(data, dtype=np.float32),)
def _count_unequal_element(data_expected, data_me, rtol, atol):
def count_unequal_element(data_expected, data_me, rtol, atol):
""" Precision calculation func """
assert data_expected.shape == data_me.shape
total_count = len(data_expected.flatten())
error = np.abs(data_expected - data_me)
greater = np.greater(error, atol + np.abs(data_expected) * rtol)
loss_count = np.count_nonzero(greater)
assert (loss_count / total_count) < rtol, \
"\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \
format(data_expected[greater], data_me[greater], error[greater])
assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format(
data_expected[greater], data_me[greater], error[greater])
def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True):
@ -52,16 +50,14 @@ def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True):
if np.any(np.isnan(data_expected)):
assert np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan)
elif not np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan):
_count_unequal_element(data_expected, data_me, rtol, atol)
else:
assert True
count_unequal_element(data_expected, data_me, rtol, atol)
def test_func_time_masking_eager_random_input():
""" mindspore eager mode normal testcase:time_masking op"""
logger.info("test time_masking op")
spectrogram = next(gen((CHANNEL, FREQ, TIME)))[0]
out_put = atf.TimeMasking(False, 3, 1, 10)(spectrogram)
out_put = audio.TimeMasking(False, 3, 1, 10)(spectrogram)
assert out_put.shape == (CHANNEL, FREQ, TIME)
@ -74,7 +70,7 @@ def test_func_time_masking_eager_precision():
[[-1.205032, 0.18922766, -0.5277673, -1.3090396],
[1.8914849, -0.97001046, -0.23726775, 0.00525892],
[-1.0271876, 0.33526883, 1.7413973, 0.12313101]]]).astype(np.float32)
out_ms = atf.TimeMasking(False, 2, 0, 0)(spectrogram)
out_ms = audio.TimeMasking(False, 2, 0, 0)(spectrogram)
out_benchmark = np.array([[[0., 0., 0.07162686, -0.45436913],
[0., 0., 0.62333095, -0.09532598],
[0., 0., -0.58152324, -0.00221091]],
@ -89,14 +85,10 @@ def test_func_time_masking_pipeline():
logger.info("test time_masking op, pipeline")
generator = gen([CHANNEL, FREQ, TIME])
data1 = ds.GeneratorDataset(source=generator, column_names=[
"multi_dimensional_data"])
data1 = ds.GeneratorDataset(source=generator, column_names=["multi_dimensional_data"])
transforms = [
atf.TimeMasking(True, 8)
]
data1 = data1.map(operations=transforms, input_columns=[
"multi_dimensional_data"])
transforms = [audio.TimeMasking(True, 8)]
data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
out_put = item["multi_dimensional_data"]
@ -107,14 +99,14 @@ def test_time_masking_invalid_input():
def test_invalid_param(test_name, iid_masks, time_mask_param, mask_start, error, error_msg):
logger.info("Test TimeMasking with wrong params: {0}".format(test_name))
with pytest.raises(error) as error_info:
atf.TimeMasking(iid_masks, time_mask_param, mask_start)
audio.TimeMasking(iid_masks, time_mask_param, mask_start)
assert error_msg in str(error_info.value)
def test_invalid_input(test_name, iid_masks, time_mask_param, mask_start, error, error_msg):
logger.info("Test TimeMasking with wrong params: {0}".format(test_name))
with pytest.raises(error) as error_info:
spectrogram = next(gen((CHANNEL, FREQ, TIME)))[0]
_ = atf.TimeMasking(iid_masks, time_mask_param, mask_start)(spectrogram)
_ = audio.TimeMasking(iid_masks, time_mask_param, mask_start)(spectrogram)
assert error_msg in str(error_info.value)
test_invalid_param("invalid mask_start", True, 2, -10, ValueError,

39
tests/ut/python/dataset/test_time_stretch.py
View File

@ -31,27 +31,24 @@ COMPLEX = 2
def gen(shape):
np.random.seed(0)
data = np.random.random(shape)
yield(np.array(data, dtype=np.float32),)
yield (np.array(data, dtype=np.float32),)
def _count_unequal_element(data_expected, data_me, rtol, atol):
def count_unequal_element(data_expected, data_me, rtol, atol):
assert data_expected.shape == data_me.shape
total_count = len(data_expected.flatten())
error = np.abs(data_expected - data_me)
greater = np.greater(error, atol + np.abs(data_expected) * rtol)
loss_count = np.count_nonzero(greater)
assert (loss_count / total_count) < rtol, \
"\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}". \
format(data_expected[greater], data_me[greater], error[greater])
assert (loss_count / total_count) < rtol, "\ndata_expected_std:{0}\ndata_me_error:{1}\nloss:{2}".format(
data_expected[greater], data_me[greater], error[greater])
def allclose_nparray(data_expected, data_me, rtol, atol, equal_nan=True):
if np.any(np.isnan(data_expected)):
assert np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan)
elif not np.allclose(data_me, data_expected, rtol, atol, equal_nan=equal_nan):
_count_unequal_element(data_expected, data_me, rtol, atol)
else:
assert True
count_unequal_element(data_expected, data_me, rtol, atol)
def test_time_stretch_pipeline():
@ -60,18 +57,14 @@ def test_time_stretch_pipeline():
"""
logger.info("test TimeStretch op")
generator = gen([CHANNEL_NUM, FREQ, FRAME_NUM, COMPLEX])
data1 = ds.GeneratorDataset(source=generator, column_names=[
"multi_dimensional_data"])
data1 = ds.GeneratorDataset(source=generator, column_names=["multi_dimensional_data"])
transforms = [
c_audio.TimeStretch(512, FREQ, 1.3)
]
data1 = data1.map(operations=transforms, input_columns=[
"multi_dimensional_data"])
transforms = [c_audio.TimeStretch(512, FREQ, 1.3)]
data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
out_put = item["multi_dimensional_data"]
assert out_put.shape == (CHANNEL_NUM, FREQ, np.ceil(FRAME_NUM/1.3), COMPLEX)
assert out_put.shape == (CHANNEL_NUM, FREQ, np.ceil(FRAME_NUM / 1.3), COMPLEX)
def test_time_stretch_pipeline_invalid_param():
@ -80,19 +73,15 @@ def test_time_stretch_pipeline_invalid_param():
"""
logger.info("test TimeStretch op with invalid values")
generator = gen([CHANNEL_NUM, FREQ, FRAME_NUM, COMPLEX])
data1 = ds.GeneratorDataset(source=generator, column_names=[
"multi_dimensional_data"])
data1 = ds.GeneratorDataset(source=generator, column_names=["multi_dimensional_data"])
with pytest.raises(ValueError, match=r"Input fixed_rate is not within the required interval of \(0, 16777216\]."):
transforms = [
c_audio.TimeStretch(512, FREQ, -1.3)
]
data1 = data1.map(operations=transforms, input_columns=[
"multi_dimensional_data"])
transforms = [c_audio.TimeStretch(512, FREQ, -1.3)]
data1 = data1.map(operations=transforms, input_columns=["multi_dimensional_data"])
for item in data1.create_dict_iterator(num_epochs=1, output_numpy=True):
out_put = item["multi_dimensional_data"]
assert out_put.shape == (CHANNEL_NUM, FREQ, np.ceil(FRAME_NUM/1.3), COMPLEX)
assert out_put.shape == (CHANNEL_NUM, FREQ, np.ceil(FRAME_NUM / 1.3), COMPLEX)
def test_time_stretch_eager():
@ -102,7 +91,7 @@ def test_time_stretch_eager():
logger.info("test TimeStretch op with customized parameter values")
spectrogram = next(gen([CHANNEL_NUM, FREQ, FRAME_NUM, COMPLEX]))[0]
out_put = c_audio.TimeStretch(512, FREQ, 1.3)(spectrogram)
assert out_put.shape == (CHANNEL_NUM, FREQ, np.ceil(FRAME_NUM/1.3), COMPLEX)
assert out_put.shape == (CHANNEL_NUM, FREQ, np.ceil(FRAME_NUM / 1.3), COMPLEX)
def test_percision_time_stretch_eager():