p85620714
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!23704 fix move profiling 

Merge pull request !23704 from shenwei41/operation_dev
This commit is contained in:
i-robot 2021-09-19 11:05:02 +00:00 committed by Gitee
parent cc6da6bc10 42f1cc9e48
commit d474ae0e67
84 changed files with 22 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mindspore/ccsrc/minddata/dataset/api/audio.cc
View File

@ -42,7 +42,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
// AllpassBiquad Transform Operation.
struct AllpassBiquad::Data {

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/allpass_biquad_ir.cc
View File

@ -22,7 +22,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
// AllpassBiquadOperation
AllpassBiquadOperation::AllpassBiquadOperation(int32_t sample_rate, float central_freq, float Q)
: sample_rate_(sample_rate), central_freq_(central_freq), Q_(Q) {}

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/amplitude_to_db_ir.cc
View File

@ -22,7 +22,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
// AmplitudeToDBOperation
AmplitudeToDBOperation::AmplitudeToDBOperation(ScaleType stype, float ref_value, float amin, float top_db)
: stype_(stype), ref_value_(ref_value), amin_(amin), top_db_(top_db) {}

2
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/angle_ir.cc
View File

@ -21,7 +21,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
// AngleOperation
AngleOperation::AngleOperation() {}
@ -31,7 +30,6 @@ std::shared_ptr<TensorOp> AngleOperation::Build() {
std::shared_ptr<AngleOp> tensor_op = std::make_shared<AngleOp>();
return tensor_op;
}
} // namespace audio
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/band_biquad_ir.cc
View File

@ -22,7 +22,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
// BandBiquadOperation
BandBiquadOperation::BandBiquadOperation(int32_t sample_rate, float central_freq, float Q, bool noise)
: sample_rate_(sample_rate), central_freq_(central_freq), Q_(Q), noise_(noise) {}

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandpass_biquad_ir.cc
View File

@ -22,7 +22,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
// BandpassBiquadOperation
BandpassBiquadOperation::BandpassBiquadOperation(int32_t sample_rate, float central_freq, float Q,
bool const_skirt_gain)

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bandreject_biquad_ir.cc
View File

@ -22,7 +22,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
// BandrejectBiquadOperation
BandrejectBiquadOperation::BandrejectBiquadOperation(int32_t sample_rate, float central_freq, float Q)
: sample_rate_(sample_rate), central_freq_(central_freq), Q_(Q) {}

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/bass_biquad_ir.cc
View File

@ -22,7 +22,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
// BassBiquadOperation
BassBiquadOperation::BassBiquadOperation(int32_t sample_rate, float gain, float central_freq, float Q)
: sample_rate_(sample_rate), gain_(gain), central_freq_(central_freq), Q_(Q) {}

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/biquad_ir.cc
View File

@ -47,7 +47,6 @@ Status BiquadOperation::to_json(nlohmann::json *out_json) {
*out_json = args;
return Status::OK();
}
} // namespace audio
} // namespace dataset
} // namespace mindspore

3
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/complex_norm_ir.cc
View File

@ -21,9 +21,7 @@
namespace mindspore {
namespace dataset {
namespace audio {
ComplexNormOperation::ComplexNormOperation(float power) : power_(power) {}
ComplexNormOperation::~ComplexNormOperation() = default;
@ -46,7 +44,6 @@ std::shared_ptr<TensorOp> ComplexNormOperation::Build() {
}
std::string ComplexNormOperation::Name() const { return kComplexNormOperation; }
} // namespace audio
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/contrast_ir.cc
View File

@ -22,7 +22,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
// ContrastOperation
ContrastOperation::ContrastOperation(float enhancement_amount) : enhancement_amount_(enhancement_amount) {}

2
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/dc_shift_ir.cc
View File

@ -22,7 +22,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
// DCShiftOperation
DCShiftOperation::DCShiftOperation(float shift, float limiter_gain) : shift_(shift), limiter_gain_(limiter_gain) {}
@ -43,7 +42,6 @@ Status DCShiftOperation::to_json(nlohmann::json *out_json) {
*out_json = args;
return Status::OK();
}
} // namespace audio
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/equalizer_biquad_ir.cc
View File

@ -22,7 +22,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
EqualizerBiquadOperation::EqualizerBiquadOperation(int32_t sample_rate, float center_freq, float gain, float Q)
: sample_rate_(sample_rate), center_freq_(center_freq), gain_(gain), Q_(Q) {}

2
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/frequency_masking_ir.cc
View File

@ -20,9 +20,7 @@
namespace mindspore {
namespace dataset {
namespace audio {
FrequencyMaskingOperation::FrequencyMaskingOperation(bool iid_masks, int32_t frequency_mask_param, int32_t mask_start,
float mask_value)
: iid_masks_(iid_masks),

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/highpass_biquad_ir.cc
View File

@ -21,7 +21,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
// HighpassBiquadOperation
HighpassBiquadOperation::HighpassBiquadOperation(int32_t sample_rate, float cutoff_freq, float Q)

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/lfilter_ir.cc
View File

@ -21,7 +21,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
// LFilterOperation
LFilterOperation::LFilterOperation(std::vector<float> a_coeffs, std::vector<float> b_coeffs, bool clamp)

3
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/mu_law_decoding_ir.cc
View File

@ -20,9 +20,7 @@
namespace mindspore {
namespace dataset {
namespace audio {
MuLawDecodingOperation::MuLawDecodingOperation(int quantization_channels)
: quantization_channels_(quantization_channels) {}
@ -46,7 +44,6 @@ std::shared_ptr<TensorOp> MuLawDecodingOperation::Build() {
}
std::string MuLawDecodingOperation::Name() const { return kMuLawDecodingOperation; }
} // namespace audio
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/time_masking_ir.cc
View File

@ -22,7 +22,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
TimeMaskingOperation::TimeMaskingOperation(bool iid_masks, int32_t time_mask_param, int32_t mask_start,
float mask_value)
: iid_masks_(iid_masks), time_mask_param_(time_mask_param), mask_start_(mask_start), mask_value_(mask_value) {}

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/time_stretch_ir.cc
View File

@ -21,7 +21,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
// TimeStretchOperation
TimeStretchOperation::TimeStretchOperation(float hop_length, int n_freq, float fixed_rate)
: hop_length_(hop_length), n_freq_(n_freq), fixed_rate_(fixed_rate) {}

1
mindspore/ccsrc/minddata/dataset/audio/ir/kernels/vol_ir.cc
View File

@ -21,7 +21,6 @@
namespace mindspore {
namespace dataset {
namespace audio {
// Vol
VolOperation::VolOperation(float gain, GainType gain_type) : gain_(gain), gain_type_(gain_type) {}

2
mindspore/ccsrc/minddata/dataset/audio/ir/validators.cc
View File

@ -17,7 +17,6 @@
namespace mindspore {
namespace dataset {
Status ValidateIntScalarNonNegative(const std::string &op_name, const std::string &scalar_name, int32_t scalar) {
RETURN_IF_NOT_OK(ValidateScalar(op_name, scalar_name, scalar, {0}, false));
return Status::OK();
@ -31,6 +30,5 @@ Status ValidateFloatScalarNotNan(const std::string &op_name, const std::string &
}
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/audio/kernels/allpass_biquad_op.cc
View File

@ -20,7 +20,6 @@
namespace mindspore {
namespace dataset {
Status AllpassBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
TensorShape input_shape = input->shape();

2
mindspore/ccsrc/minddata/dataset/audio/kernels/amplitude_to_db_op.cc
View File

@ -21,7 +21,6 @@
namespace mindspore {
namespace dataset {
Status AmplitudeToDBOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
if (input->shape().Rank() < 2) {
@ -50,6 +49,5 @@ Status AmplitudeToDBOp::Compute(const std::shared_ptr<Tensor> &input, std::share
return AmplitudeToDB<double>(input_tensor, output, multiplier, amin, db_multiplier, top_db);
}
}
} // namespace dataset
} // namespace mindspore

2
mindspore/ccsrc/minddata/dataset/audio/kernels/angle_op.cc
View File

@ -21,7 +21,6 @@
namespace mindspore {
namespace dataset {
Status AngleOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
// if If the last dimension is not 2, then it's not a complex number
@ -59,6 +58,5 @@ Status AngleOp::OutputType(const std::vector<DataType> &inputs, std::vector<Data
}
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/audio/kernels/audio_utils.cc
View File

@ -26,7 +26,6 @@
namespace mindspore {
namespace dataset {
/// \brief Generate linearly spaced vector.
/// \param[in] start - Value of the startpoint.
/// \param[in] end - Value of the endpoint.

2
mindspore/ccsrc/minddata/dataset/audio/kernels/band_biquad_op.cc
View File

@ -20,7 +20,6 @@
namespace mindspore {
namespace dataset {
Status BandBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
TensorShape input_shape = input->shape();
@ -56,6 +55,5 @@ Status BandBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_p
static_cast<float16>(a0), static_cast<float16>(a1), static_cast<float16>(a2));
}
}
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/audio/kernels/bandpass_biquad_op.cc
View File

@ -20,7 +20,6 @@
namespace mindspore {
namespace dataset {
Status BandpassBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
TensorShape input_shape = input->shape();

1
mindspore/ccsrc/minddata/dataset/audio/kernels/bandreject_biquad_op.cc
View File

@ -20,7 +20,6 @@
namespace mindspore {
namespace dataset {
Status BandrejectBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
// check input type and input shape

1
mindspore/ccsrc/minddata/dataset/audio/kernels/bass_biquad_op.cc
View File

@ -20,7 +20,6 @@
namespace mindspore {
namespace dataset {
Status BassBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
TensorShape input_shape = input->shape();

2
mindspore/ccsrc/minddata/dataset/audio/kernels/complex_norm_op.cc
View File

@ -19,7 +19,6 @@
namespace mindspore {
namespace dataset {
// constructor
ComplexNormOp::ComplexNormOp(float power) : power_(power) {}
@ -53,6 +52,5 @@ Status ComplexNormOp::OutputType(const std::vector<DataType> &inputs, std::vecto
}
void ComplexNormOp::Print(std::ostream &out) const { out << "ComplexNormOp: power " << power_; }
} // namespace dataset
} // namespace mindspore

2
mindspore/ccsrc/minddata/dataset/audio/kernels/contrast_op.cc
View File

@ -22,7 +22,6 @@
namespace mindspore {
namespace dataset {
Status ContrastOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
TensorShape input_shape = input->shape();
@ -54,6 +53,5 @@ Status ContrastOp::OutputType(const std::vector<DataType> &inputs, std::vector<D
}
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

2
mindspore/ccsrc/minddata/dataset/audio/kernels/dc_shift_op.cc
View File

@ -20,7 +20,6 @@
namespace mindspore {
namespace dataset {
Status DCShiftOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
// input <..., time>.
@ -50,6 +49,5 @@ Status DCShiftOp::OutputType(const std::vector<DataType> &inputs, std::vector<Da
}
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/audio/kernels/equalizer_biquad_op.cc
View File

@ -20,7 +20,6 @@
namespace mindspore {
namespace dataset {
constexpr float_t EqualizerBiquadOp::kQ = 0.707;
Status EqualizerBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {

1
mindspore/ccsrc/minddata/dataset/audio/kernels/fade_op.cc
View File

@ -21,7 +21,6 @@
namespace mindspore {
namespace dataset {
constexpr int32_t FadeOp::kFadeInLen = 0;
constexpr int32_t FadeOp::kFadeOutLen = 0;
constexpr FadeShape FadeOp::kFadeShape = FadeShape::kLinear;

1
mindspore/ccsrc/minddata/dataset/audio/kernels/frequency_masking_op.cc
View File

@ -22,7 +22,6 @@
namespace mindspore {
namespace dataset {
// constructor
FrequencyMaskingOp::FrequencyMaskingOp(bool iid_masks, int32_t frequency_mask_param, int32_t mask_start,
float mask_value)

1
mindspore/ccsrc/minddata/dataset/audio/kernels/highpass_biquad_op.cc
View File

@ -20,7 +20,6 @@
namespace mindspore {
namespace dataset {
Status HighpassBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
TensorShape input_shape = input->shape();

1
mindspore/ccsrc/minddata/dataset/audio/kernels/lfilter_op.cc
View File

@ -20,7 +20,6 @@
namespace mindspore {
namespace dataset {
Status LFilterOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
TensorShape input_shape = input->shape();

2
mindspore/ccsrc/minddata/dataset/audio/kernels/lowpass_biquad_op.cc
View File

@ -21,7 +21,6 @@
namespace mindspore {
namespace dataset {
const float LowpassBiquadOp::kQ = 0.707;
// constructor
@ -55,6 +54,5 @@ Status LowpassBiquadOp::Compute(const std::shared_ptr<Tensor> &input, std::share
static_cast<float16>(a0), static_cast<float16>(a1), static_cast<float16>(a2));
}
}
} // namespace dataset
} // namespace mindspore

2
mindspore/ccsrc/minddata/dataset/audio/kernels/mu_law_decoding_op.cc
View File

@ -19,7 +19,6 @@
namespace mindspore {
namespace dataset {
// constructor
MuLawDecodingOp::MuLawDecodingOp(int quantization_channels) : quantization_channels_(quantization_channels) {}
@ -49,6 +48,5 @@ Status MuLawDecodingOp::OutputType(const std::vector<DataType> &inputs, std::vec
}
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/audio/kernels/time_masking_op.cc
View File

@ -22,7 +22,6 @@
namespace mindspore {
namespace dataset {
// constructor
TimeMaskingOp::TimeMaskingOp(bool iid_masks, int32_t time_mask_param, int32_t mask_start, float mask_value)
: time_mask_param_(time_mask_param), mask_start_(mask_start), iid_masks_(iid_masks), mask_value_(mask_value) {

1
mindspore/ccsrc/minddata/dataset/audio/kernels/time_stretch_op.cc
View File

@ -23,7 +23,6 @@
namespace mindspore {
namespace dataset {
const float TimeStretchOp::kHopLength = std::numeric_limits<float>::quiet_NaN();
const int TimeStretchOp::kNFreq = 201;
const float TimeStretchOp::kFixedRate = std::numeric_limits<float>::quiet_NaN();

2
mindspore/ccsrc/minddata/dataset/audio/kernels/vol_op.cc
View File

@ -21,7 +21,6 @@
namespace mindspore {
namespace dataset {
Status VolOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
std::shared_ptr<Tensor> input_tensor;
@ -50,6 +49,5 @@ Status VolOp::OutputType(const std::vector<DataType> &inputs, std::vector<DataTy
}
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/engine/cache/cache_hw.cc vendored
View File

@ -264,7 +264,6 @@ uint64_t CacheServerHW::GetAvailableMemory() {
// get title
std::string::size_type position = line.find(":");
std::string title = line.substr(0, position);
// get the value of MemAvailable
if (title == "MemAvailable") {
std::string::size_type pos1 = line.find_last_of(" ");

4
mindspore/ccsrc/minddata/dataset/engine/datasetops/device_queue_op.cc
View File

@ -399,13 +399,11 @@ Status DeviceQueueOp::PushDataToGPU() {
// Every thread use cuda api should SetThreadDevice
RETURN_IF_NOT_OK(SetThreadDevice());
TaskManager::FindMe()->Post();
#ifndef ENABLE_SECURITY
uint64_t batch_start_time = 0;
int32_t push_cost = 0;
#ifndef ENABLE_SECURITY
int32_t connector_size = 0;
int32_t connector_capacity = 0;
#endif
#ifndef ENABLE_SECURITY
std::shared_ptr<DeviceQueueTracing> profiling_node;
bool isProfilingEnable = tree_->GetProfilingManager()->IsProfilingEnable();
if (isProfilingEnable) {

1
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sbu_op.cc
View File

@ -229,6 +229,5 @@ Status SBUOp::ReplaceAll(std::string *str, const std::string &from, const std::s
}
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/usps_op.cc
View File

@ -347,6 +347,5 @@ Status USPSOp::ComputeColMap() {
}
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/engine/gnn/local_node.cc
View File

@ -140,7 +140,6 @@ Status LocalNode::AddAdjacent(const std::shared_ptr<Node> &node, const std::shar
Status LocalNode::GetEdgeByAdjNodeId(const NodeIdType &adj_node_id, EdgeIdType *out_edge_id) {
auto itr = adjacent_nodes_.find(adj_node_id);
if (itr != adjacent_nodes_.end()) {
(*out_edge_id) = itr->second;
} else {

2
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/cityscapes_node.cc
View File

@ -25,7 +25,6 @@
namespace mindspore {
namespace dataset {
// Constructor for CityscapesNode
CityscapesNode::CityscapesNode(const std::string &dataset_dir, const std::string &usage,
const std::string &quality_mode, const std::string &task, bool decode,
@ -114,7 +113,6 @@ Status CityscapesNode::GetDatasetSize(const std::shared_ptr<DatasetSizeGetter> &
std::shared_ptr<SamplerRT> sampler_rt = nullptr;
RETURN_IF_NOT_OK(sampler_->SamplerBuild(&sampler_rt));
sample_size = sampler_rt->CalculateNumSamples(num_rows);
if (sample_size == -1) {
RETURN_IF_NOT_OK(size_getter->DryRun(shared_from_this(), &sample_size));
}

1
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/div2k_node.cc
View File

@ -26,7 +26,6 @@
namespace mindspore {
namespace dataset {
// Constructor for DIV2KNode
DIV2KNode::DIV2KNode(const std::string &dataset_dir, const std::string &usage, const std::string &downgrade,
int32_t scale, bool decode, const std::shared_ptr<SamplerObj> &sampler,

2
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/flickr_node.cc
View File

@ -30,7 +30,6 @@
namespace mindspore {
namespace dataset {
// Constructor for FlickrNode
FlickrNode::FlickrNode(const std::string &dataset_dir, const std::string &annotation_file, bool decode,
std::shared_ptr<SamplerObj> sampler, std::shared_ptr<DatasetCache> cache)
@ -126,7 +125,6 @@ Status FlickrNode::GetDatasetSize(const std::shared_ptr<DatasetSizeGetter> &size
std::shared_ptr<SamplerRT> sampler_rt = nullptr;
RETURN_IF_NOT_OK(sampler_->SamplerBuild(&sampler_rt));
sample_size = sampler_rt->CalculateNumSamples(num_rows);
if (sample_size == -1) {
RETURN_IF_NOT_OK(size_getter->DryRun(shared_from_this(), &sample_size));
}

1
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/sbu_node.cc
View File

@ -26,7 +26,6 @@
namespace mindspore {
namespace dataset {
SBUNode::SBUNode(const std::string &dataset_dir, bool decode, const std::shared_ptr<SamplerObj> &sampler,
const std::shared_ptr<DatasetCache> &cache)
: MappableSourceNode(std::move(cache)), dataset_dir_(dataset_dir), decode_(decode), sampler_(sampler) {}

2
mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/usps_node.cc
View File

@ -27,7 +27,6 @@
namespace mindspore {
namespace dataset {
USPSNode::USPSNode(std::string dataset_dir, std::string usage, int32_t num_samples, ShuffleMode shuffle,
int32_t num_shards, int32_t shard_id, std::shared_ptr<DatasetCache> cache)
: NonMappableSourceNode(std::move(cache)),
@ -168,6 +167,5 @@ Status USPSNode::MakeSimpleProducer() {
num_samples_ = 0;
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

2
mindspore/ccsrc/minddata/dataset/engine/tdt/tdt_plugin.cc
View File

@ -25,7 +25,6 @@
namespace mindspore {
namespace dataset {
constexpr auto kUnknownErrorString = "Unknown error occurred";
TdtPlugin::TdtPlugin(const std::string &channel_name, int32_t device_id) {
@ -240,6 +239,5 @@ void TdtPlugin::ReportErrorMessage() {
MS_LOG(ERROR) << "Ascend error occurred, error message:\n" << error_message;
}
}
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/kernels/data/one_hot_op.cc
View File

@ -37,6 +37,5 @@ Status OneHotOp::OutputShape(const std::vector<TensorShape> &inputs, std::vector
if (!outputs.empty()) return Status::OK();
return Status(StatusCode::kMDUnexpectedError, "OneHot: invalid input shape.");
}
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/kernels/data/type_cast_op.cc
View File

@ -34,6 +34,5 @@ Status TypeCastOp::OutputType(const std::vector<DataType> &inputs, std::vector<D
outputs[0] = type_;
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/kernels/image/adjust_gamma_op.cc
View File

@ -21,7 +21,6 @@
namespace mindspore {
namespace dataset {
constexpr float AdjustGammaOp::kGain = 1.0;
Status AdjustGammaOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {

1
mindspore/ccsrc/minddata/dataset/kernels/image/gaussian_blur_op.cc
View File

@ -24,7 +24,6 @@
namespace mindspore {
namespace dataset {
Status GaussianBlurOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
RETURN_IF_NOT_OK(ValidateImageRank("GaussianBlur", input->Rank()));

5
mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc
View File

@ -489,9 +489,9 @@ Status ConvertColor(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor
if (!input_cv->mat().data) {
RETURN_STATUS_UNEXPECTED("[Internal ERROR] ConvertColor: load image failed.");
}
if (input_cv->Rank() == 3) {
if (input_cv->Rank() == DEFAULT_IMAGE_RANK) {
int num_channels = input_cv->shape()[CHANNEL_INDEX];
if (num_channels != 3 && num_channels != 4) {
if (num_channels != DEFAULT_IMAGE_CHANNELS && num_channels != MAX_IMAGE_CHANNELS) {
RETURN_STATUS_UNEXPECTED("ConvertColor: number of channels of image should be 3 or 4, but got:" +
std::to_string(num_channels));
}
@ -991,7 +991,6 @@ Status AdjustGamma(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor>
*itr = std::min(std::max((*itr), 0.0f), 1.0f);
}
*output = input;
} else {
std::shared_ptr<CVTensor> input_cv = CVTensor::AsCVTensor(input);
if (!input_cv->mat().data) {

1
mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.h
View File

@ -42,6 +42,7 @@
#define DEFAULT_IMAGE_RANK 3 // images are hwc channels in general
#define MAX_BIT_VALUE 255 // max bit value after decode is 256
#define MIN_IMAGE_CHANNELS 1 // image ops support minimum of 1 channel
#define MAX_IMAGE_CHANNELS 4 // image ops support maximum of 4 channel
#define MIN_IMAGE_DIMENSION 2 // images are at least 2 dimensional
namespace mindspore {
namespace dataset {

5
mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_and_resize_op.cc
View File

@ -49,6 +49,11 @@ Status RandomCropAndResizeOp::Compute(const TensorRow &input, TensorRow *output)
IO_CHECK_VECTOR(input, output);
if (input.size() != 1) {
for (size_t i = 0; i < input.size() - 1; i++) {
if (input[i]->Rank() != 2 && input[i]->Rank() != 3) {
std::string err_msg = "RandomCropAndResizeOp: image shape is not <H,W,C> or <H, W>, but got rank:" +
std::to_string(input[i]->Rank());
RETURN_STATUS_UNEXPECTED(err_msg);
}
if (input[i]->shape()[0] != input[i + 1]->shape()[0] || input[i]->shape()[1] != input[i + 1]->shape()[1]) {
RETURN_STATUS_UNEXPECTED("RandomCropAndResizeOp: The width and height of the image need to be the same size.");
}

5
mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_op.cc
View File

@ -122,6 +122,11 @@ Status RandomCropOp::Compute(const TensorRow &input, TensorRow *output) {
IO_CHECK_VECTOR(input, output);
if (input.size() != 1) {
for (size_t i = 0; i < input.size() - 1; i++) {
if (input[i]->Rank() != 2 && input[i]->Rank() != 3) {
std::string err_msg =
"RandomCropOp: image shape is not <H,W,C> or <H, W>, but got rank:" + std::to_string(input[i]->Rank());
RETURN_STATUS_UNEXPECTED(err_msg);
}
if (input[i]->shape()[0] != input[i + 1]->shape()[0] || input[i]->shape()[1] != input[i + 1]->shape()[1]) {
RETURN_STATUS_UNEXPECTED("RandomCropOp: The width and height of the image need to be the same size.");
}

1
mindspore/ccsrc/minddata/dataset/kernels/image/resize_cubic_op.cc
View File

@ -17,7 +17,6 @@
namespace mindspore {
namespace dataset {
// using 8 bits for result
constexpr uint8_t PrecisionBits = 22;

2
mindspore/ccsrc/minddata/dataset/kernels/image/rgb_to_bgr_op.cc
View File

@ -22,7 +22,6 @@
namespace mindspore {
namespace dataset {
Status RgbToBgrOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output);
auto input_type = input->type();
@ -32,6 +31,5 @@ Status RgbToBgrOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr
return RgbToBgr(input, output);
}
} // namespace dataset
} // namespace mindspore

2
mindspore/ccsrc/minddata/dataset/kernels/image/slice_patches_op.cc
View File

@ -19,7 +19,6 @@
namespace mindspore {
namespace dataset {
const int32_t SlicePatchesOp::kDefNumH = 1;
const int32_t SlicePatchesOp::kDefNumW = 1;
const uint8_t SlicePatchesOp::kDefFillV = 0;
@ -44,6 +43,5 @@ Status SlicePatchesOp::Compute(const TensorRow &input, TensorRow *output) {
(void)std::copy(out.begin(), out.end(), std::back_inserter(*output));
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

2
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/adjust_gamma_ir.cc
View File

@ -23,9 +23,7 @@
namespace mindspore {
namespace dataset {
namespace vision {
#ifndef ENABLE_ANDROID
// AdjustGammaOperation

1
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/affine_ir.cc
View File

@ -104,7 +104,6 @@ Status AffineOperation::from_json(nlohmann::json op_params, std::shared_ptr<Tens
*operation = std::make_shared<vision::AffineOperation>(degrees, translation, scale, shear, interpolation, fill_value);
return Status::OK();
}
} // namespace vision
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/center_crop_ir.cc
View File

@ -60,7 +60,6 @@ Status CenterCropOperation::from_json(nlohmann::json op_params, std::shared_ptr<
*operation = std::make_shared<CenterCropOperation>(size);
return Status::OK();
}
} // namespace vision
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/decode_ir.cc
View File

@ -45,7 +45,6 @@ Status DecodeOperation::from_json(nlohmann::json op_params, std::shared_ptr<Tens
*operation = std::make_shared<vision::DecodeOperation>(rgb);
return Status::OK();
}
} // namespace vision
} // namespace dataset
} // namespace mindspore

3
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/gaussian_blur_ir.cc
View File

@ -20,9 +20,7 @@
namespace mindspore {
namespace dataset {
namespace vision {
GaussianBlurOperation::GaussianBlurOperation(const std::vector<int32_t> kernel_size, const std::vector<float> sigma)
: kernel_size_(kernel_size), sigma_(sigma) {}
@ -72,7 +70,6 @@ Status GaussianBlurOperation::from_json(nlohmann::json op_params, std::shared_pt
*operation = std::make_shared<vision::GaussianBlurOperation>(kernel_size, sigma);
return Status::OK();
}
} // namespace vision
} // namespace dataset
} // namespace mindspore

3
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/horizontal_flip_ir.cc
View File

@ -21,9 +21,7 @@
namespace mindspore {
namespace dataset {
namespace vision {
#ifndef ENABLE_ANDROID
// VerticalFlipOperation
@ -45,7 +43,6 @@ Status HorizontalFlipOperation::from_json(nlohmann::json op_params, std::shared_
return Status::OK();
}
#endif
} // namespace vision
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/normalize_ir.cc
View File

@ -55,7 +55,6 @@ Status NormalizeOperation::from_json(nlohmann::json op_params, std::shared_ptr<T
*operation = std::make_shared<vision::NormalizeOperation>(mean, std);
return Status::OK();
}
} // namespace vision
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/random_affine_ir.cc
View File

@ -179,7 +179,6 @@ Status RandomAffineOperation::from_json(nlohmann::json op_params, std::shared_pt
interpolation, fill_value);
return Status::OK();
}
} // namespace vision
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/resize_ir.cc
View File

@ -79,7 +79,6 @@ Status ResizeOperation::from_json(nlohmann::json op_params, std::shared_ptr<Tens
*operation = std::make_shared<vision::ResizeOperation>(size, interpolation);
return Status::OK();
}
} // namespace vision
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/resize_preserve_ar_ir.cc
View File

@ -65,7 +65,6 @@ Status ResizePreserveAROperation::from_json(nlohmann::json op_params, std::share
*operation = std::make_shared<vision::ResizePreserveAROperation>(height, width, img_orientation);
return Status::OK();
}
} // namespace vision
} // namespace dataset
} // namespace mindspore

3
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/rgb_to_bgr_ir.cc
View File

@ -19,9 +19,7 @@
namespace mindspore {
namespace dataset {
namespace vision {
RgbToBgrOperation::RgbToBgrOperation() = default;
// RGB2BGROperation
@ -37,7 +35,6 @@ Status RgbToBgrOperation::from_json(nlohmann::json op_params, std::shared_ptr<Te
*operation = std::make_shared<vision::RgbToBgrOperation>();
return Status::OK();
}
} // namespace vision
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/rgb_to_gray_ir.cc
View File

@ -39,7 +39,6 @@ Status RgbToGrayOperation::from_json(nlohmann::json op_params, std::shared_ptr<T
*operation = std::make_shared<vision::RgbToGrayOperation>();
return Status::OK();
}
} // namespace vision
} // namespace dataset
} // namespace mindspore

1
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/slice_patches_ir.cc
View File

@ -21,7 +21,6 @@
namespace mindspore {
namespace dataset {
namespace vision {
// SlicePatchesOperation
SlicePatchesOperation::SlicePatchesOperation(int32_t num_height, int32_t num_width, SliceMode slice_mode,

3
mindspore/ccsrc/minddata/dataset/kernels/ir/vision/vertical_flip_ir.cc
View File

@ -21,9 +21,7 @@
namespace mindspore {
namespace dataset {
namespace vision {
#ifndef ENABLE_ANDROID
// VerticalFlipOperation
@ -46,7 +44,6 @@ Status VerticalFlipOperation::from_json(nlohmann::json op_params, std::shared_pt
}
#endif
} // namespace vision
} // namespace dataset
} // namespace mindspore

3
mindspore/dataset/engine/datasets.py
View File

@ -2426,6 +2426,7 @@ def wait_pid():
# waitpid may be failed for some reasons so we ignore this error
pass
# Dataset need _watch_dog thread to monitoring fork multi-processing,
# and thread can't be a member function otherwise python won't collect and release resources.
def _watch_dog(eot, pids):
@ -3710,7 +3711,7 @@ class SamplerFn:
if multi_process is True:
try:
worker = _GeneratorWorkerMp(dataset, self.eof, max_rowsize, queue_size)
except:
except Exception:
raise RuntimeError("Init multiprocessing.Queue() failed, This might be caused by insufficient shm,"
+ " and the recommended shm size is at least 5 GB.")
worker.daemon = True

1
mindspore/dataset/utils/browse_dataset.py
View File

@ -20,6 +20,7 @@ import importlib
import numpy as np
from mindspore import log as logger
def imshow_det_bbox(image,
bboxes,
labels,

4
mindspore/dataset/vision/c_transforms.py
View File

@ -48,10 +48,10 @@ from PIL import Image
import mindspore._c_dataengine as cde
from .utils import Inter, Border, ImageBatchFormat, ConvertMode, SliceMode
from .validators import check_prob, check_crop, check_center_crop, check_resize_interpolation, check_random_resize_crop, \
from .validators import check_prob, check_crop, check_center_crop, check_resize_interpolation, \
check_mix_up_batch_c, check_normalize_c, check_normalizepad_c, check_random_crop, check_random_color_adjust, \
check_random_rotation, check_range, check_resize, check_rescale, check_pad, check_cutout, \
check_uniform_augment_cpp, check_convert_color, \
check_uniform_augment_cpp, check_convert_color, check_random_resize_crop, \
check_bounding_box_augment_cpp, check_random_select_subpolicy_op, check_auto_contrast, check_random_affine, \
check_random_solarize, check_soft_dvpp_decode_random_crop_resize_jpeg, check_positive_degrees, FLOAT_MAX_INTEGER, \
check_cut_mix_batch_c, check_posterize, check_gaussian_blur, check_rotate, check_slice_patches, check_adjust_gamma

1
mindspore/dataset/vision/validators.py
View File

@ -965,6 +965,7 @@ def check_gaussian_blur(method):
return new_method
def check_convert_color(method):
"""Wrapper method to check the parameters of convertcolor."""

2
mindspore/mindrecord/tools/cifar10.py
View File

@ -66,7 +66,7 @@ def restricted_loads(s):
raise RuntimeError("Not a valid Cifar10 Dataset.")
except UnicodeDecodeError:
raise RuntimeError("Not a valid Cifar10 Dataset.")
except:
except Exception:
raise RuntimeError("Unexpected error while Unpickling Cifar10 Dataset.")

2
mindspore/mindrecord/tools/cifar100.py
View File

@ -65,7 +65,7 @@ def restricted_loads(s):
raise RuntimeError("Not a valid Cifar100 Dataset.")
except UnicodeDecodeError:
raise RuntimeError("Not a valid Cifar100 Dataset.")
except:
except Exception:
raise RuntimeError("Unexpected error while Unpickling Cifar100 Dataset.")