From 3c0129226ba5a1e35f889b2feeb4c1906cfb0a3c Mon Sep 17 00:00:00 2001 From: Lixia Chen Date: Mon, 12 Jul 2021 13:48:34 -0400 Subject: [PATCH] Fix codex, codecheck, shellcheck issues --- .../ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc | 2 +- .../ccsrc/minddata/dataset/engine/cache/storage_manager.cc | 5 +++-- .../dataset/engine/datasetops/source/random_data_op.cc | 1 - .../datasetops/source/sampler/mind_record_sampler.cc | 3 +-- .../ir/datasetops/source/samplers/mindrecord_sampler_ir.cc | 2 -- mindspore/ccsrc/minddata/dataset/util/path.cc | 3 +-- mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc | 3 +-- mindspore/ccsrc/minddata/mindrecord/meta/shard_shuffle.cc | 3 +-- .../ccsrc/minddata/mindrecord/meta/shard_task_list.cc | 3 ++- mindspore/dataset/core/validator_helpers.py | 7 +++++++ mindspore/dataset/transforms/py_transforms.py | 1 + mindspore/dataset/transforms/py_transforms_util.py | 2 +- .../official/cv/mobilenetv2/scripts/run_train_nfs_cache.sh | 1 + .../official/cv/resnet/scripts/run_distribute_train.sh | 1 + .../official/cv/resnet/scripts/run_distribute_train_gpu.sh | 1 + .../official/cv/resnet/scripts/run_standalone_train.sh | 1 + .../official/cv/resnet/scripts/run_standalone_train_gpu.sh | 1 + 17 files changed, 24 insertions(+), 16 deletions(-) diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc index d8e9514c31a..a50986d76cf 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/cache_admin_arg.cc @@ -129,7 +129,7 @@ Status CacheAdminArgHandler::AssignArg(const std::string &option, std::vector> value_as_uint; if (arg_stream->fail()) { arg_stream->clear(); - arg_stream->seekg(pos, arg_stream->beg); + (void)arg_stream->seekg(pos, std::ios::beg); break; } else { out_arg->push_back(value_as_uint); diff --git a/mindspore/ccsrc/minddata/dataset/engine/cache/storage_manager.cc b/mindspore/ccsrc/minddata/dataset/engine/cache/storage_manager.cc index 7f38d4f5d98..4c088a33281 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/cache/storage_manager.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/cache/storage_manager.cc @@ -59,6 +59,7 @@ Status StorageManager::DoServiceStart() { writable_containers_pool_.reserve(pool_size_); if (root_.IsDirectory()) { // create multiple containers and store their index in a pool + CHECK_FAIL_RETURN_UNEXPECTED(pool_size_ > 0, "Expect positive pool_size_, but got:" + std::to_string(pool_size_)); for (int i = 0; i < pool_size_; i++) { RETURN_IF_NOT_OK(AddOneContainer()); } @@ -172,10 +173,10 @@ Status StorageManager::DoServiceStop() noexcept { return rc1; } -StorageManager::StorageManager(const Path &root) : root_(root), pool_size_(1), file_id_(0), index_() {} +StorageManager::StorageManager(const Path &root) : root_(root), file_id_(0), index_(), pool_size_(1) {} StorageManager::StorageManager(const Path &root, int pool_size) - : root_(root), pool_size_(pool_size), file_id_(0), index_() {} + : root_(root), file_id_(0), index_(), pool_size_(pool_size) {} StorageManager::~StorageManager() { (void)StorageManager::DoServiceStop(); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc index d68f0b5e0e1..64cdb151a7d 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/random_data_op.cc @@ -248,7 +248,6 @@ Status RandomDataOp::WorkerEntry(int32_t worker_id) { // Send new_row out RETURN_IF_NOT_OK(out_connector_->Add(std::move(new_row), worker_id)); - } else { // Now, let's enter the epoch sync RETURN_IF_NOT_OK(EpochSync(worker_id, &quitting)); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/mind_record_sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/mind_record_sampler.cc index e28ea44ef35..a0e9076eac7 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/mind_record_sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/mind_record_sampler.cc @@ -24,7 +24,7 @@ namespace mindspore { namespace dataset { MindRecordSamplerRT::MindRecordSamplerRT(mindrecord::ShardReader *shard_reader, int64_t samples_per_tensor) - : SamplerRT(0, samples_per_tensor), next_id_(0), shard_reader_(shard_reader) {} + : SamplerRT(0, samples_per_tensor), shard_reader_(shard_reader), sample_ids_(nullptr), next_id_(0) {} Status MindRecordSamplerRT::GetNextSample(TensorRow *out) { if (next_id_ > num_samples_) { @@ -50,7 +50,6 @@ Status MindRecordSamplerRT::GetNextSample(TensorRow *out) { Status MindRecordSamplerRT::InitSampler() { sample_ids_ = shard_reader_->GetSampleIds(); - if (!sample_ids_) { // Note, sample_ids_.empty() is okay and will just give no sample ids. RETURN_STATUS_UNEXPECTED( diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/samplers/mindrecord_sampler_ir.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/samplers/mindrecord_sampler_ir.cc index fd07a2f3d10..d2babdf39a4 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/samplers/mindrecord_sampler_ir.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/samplers/mindrecord_sampler_ir.cc @@ -26,7 +26,6 @@ namespace mindspore { namespace dataset { - #ifndef ENABLE_ANDROID // This function not only creates a runtime sampler object, but also creates a ShardReader, // which will also be needed to build a runtime MindRecordOp @@ -51,6 +50,5 @@ Status MindRecordSamplerObj::GetShardReader(std::unique_ptr(task); diff --git a/mindspore/ccsrc/minddata/mindrecord/meta/shard_shuffle.cc b/mindspore/ccsrc/minddata/mindrecord/meta/shard_shuffle.cc index 5cd3f754ac8..fe472a9cd89 100644 --- a/mindspore/ccsrc/minddata/mindrecord/meta/shard_shuffle.cc +++ b/mindspore/ccsrc/minddata/mindrecord/meta/shard_shuffle.cc @@ -43,8 +43,7 @@ int64_t ShardShuffle::GetNumSamples(int64_t dataset_size, int64_t num_classes) { } MSRStatus ShardShuffle::CategoryShuffle(ShardTaskList &tasks) { - uint32_t individual_size; - individual_size = tasks.sample_ids_.size() / tasks.categories; + uint32_t individual_size = tasks.sample_ids_.size() / tasks.categories; std::vector> new_permutations(tasks.categories, std::vector(individual_size)); for (uint32_t i = 0; i < tasks.categories; i++) { for (uint32_t j = 0; j < individual_size; j++) new_permutations[i][j] = static_cast(j); diff --git a/mindspore/ccsrc/minddata/mindrecord/meta/shard_task_list.cc b/mindspore/ccsrc/minddata/mindrecord/meta/shard_task_list.cc index a4d74ad0fd5..5f52630ccf5 100644 --- a/mindspore/ccsrc/minddata/mindrecord/meta/shard_task_list.cc +++ b/mindspore/ccsrc/minddata/mindrecord/meta/shard_task_list.cc @@ -79,7 +79,8 @@ uint32_t ShardTaskList::SizeOfRows() const { if (task_list_.size() == 0) return static_cast(0); // 1 task is 1 page - auto sum_num_rows = [](int x, ShardTask y) { return x + std::get<2>(y)[0]; }; + const size_t kBlobInfoIndex = 2; + auto sum_num_rows = [](int x, ShardTask y) { return x + std::get(y)[0]; }; uint32_t nRows = std::accumulate(task_list_.begin(), task_list_.end(), 0, sum_num_rows); return nRows; } diff --git a/mindspore/dataset/core/validator_helpers.py b/mindspore/dataset/core/validator_helpers.py index b9efb148823..60436eadaf1 100644 --- a/mindspore/dataset/core/validator_helpers.py +++ b/mindspore/dataset/core/validator_helpers.py @@ -229,6 +229,13 @@ def check_uint32(value, arg_name=""): def check_pos_uint32(value, arg_name=""): + """ + Validates the value of a variable is within the range of positive uint32. + + :param value: the value of the variable + :param arg_name: name of the variable to be validated + :return: Exception: when the validation fails, nothing otherwise. + """ type_check(value, (int,), arg_name) check_value(value, [POS_INT_MIN, UINT32_MAX]) diff --git a/mindspore/dataset/transforms/py_transforms.py b/mindspore/dataset/transforms/py_transforms.py index bfc83b4a527..a804bcc7190 100644 --- a/mindspore/dataset/transforms/py_transforms.py +++ b/mindspore/dataset/transforms/py_transforms.py @@ -21,6 +21,7 @@ from .validators import check_one_hot_op, check_compose_list, check_random_apply from . import py_transforms_util as util from .c_transforms import TensorOperation + def not_random(function): """ Specify the function as "not random", i.e., it produces deterministic result. diff --git a/mindspore/dataset/transforms/py_transforms_util.py b/mindspore/dataset/transforms/py_transforms_util.py index 20230902e49..05d5b2a5f7a 100644 --- a/mindspore/dataset/transforms/py_transforms_util.py +++ b/mindspore/dataset/transforms/py_transforms_util.py @@ -148,7 +148,7 @@ class FuncWrapper: try: if hasattr(self.transform, "random") and not self.transform.random: self.random = False - except Exception: + except KeyError: self.random = True def __call__(self, *args): diff --git a/model_zoo/official/cv/mobilenetv2/scripts/run_train_nfs_cache.sh b/model_zoo/official/cv/mobilenetv2/scripts/run_train_nfs_cache.sh index 6c6b872bcd6..cc9ee9b98e8 100644 --- a/model_zoo/official/cv/mobilenetv2/scripts/run_train_nfs_cache.sh +++ b/model_zoo/official/cv/mobilenetv2/scripts/run_train_nfs_cache.sh @@ -15,6 +15,7 @@ # ============================================================================ CURPATH="$(dirname "$0")" +# shellcheck source=/dev/null . ${CURPATH}/cache_util.sh run_ascend() diff --git a/model_zoo/official/cv/resnet/scripts/run_distribute_train.sh b/model_zoo/official/cv/resnet/scripts/run_distribute_train.sh index 017c3cffcd3..f85fbbdb81f 100755 --- a/model_zoo/official/cv/resnet/scripts/run_distribute_train.sh +++ b/model_zoo/official/cv/resnet/scripts/run_distribute_train.sh @@ -15,6 +15,7 @@ # ============================================================================ CURPATH="$(dirname "$0")" +# shellcheck source=/dev/null . ${CURPATH}/cache_util.sh if [ $# != 3 ] && [ $# != 4 ] && [ $# != 5 ] diff --git a/model_zoo/official/cv/resnet/scripts/run_distribute_train_gpu.sh b/model_zoo/official/cv/resnet/scripts/run_distribute_train_gpu.sh index ebd382a6a57..b44116f9923 100755 --- a/model_zoo/official/cv/resnet/scripts/run_distribute_train_gpu.sh +++ b/model_zoo/official/cv/resnet/scripts/run_distribute_train_gpu.sh @@ -15,6 +15,7 @@ # ============================================================================ CURPATH="$(dirname "$0")" +# shellcheck source=/dev/null . ${CURPATH}/cache_util.sh if [ $# != 2 ] && [ $# != 3 ] && [ $# != 4 ] diff --git a/model_zoo/official/cv/resnet/scripts/run_standalone_train.sh b/model_zoo/official/cv/resnet/scripts/run_standalone_train.sh index 7e8e24f1575..402e01a6869 100755 --- a/model_zoo/official/cv/resnet/scripts/run_standalone_train.sh +++ b/model_zoo/official/cv/resnet/scripts/run_standalone_train.sh @@ -15,6 +15,7 @@ # ============================================================================ CURPATH="$(dirname "$0")" +# shellcheck source=/dev/null . ${CURPATH}/cache_util.sh if [ $# != 2 ] && [ $# != 3 ] && [ $# != 4 ] diff --git a/model_zoo/official/cv/resnet/scripts/run_standalone_train_gpu.sh b/model_zoo/official/cv/resnet/scripts/run_standalone_train_gpu.sh index 1a38f7ba459..edb85580acb 100755 --- a/model_zoo/official/cv/resnet/scripts/run_standalone_train_gpu.sh +++ b/model_zoo/official/cv/resnet/scripts/run_standalone_train_gpu.sh @@ -15,6 +15,7 @@ # ============================================================================ CURPATH="$(dirname "$0")" +# shellcheck source=/dev/null . ${CURPATH}/cache_util.sh if [ $# != 2 ] && [ $# != 3 ] && [ $# != 4 ]