diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc index 08f6bb93855..3effedafbc7 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/map_op/map_op.cc @@ -121,8 +121,6 @@ Status MapOp::GenerateWorkerJob(const std::unique_ptr *worker_job) for (size_t i = 0; i < tfuncs_.size(); i++) { // Currently we only have CPU as the device target // In the future, we will have heuristic or control from user to select target device - // MapTargetDevice target_device; - // RETURN_IF_NOT_OK(SelectTarget(tfuncs_[i], &target_device)); MapTargetDevice target_device = MapTargetDevice::kCpu; switch (target_device) { @@ -222,7 +220,6 @@ Status MapOp::operator()() { RETURN_IF_NOT_OK(child_[0]->GetNextBuffer(&buff, 0)); } // End() is commented out because it might never be called due to the lack of EOF when EpochCtrl is -1 - // RETURN_IF_NOT_OK(callback_manager_.End(CallbackParam(op_current_epochs_, ep_step, total_step))); // Handle eof logic, this code might never be reached if epoch_ctrl = -1. std::unique_ptr worker_job = std::make_unique(std::move(buff)); RETURN_IF_NOT_OK(local_queues_[num_buf++ % num_workers_]->Add(std::move(worker_job))); diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/sampler.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/sampler.cc index 4498455e4fe..e0af135a037 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/sampler.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/sampler/sampler.cc @@ -155,8 +155,6 @@ Status Sampler::AddChild(std::shared_ptr child) { child_.push_back(child); - // doesn't work, protected? - // child->AddParent(this); return Status::OK(); } diff --git a/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_data_server.cc b/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_data_server.cc index 2658ba7256d..d6fde79f16b 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_data_server.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_data_server.cc @@ -51,7 +51,6 @@ Status GraphDataServer::Init() { #else set_state(kGdsInitializing); RETURN_IF_NOT_OK(async_server_->Run()); - // RETURN_IF_NOT_OK(InitGraphDataImpl()); RETURN_IF_NOT_OK(tg_->CreateAsyncTask("init graph data impl", std::bind(&GraphDataServer::InitGraphDataImpl, this))); for (int32_t i = 0; i < num_workers_; ++i) { RETURN_IF_NOT_OK( diff --git a/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_data_service_impl.cc b/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_data_service_impl.cc index b926186870d..3fe4c96ce81 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_data_service_impl.cc +++ b/mindspore/ccsrc/minddata/dataset/engine/gnn/graph_data_service_impl.cc @@ -101,7 +101,6 @@ grpc::Status GraphDataServiceImpl::ClientUnRegister(grpc::ServerContext *context grpc::Status GraphDataServiceImpl::GetGraphData(grpc::ServerContext *context, const GnnGraphDataRequestPb *request, GnnGraphDataResponsePb *response) { - // MS_LOG(INFO) << "#### receive GetGraphData:" << request->op_name(); Status s; auto iter = g_get_graph_data_func_.find(request->op_name()); if (iter != g_get_graph_data_func_.end()) { @@ -115,7 +114,6 @@ grpc::Status GraphDataServiceImpl::GetGraphData(grpc::ServerContext *context, co } else { response->set_error_msg("Invalid op name."); } - // MS_LOG(INFO) << "#### end receive GetGraphData:" << request->op_name(); return ::grpc::Status::OK; } diff --git a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.cc b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.cc index f35d65bc397..cbd6517986e 100644 --- a/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.cc +++ b/mindspore/ccsrc/minddata/dataset/kernels/image/lite_cv/lite_mat.cc @@ -40,6 +40,7 @@ LiteMat::LiteMat(int width, LDataType data_type) { dims_ = 0; data_type_ = LDataType::UINT8; ref_count_ = 0; + size_ = 0; Init(width, data_type); } @@ -53,6 +54,7 @@ LiteMat::LiteMat(int width, int height, LDataType data_type) { dims_ = 0; data_type_ = LDataType::UINT8; ref_count_ = 0; + size_ = 0; Init(width, height, data_type); } @@ -66,6 +68,7 @@ LiteMat::LiteMat(int width, int height, int channel, LDataType data_type) { dims_ = 0; data_type_ = LDataType::UINT8; ref_count_ = 0; + size_ = 0; Init(width, height, channel, data_type); } @@ -87,6 +90,7 @@ LiteMat::LiteMat(const LiteMat &m) { dims_ = m.dims_; data_type_ = m.data_type_; ref_count_ = m.ref_count_; + size_ = 0; if (ref_count_) { addRef(ref_count_, 1); } @@ -111,6 +115,7 @@ LiteMat &LiteMat::operator=(const LiteMat &m) { dims_ = m.dims_; data_type_ = m.data_type_; ref_count_ = m.ref_count_; + size_ = 0; return *this; } @@ -180,6 +185,7 @@ void LiteMat::Release() { channel_ = 0; c_step_ = 0; ref_count_ = 0; + size_ = 0; } void *LiteMat::AlignMalloc(unsigned int size) { diff --git a/mindspore/ccsrc/minddata/mindrecord/include/shard_column.h b/mindspore/ccsrc/minddata/mindrecord/include/shard_column.h index f3bd43c5d81..b272d3ec7a4 100644 --- a/mindspore/ccsrc/minddata/mindrecord/include/shard_column.h +++ b/mindspore/ccsrc/minddata/mindrecord/include/shard_column.h @@ -48,7 +48,6 @@ enum ColumnDataType { ColumnNoDataType = 6 }; -// mapping as {"bytes", "string", "int32", "int64", "float32", "float64"}; const uint32_t ColumnDataTypeSize[kDataTypes] = {1, 1, 4, 8, 4, 8}; const std::vector ColumnDataTypeNameNormalized = {"uint8", "string", "int32", diff --git a/mindspore/ccsrc/minddata/mindrecord/include/shard_reader.h b/mindspore/ccsrc/minddata/mindrecord/include/shard_reader.h index 32bf1431ab7..86b70d547ae 100644 --- a/mindspore/ccsrc/minddata/mindrecord/include/shard_reader.h +++ b/mindspore/ccsrc/minddata/mindrecord/include/shard_reader.h @@ -199,9 +199,6 @@ class ShardReader { /// \return null void SetAllInIndex(bool all_in_index) { all_in_index_ = all_in_index; } - /// \brief get NLP flag - bool GetNlpFlag(); - /// \brief get all classes MSRStatus GetAllClasses(const std::string &category_field, std::set &categories); @@ -264,9 +261,6 @@ class ShardReader { MSRStatus CreateTasks(const std::vector> &row_group_summary, const std::vector> &operators); - /// \brief set NLP flag - void CheckNlp(); - /// \brief check if all specified columns are in index table void CheckIfColumnInIndex(const std::vector &columns); diff --git a/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc b/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc index e2d6247735f..c672bf6cb94 100644 --- a/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc +++ b/mindspore/ccsrc/minddata/mindrecord/io/shard_index_generator.cc @@ -257,7 +257,9 @@ std::pair ShardIndexGenerator::CreateDatabase(int shard_no sql += ",INC_" + std::to_string(field_no++) + " INT, " + ret.second + " " + type; } sql += ", PRIMARY KEY(ROW_ID"; - for (uint64_t i = 0; i < fields_.size(); ++i) sql += ",INC_" + std::to_string(i); + for (uint64_t i = 0; i < fields_.size(); ++i) { + sql += ",INC_" + std::to_string(i); + } sql += "));"; if (ExecuteSQL(sql, db, "create table successfully.") != SUCCESS) { return {FAILED, nullptr}; diff --git a/mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc b/mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc index cc0e9decbf3..7243e573a7a 100644 --- a/mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc +++ b/mindspore/ccsrc/minddata/mindrecord/io/shard_reader.cc @@ -35,17 +35,16 @@ Type StringToNum(const std::string &str) { return num; } -ShardReader::ShardReader() { - task_id_ = 0; - deliver_id_ = 0; - shard_count_ = 0; - n_consumer_ = 0; - page_size_ = 0; - header_size_ = 0; - num_rows_ = 0; - total_blob_size_ = 0; - num_padded_ = 0; -} +ShardReader::ShardReader() + : header_size_(0), + page_size_(0), + shard_count_(0), + n_consumer_(0), + num_padded_(0), + num_rows_(0), + total_blob_size_(0), + task_id_(0), + deliver_id_(0) {} std::pair> ShardReader::GetMeta(const std::string &file_path, json &meta_data) { if (!IsLegalFile(file_path)) { @@ -880,13 +879,7 @@ MSRStatus ShardReader::Open(const std::vector &file_paths, bool loa if (n_consumer < kMinConsumerCount) { n_consumer = kMinConsumerCount; } - vector blob_fields = GetBlobFields().second; - for (unsigned int i = 0; i < selected_columns.size(); ++i) { - if (!std::any_of(blob_fields.begin(), blob_fields.end(), - [&selected_columns, i](std::string item) { return selected_columns[i] == item; })) { - selected_columns_.push_back(selected_columns[i]); - } - } + selected_columns_ = selected_columns; if (CheckColumnList(selected_columns_) == FAILED) { diff --git a/mindspore/dataset/engine/cache_client.py b/mindspore/dataset/engine/cache_client.py index d0e3ce15042..3128432b06c 100644 --- a/mindspore/dataset/engine/cache_client.py +++ b/mindspore/dataset/engine/cache_client.py @@ -16,7 +16,6 @@ """ import copy -# from mindspore._c_dataengine import CacheClient from ..core.validator_helpers import type_check, check_uint32, check_uint64 @@ -39,7 +38,6 @@ class DatasetCache: self.port = port self.prefetch_size = prefetch_size # temporary disable cache feature in the current release - # self.cache_client = CacheClient(session_id, size, spilling, hostname, port, prefetch_size) self.cache_client = None def GetStat(self): diff --git a/mindspore/dataset/engine/graphdata.py b/mindspore/dataset/engine/graphdata.py index a3e0ea89d71..622160e210d 100644 --- a/mindspore/dataset/engine/graphdata.py +++ b/mindspore/dataset/engine/graphdata.py @@ -89,7 +89,6 @@ class GraphData: while self._graph_data.is_stoped() is not True: time.sleep(1) except KeyboardInterrupt: - # self._graph_data.stop() raise Exception("Graph data server receives KeyboardInterrupt") @check_gnn_get_all_nodes diff --git a/mindspore/dataset/engine/validators.py b/mindspore/dataset/engine/validators.py index 816eeeca4de..2082f5be2ca 100644 --- a/mindspore/dataset/engine/validators.py +++ b/mindspore/dataset/engine/validators.py @@ -1235,5 +1235,4 @@ def check_cache_option(cache): """Sanity check for cache parameter""" if cache is not None: # temporary disable cache feature in the current release - # type_check(cache, (cache_client.DatasetCache,), "cache") raise ValueError("Caching is disabled in the current release") diff --git a/tests/ut/python/dataset/test_datasets_csv.py b/tests/ut/python/dataset/test_datasets_csv.py index 9907fa0bf8b..17819532351 100644 --- a/tests/ut/python/dataset/test_datasets_csv.py +++ b/tests/ut/python/dataset/test_datasets_csv.py @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -import mindspore.dataset as ds import numpy as np import pytest +import mindspore.dataset as ds DATA_FILE = '../data/dataset/testCSV/1.csv' diff --git a/tests/ut/python/dataset/test_epoch_ctrl.py b/tests/ut/python/dataset/test_epoch_ctrl.py index 2b0a309b19c..092cba5b82f 100644 --- a/tests/ut/python/dataset/test_epoch_ctrl.py +++ b/tests/ut/python/dataset/test_epoch_ctrl.py @@ -16,9 +16,9 @@ Testing Epoch Control op in DE """ import itertools -import cv2 import numpy as np import pytest +import cv2 import mindspore.dataset as ds import mindspore.dataset.vision.c_transforms as vision diff --git a/tests/ut/python/dataset/test_save_op.py b/tests/ut/python/dataset/test_save_op.py index 0679fab56fb..c0484a2b63a 100644 --- a/tests/ut/python/dataset/test_save_op.py +++ b/tests/ut/python/dataset/test_save_op.py @@ -17,11 +17,11 @@ This is the test module for saveOp. """ import os from string import punctuation +import numpy as np +import pytest import mindspore.dataset as ds from mindspore import log as logger from mindspore.mindrecord import FileWriter -import numpy as np -import pytest CV_FILE_NAME1 = "../data/mindrecord/testMindDataSet/temp.mindrecord" CV_FILE_NAME2 = "../data/mindrecord/testMindDataSet/auto.mindrecord" diff --git a/tests/ut/python/dataset/test_tensor_string.py b/tests/ut/python/dataset/test_tensor_string.py index ba48097c4cb..d9a2d2f7aa4 100644 --- a/tests/ut/python/dataset/test_tensor_string.py +++ b/tests/ut/python/dataset/test_tensor_string.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== -import mindspore._c_dataengine as cde import numpy as np +import mindspore._c_dataengine as cde import mindspore.common.dtype as mstype import mindspore.dataset as ds diff --git a/tests/ut/python/dataset/util.py b/tests/ut/python/dataset/util.py index 3f20bfe2d5c..e40e6624b77 100644 --- a/tests/ut/python/dataset/util.py +++ b/tests/ut/python/dataset/util.py @@ -18,9 +18,9 @@ import json import os import itertools from enum import Enum +import numpy as np import matplotlib.pyplot as plt import matplotlib.patches as patches -import numpy as np # import jsbeautifier import mindspore.dataset as ds from mindspore import log as logger