p47068915
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

enhance: 1. fix api comment, 2. del dead code

This commit is contained in:
jonyguo 2020-06-29 15:47:06 +08:00
parent 6fb5538117
commit 41d16a3c77
50 changed files with 60 additions and 244 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mindspore/ccsrc/dataset/core/data_type.cc
View File

@ -18,7 +18,6 @@
#include "utils/log_adapter.h"
#include "dataset/core/pybind_support.h"
#include "dataset/util/de_error.h"
namespace mindspore {
namespace dataset {

6
mindspore/ccsrc/dataset/core/tensor.cc
View File

@ -152,7 +152,7 @@ Tensor::Tensor(const std::vector<std::string> &strings, const TensorShape &shape
this->data_end_ = data_ + offset_arr[i];
DS_ASSERT(num_bytes == 0);
MS_ASSERT(num_bytes == 0);
if (shape.known()) Tensor::Reshape(shape);
}
Tensor::Tensor(const dataengine::BytesList &bytes_list, const TensorShape &shape)
@ -191,7 +191,7 @@ Tensor::Tensor(const dataengine::BytesList &bytes_list, const TensorShape &shape
data_end_ = data_ + offset_arr[i];
DS_ASSERT(num_bytes == 0);
MS_ASSERT(num_bytes == 0);
if (shape.known()) Tensor::Reshape(shape);
}
Status Tensor::CreateTensor(std::shared_ptr<Tensor> *ptr, TensorImpl tensor_impl, const TensorShape &shape,
@ -420,7 +420,7 @@ bool Tensor::operator==(const Tensor &rhs) const {
// Description: A function that print the value as specified by its index
void Tensor::PrintItemAt(const std::vector<dsize_t> &index, std::ostream &out) const {
Status rc;
DS_ASSERT(data_);
MS_ASSERT(data_);
switch (type_.value()) {
CASE_PRINT_HEX(DataType::DE_BOOL, bool);

1
mindspore/ccsrc/dataset/core/tensor.h
View File

@ -33,7 +33,6 @@
#include "dataset/core/data_type.h"
#include "dataset/core/tensor_shape.h"
#include "dataset/util/allocator.h"
#include "dataset/util/de_error.h"
#include "dataset/util/status.h"
#include "proto/example.pb.h"

1
mindspore/ccsrc/dataset/core/tensor_shape.cc
View File

@ -22,7 +22,6 @@
#include "common/utils.h"
#include "utils/log_adapter.h"
#include "dataset/core/constants.h"
#include "dataset/util/de_error.h"
namespace mindspore {
namespace dataset {

10
mindspore/ccsrc/dataset/engine/connector.h
View File

@ -97,7 +97,7 @@ class Connector {
virtual Status Pop(int32_t worker_id, // The worker-id of the caller. See the requirement at the top of this file.
T *result) noexcept {
{
DS_ASSERT(worker_id < num_consumers_);
MS_ASSERT(worker_id < num_consumers_);
std::unique_lock<std::mutex> lk(m_);
RETURN_IF_NOT_OK(cv_.Wait(&lk, [this, worker_id]() { return expect_consumer_ == worker_id; }));
RETURN_IF_NOT_OK(queues_[pop_from_]->PopFront(result));
@ -114,8 +114,8 @@ class Connector {
// @param worker_id The id of a worker thread calling this method.
// @param el A const lvalue element to be passed/added/pushed.
Status Push(int32_t worker_id, const T &el) noexcept {
DS_ASSERT(worker_id < static_cast<int32_t>(queues_.size()));
DS_ASSERT(queues_[worker_id] != nullptr);
MS_ASSERT(worker_id < static_cast<int32_t>(queues_.size()));
MS_ASSERT(queues_[worker_id] != nullptr);
return (queues_[worker_id]->Add(el));
}
@ -125,8 +125,8 @@ class Connector {
// @param worker_id The id of a worker thread calling this method.
// @param el An element to be passed/added/pushed.
virtual Status Push(int32_t worker_id, T &&el) noexcept {
DS_ASSERT(worker_id < static_cast<int32_t>(queues_.size()));
DS_ASSERT(queues_[worker_id] != nullptr);
MS_ASSERT(worker_id < static_cast<int32_t>(queues_.size()));
MS_ASSERT(queues_[worker_id] != nullptr);
return (queues_[worker_id]->Add(std::forward<T>(el)));
}

3
mindspore/ccsrc/dataset/engine/data_schema.cc
View File

@ -27,7 +27,6 @@
#include "dataset/util/status.h"
#include "dataset/core/tensor_shape.h"
#include "utils/log_adapter.h"
#include "dataset/util/de_error.h"
namespace mindspore {
namespace dataset {
@ -425,7 +424,7 @@ DataSchema::~DataSchema() = default;
// Getter for the ColDescriptor by index
const ColDescriptor &DataSchema::column(int32_t idx) const {
DS_ASSERT(idx < static_cast<int>(col_descs_.size()));
MS_ASSERT(idx < static_cast<int>(col_descs_.size()));
return col_descs_[idx];
}

2
mindspore/ccsrc/dataset/engine/datasetops/dataset_op.cc
View File

@ -110,7 +110,7 @@ void DatasetOp::RemoveParent(const DatasetOp *parent) {
// Getter function to get a shared pointer to our childAdds a operator to become our child.
std::shared_ptr<DatasetOp> DatasetOp::child(int32_t child_index) const {
DS_ASSERT(child_index < static_cast<int>(child_.size()));
MS_ASSERT(child_index < static_cast<int>(child_.size()));
// Return a shared pointer
return child_[child_index];
}

2
mindspore/ccsrc/dataset/engine/datasetops/source/image_folder_op.cc
View File

@ -104,7 +104,7 @@ Status ImageFolderOp::PrescanMasterEntry(const std::string &filedir) {
// following loop puts the 2 level of shuffles together into 1 vector
for (size_t ind = 0; ind < v.size(); ++ind) {
while (v[ind]->second.empty() == false) {
DS_ASSERT(!(v[ind]->first.empty())); // make sure that v[ind]->first.substr(1) is not out of bound
MS_ASSERT(!(v[ind]->first.empty())); // make sure that v[ind]->first.substr(1) is not out of bound
v[ind]->second.front()->second = class_index_.empty() ? ind : class_index_[v[ind]->first.substr(1)];
image_label_pairs_.push_back(v[ind]->second.front());
v[ind]->second.pop();

2
mindspore/ccsrc/dataset/engine/jagged_connector.h
View File

@ -44,7 +44,7 @@ class JaggedConnector : public Connector<std::unique_ptr<DataBuffer>> {
Status Pop(int32_t worker_id, std::unique_ptr<DataBuffer> *result) noexcept override {
{
DS_ASSERT(worker_id < num_consumers_);
MS_ASSERT(worker_id < num_consumers_);
std::unique_lock<std::mutex> lock(m_);
RETURN_IF_NOT_OK(cv_.Wait(&lock, [this, worker_id]() { return expect_consumer_ == worker_id; }));
if (is_queue_finished_[pop_from_]) {

13
mindspore/ccsrc/dataset/util/arena.cc
View File

@ -17,7 +17,6 @@
#include <unistd.h>
#include <utility>
#include "dataset/util/system_pool.h"
#include "dataset/util/de_error.h"
#include "./securec.h"
#include "utils/log_adapter.h"
@ -79,7 +78,7 @@ void Arena::Deallocate(void *p) {
auto *q = get_base_addr(p);
MemHdr hdr(0, 0);
MemHdr::getHdr(q, &hdr);
DS_ASSERT(hdr.sig == 0xDEADBEEF);
MS_ASSERT(hdr.sig == 0xDEADBEEF);
// We are going to insert a free block back to the treap. But first, check if we can combine
// with the free blocks before and after to form a bigger block.
std::unique_lock<std::mutex> lck(mux_);
@ -103,8 +102,8 @@ void Arena::Deallocate(void *p) {
}
Status Arena::Reallocate(void **pp, size_t old_sz, size_t new_sz) {
DS_ASSERT(pp);
DS_ASSERT(*pp);
MS_ASSERT(pp);
MS_ASSERT(*pp);
uint64_t actual_size = static_cast<uint64_t>(new_sz) + ARENA_WALL_OVERHEAD_SZ;
if (actual_size > this->get_max_size()) {
RETURN_STATUS_UNEXPECTED("Request size too big : " + std::to_string(new_sz));
@ -114,7 +113,7 @@ Status Arena::Reallocate(void **pp, size_t old_sz, size_t new_sz) {
auto *oldHdr = get_base_addr(oldAddr);
MemHdr hdr(0, 0);
MemHdr::getHdr(oldHdr, &hdr);
DS_ASSERT(hdr.sig == 0xDEADBEEF);
MS_ASSERT(hdr.sig == 0xDEADBEEF);
std::unique_lock<std::mutex> lck(mux_);
if (hdr.blk_size > req_blk) {
// Refresh the header with the new smaller size.
@ -239,8 +238,8 @@ bool Arena::BlockEnlarge(uint64_t *addr, uint64_t old_sz, uint64_t new_sz) {
}
Status Arena::FreeAndAlloc(void **pp, size_t old_sz, size_t new_sz) {
DS_ASSERT(pp);
DS_ASSERT(*pp);
MS_ASSERT(pp);
MS_ASSERT(*pp);
void *p = nullptr;
void *q = *pp;
RETURN_IF_NOT_OK(Allocate(new_sz, &p));

8
mindspore/ccsrc/dataset/util/btree_impl.tpp
View File

@ -61,8 +61,8 @@ typename BPlusTree<K, V, A, C, T>::IndexRc BPlusTree<K, V, A, C, T>::InnerNode::
template <typename K, typename V, typename A, typename C, typename T>
typename BPlusTree<K, V, A, C, T>::IndexRc BPlusTree<K, V, A, C, T>::InnerNode::Split(
BPlusTree<K, V, A, C, T>::InnerNode *to, key_type *split_key) {
DS_ASSERT(to);
DS_ASSERT(to->slotuse_ == 0);
MS_ASSERT(to);
MS_ASSERT(to->slotuse_ == 0);
// It is simpler to sort first, then split. Other alternative is to move key by key to the
// new node. Also we need to deal with the 'holes' after a key is moved.
RETURN_IF_BAD_RC(this->Sort());
@ -153,8 +153,8 @@ typename BPlusTree<K, V, A, C, T>::IndexRc BPlusTree<K, V, A, C, T>::LeafNode::S
template <typename K, typename V, typename A, typename C, typename T>
typename BPlusTree<K, V, A, C, T>::IndexRc BPlusTree<K, V, A, C, T>::LeafNode::Split(
BPlusTree<K, V, A, C, T>::LeafNode *to) {
DS_ASSERT(to);
DS_ASSERT(to->slotuse_ == 0);
MS_ASSERT(to);
MS_ASSERT(to->slotuse_ == 0);
// It is simpler to sort first, then split. Other alternative is to move key by key to the
// new node. Also we need to deal with the 'holes' after a key is moved.
RETURN_IF_BAD_RC(this->Sort());

1
mindspore/ccsrc/dataset/util/btree_iterator.tpp
View File

@ -15,7 +15,6 @@
#ifndef DATASET_UTIL_BTREE_ITERATOR_H_
#define DATASET_UTIL_BTREE_ITERATOR_H_
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
#include "btree.h"

18
mindspore/ccsrc/dataset/util/buddy.cc
View File

@ -16,9 +16,9 @@
#include "dataset/util/buddy.h"
#include <iomanip>
#include <stdexcept>
#include "dataset/util/de_error.h"
#include "dataset/util/memory_pool.h"
#include "dataset/util/system_pool.h"
#include "utils/log_adapter.h"
#include "./securec.h"
inline uint64_t BitLeftShift(uint64_t v, uint64_t n) { return (v << n); }
@ -68,7 +68,7 @@ Status BuddySpace::Alloc(const uint64_t sz, BSpaceDescriptor *desc, addr_t *p) n
}
addr_t BuddySpace::AllocNoLock(const uint64_t sz, BSpaceDescriptor *desc) noexcept {
DS_ASSERT(sz <= max_);
MS_ASSERT(sz <= max_);
uint32_t reqSize = SizeToBlock(sz);
rel_addr_t rel_addr = AllocBuddySeg(reqSize);
if (rel_addr != static_cast<rel_addr_t>(NOSPACE)) {
@ -84,7 +84,7 @@ addr_t BuddySpace::AllocNoLock(const uint64_t sz, BSpaceDescriptor *desc) noexce
}
void BuddySpace::FreeNoLock(const BSpaceDescriptor *desc) {
DS_ASSERT(desc->sig == 0XDEADBEEF);
MS_ASSERT(desc->sig == 0XDEADBEEF);
rel_addr_t rel_addr = desc->addr;
size_t blk_size = desc->blk_size;
size_t req_size = desc->req_size;
@ -217,7 +217,7 @@ void BuddySpace::JoinBuddySeg(rel_addr_t addr, size_t blk_sz) {
auto log_sz = static_cast<log_t>(Log2(blk_sz));
rel_addr_t left = (buddy < addr) ? buddy : addr;
rel_addr_t right = left + blk_sz;
DS_ASSERT(count_[log_sz] >= 2);
MS_ASSERT(count_[log_sz] >= 2);
count_[log_sz] -= 2;
SetBuddySegState(right, blk_sz, STATE::kEmpty);
SetBuddySegState(left, BitLeftShift(blk_sz, 1), STATE::kFree);
@ -235,7 +235,7 @@ void BuddySpace::JoinBuddySeg(rel_addr_t addr, size_t blk_sz) {
}
void BuddySpace::TrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) {
DS_ASSERT(ask_sz < blk_sz);
MS_ASSERT(ask_sz < blk_sz);
uint32_t inx = Log2(blk_sz);
size_t remaining_sz = ask_sz;
for (int i = inx; i > 0; i--) {
@ -256,7 +256,7 @@ void BuddySpace::TrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) {
}
void BuddySpace::UnTrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) {
DS_ASSERT(ask_sz < blk_sz);
MS_ASSERT(ask_sz < blk_sz);
uint32_t inx = Log2(blk_sz);
size_t remaining_sz = ask_sz;
for (int i = inx; i > 0; i--) {
@ -268,7 +268,7 @@ void BuddySpace::UnTrimBuddySeg(rel_addr_t addr, size_t blk_sz, size_t ask_sz) {
size_t sz = 0;
STATE st;
GetBuddySegState(addr, &sz, &st);
DS_ASSERT(sz == half_sz && st == STATE::kAlloc);
MS_ASSERT(sz == half_sz && st == STATE::kAlloc);
}
#endif
SetBuddySegState(addr, half_sz, STATE::kFree);
@ -291,7 +291,7 @@ rel_addr_t BuddySpace::AllocBuddySeg(uint32_t req_size) noexcept {
STATE st;
size_t sz = 0;
for (int i = start_inx; !found && i < num_lvl_; i++) {
DS_ASSERT(count_[i] >= 0);
MS_ASSERT(count_[i] >= 0);
if (count_[i] == 0) {
continue;
}
@ -302,7 +302,7 @@ rel_addr_t BuddySpace::AllocBuddySeg(uint32_t req_size) noexcept {
if (st == STATE::kFree && sz == blk_sz) {
found = true;
} else {
DS_ASSERT(st != STATE::kEmpty);
MS_ASSERT(st != STATE::kEmpty);
ask_addr += ((sz > blk_sz) ? sz : blk_sz);
}
}

6
mindspore/ccsrc/dataset/util/circular_pool.cc
View File

@ -19,8 +19,8 @@
#include <limits>
#include <utility>
#include "./securec.h"
#include "dataset/util/de_error.h"
#include "dataset/util/system_pool.h"
#include "utils/log_adapter.h"
namespace mindspore {
namespace dataset {
@ -65,7 +65,7 @@ void CircularPool::CircularIterator::Reset() {
auto list_end = dp_->mem_segments_.end();
auto it = std::find_if(dp_->mem_segments_.begin(), list_end,
[this](const std::shared_ptr<Arena> &b) { return b.get() == cur_tail_; });
DS_ASSERT(it != list_end);
MS_ASSERT(it != list_end);
start_ = std::distance(dp_->mem_segments_.begin(), it);
cur_ = start_;
has_next_ = true;
@ -153,7 +153,7 @@ Status CircularPool::Reallocate(void **pp, size_t old_sz, size_t new_sz) {
return (q > base && q < base + b->get_max_size());
});
lock.Unlock();
DS_ASSERT(it != mem_segments_.end());
MS_ASSERT(it != mem_segments_.end());
Arena *ba = it->get();
Status rc = ba->Reallocate(pp, old_sz, new_sz);
if (rc.IsOutofMemory()) {

39
mindspore/ccsrc/dataset/util/de_error.h
View File

@ -1,39 +0,0 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef DATASET_UTIL_DE_ERROR_H_
#define DATASET_UTIL_DE_ERROR_H_
#ifdef DEBUG
#include <cassert>
#define DS_ASSERT(f) assert(f)
#else
#define DS_ASSERT(f) ((void)0)
#endif
#include <map>
#include "utils/error_code.h"
namespace mindspore {
namespace dataset {
DE_ERRORNO_DATASET(CATCH_EXCEPTION, 0, "try catch exception error");
DE_ERRORNO_DATASET(FILE_NOT_FOUND, 1, "file is not found");
DE_ERRORNO_DATASET(PARSE_FAILED, 2, "parse failed");
DE_ERRORNO_DATASET(COPY_ERROR, 3, "copy data error");
DE_ERRORNO_DATASET(BOUND_ERROR, 4, "variable overflow or lost of precision");
DE_ERRORNO_DATASET(ALLOC_FAILED, 5, "dynamic memory allocation failed");
} // namespace dataset
} // namespace mindspore
#endif // DATASET_UTIL_DE_ERROR_H_

7
mindspore/ccsrc/dataset/util/list.h
View File

@ -18,7 +18,8 @@
#include <iostream>
#include <iterator>
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
namespace mindspore {
namespace dataset {
@ -90,7 +91,7 @@ struct List {
// Insert elem2 after elem1 in the list.
virtual void InsertAfter(pointer elem1, pointer elem2) {
DS_ASSERT(elem1 != elem2);
MS_ASSERT(elem1 != elem2);
Node<T> &elem1_node = elem1->*node;
Node<T> &elem2_node = elem2->*node;
elem2_node.prev = elem1;
@ -108,7 +109,7 @@ struct List {
// Insert elem2 before elem1 in the list.
virtual void InsertBefore(pointer elem1, pointer elem2) {
DS_ASSERT(elem1 != elem2);
MS_ASSERT(elem1 != elem2);
Node<T> &elem1_node = elem1->*node;
Node<T> &elem2_node = elem2->*node;
elem2_node.next = elem1;

22
mindspore/ccsrc/dataset/util/lock.cc
View File

@ -14,7 +14,7 @@
* limitations under the License.
*/
#include "dataset/util/lock.h"
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
namespace mindspore {
namespace dataset {
@ -63,7 +63,7 @@ void RWLock::Unlock() noexcept {
void RWLock::Upgrade() {
std::unique_lock<std::mutex> lck(mtx_);
DS_ASSERT(status_);
MS_ASSERT(status_);
if (status_ == -1) {
// I am a writer already.
return;
@ -81,7 +81,7 @@ void RWLock::Upgrade() {
void RWLock::Downgrade() {
std::unique_lock<std::mutex> lck(mtx_);
DS_ASSERT(status_);
MS_ASSERT(status_);
if (status_ == -1) {
// If there are no other writers waiting, just change the status
if (waiting_writers_ == 0) {
@ -111,24 +111,24 @@ SharedLock::~SharedLock() {
}
void SharedLock::Unlock() {
DS_ASSERT(ownlock_ == true);
MS_ASSERT(ownlock_ == true);
rw_->Unlock();
ownlock_ = false;
}
void SharedLock::Lock() {
DS_ASSERT(ownlock_ == false);
MS_ASSERT(ownlock_ == false);
rw_->LockShared();
ownlock_ = true;
}
void SharedLock::Upgrade() {
DS_ASSERT(ownlock_ == true);
MS_ASSERT(ownlock_ == true);
rw_->Upgrade();
}
void SharedLock::Downgrade() {
DS_ASSERT(ownlock_ == true);
MS_ASSERT(ownlock_ == true);
rw_->Downgrade();
}
@ -146,13 +146,13 @@ UniqueLock::~UniqueLock() {
}
void UniqueLock::Unlock() {
DS_ASSERT(ownlock_ == true);
MS_ASSERT(ownlock_ == true);
rw_->Unlock();
ownlock_ = false;
}
void UniqueLock::Lock() {
DS_ASSERT(ownlock_ == false);
MS_ASSERT(ownlock_ == false);
rw_->LockExclusive();
ownlock_ = true;
}
@ -171,13 +171,13 @@ LockGuard::~LockGuard() {
}
void LockGuard::Unlock() {
DS_ASSERT(own_lock_);
MS_ASSERT(own_lock_);
lck_->Unlock();
own_lock_ = false;
}
void LockGuard::Lock() {
DS_ASSERT(own_lock_ == false);
MS_ASSERT(own_lock_ == false);
lck_->Lock();
own_lock_ = true;
}

1
mindspore/ccsrc/dataset/util/path.cc
View File

@ -23,7 +23,6 @@
#include <utility>
#include "common/utils.h"
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
namespace mindspore {

1
mindspore/ccsrc/dataset/util/queue.h
View File

@ -25,7 +25,6 @@
#include <vector>
#include "common/utils.h"
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
#include "dataset/util/allocator.h"
#include "dataset/util/services.h"

5
mindspore/ccsrc/dataset/util/storage_container.cc
View File

@ -20,7 +20,6 @@
#include <unistd.h>
#include <vector>
#include "common/utils.h"
#include "dataset/util/de_error.h"
#include "dataset/util/path.h"
#include "dataset/util/status.h"
#include "utils/log_adapter.h"
@ -59,7 +58,7 @@ Status StorageContainer::Close() noexcept {
}
Status StorageContainer::Read(WritableSlice *dest, off64_t offset) const noexcept {
DS_ASSERT(is_open_);
MS_ASSERT(is_open_);
RETURN_UNEXPECTED_IF_NULL(dest);
auto sz = dest->GetSize();
#if defined(_WIN32) || defined(_WIN64)
@ -83,7 +82,7 @@ Status StorageContainer::Read(WritableSlice *dest, off64_t offset) const noexcep
}
Status StorageContainer::Write(const ReadableSlice &dest, off64_t offset) const noexcept {
DS_ASSERT(is_open_);
MS_ASSERT(is_open_);
auto sz = dest.GetSize();
#if defined(_WIN32) || defined(_WIN64)
// Doesn't seem there is any pwrite64 on mingw.

1
mindspore/ccsrc/dataset/util/storage_manager.cc
View File

@ -22,7 +22,6 @@
#include "common/utils.h"
#include "dataset/util/path.h"
#include "dataset/util/services.h"
#include "dataset/util//de_error.h"
#include "utils/log_adapter.h"
namespace mindspore {

1
mindspore/ccsrc/dataset/util/task.cc
View File

@ -16,7 +16,6 @@
#include "dataset/util/task.h"
#include "common/utils.h"
#include "dataset/util/task_manager.h"
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
namespace mindspore {

1
mindspore/ccsrc/dataset/util/task.h
View File

@ -27,7 +27,6 @@
#include <stdexcept>
#include <string>
#include <thread>
#include "dataset/util/de_error.h"
#include "dataset/util/intrp_resource.h"
#include "dataset/util/list.h"
#include "dataset/util/memory_pool.h"

114
mindspore/ccsrc/utils/error_code.h
View File

@ -1,114 +0,0 @@
/**
* Copyright 2019 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_
#define MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_
#include <map>
#include <string>
namespace DataEngineBase {
// system ID
const int SYSID_MD = 20;
// Runtime location
enum LogRuntime {
RT_HOST = 0b01,
RT_DEVICE = 0b10,
};
// sub model
enum SubModuleId {
COMMON_MODULE = 0,
DATAGET_MODULE,
MINDRECORD_MODULE,
};
// error code type
enum ErrorCodeType {
ERROR_CODE = 0b01,
EXCEPTION_CODE = 0b10,
};
// error level
enum ErrorLevel {
COMMON_LEVEL = 0b000,
SUGGESTION_LEVEL = 0b001,
MINOR_LEVEL = 0b010,
MAJOR_LEVEL = 0b011,
CRITICAL_LEVEL = 0b100,
};
// code compose(4 byte), runtime: 2 bit, type: 2 bit, level: 3 bit, sysid: 8 bit, modid: 5 bit, value: 12 bit
#define DE_ERRORNO(runtime, type, level, sysid, submodid, name, value, desc) \
constexpr DataEngineBase::Status name = ((0xFF & ((uint8_t)runtime)) << 30) | ((0xFF & ((uint8_t)type)) << 28) | \
((0xFF & ((uint8_t)level)) << 25) | ((0xFF & ((uint8_t)sysid)) << 17) | \
((0xFF & ((uint8_t)submodid)) << 12) | (0x0FFF & ((uint16_t)value)); \
const DataEngineBase::ErrorNoRegisterar g_##name##_errorno(name, desc);
// each module defines error codes using the following macros
#define DE_ERRORNO_COMMON(name, value, desc) \
DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \
DataEngineBase::SYSID_MD, DataEngineBase::COMMON_MODULE, name, value, desc)
#define DE_ERRORNO_DATASET(name, value, desc) \
DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \
DataEngineBase::SYSID_MD, DataEngineBase::DATAGET_MODULE, name, value, desc)
#define DE_ERRORNO_MINDRECORD(name, value, desc) \
DE_ERRORNO(DataEngineBase::RT_HOST, DataEngineBase::ERROR_CODE, DataEngineBase::COMMON_LEVEL, \
DataEngineBase::SYSID_MD, DataEngineBase::MINDRECORD_MODULE, name, value, desc)
// get error code description
#define DE_GET_ERRORNO_STR(value) DataEngineBase::StatusFactory::Instance()->GetErrDesc(value)
class StatusFactory {
public:
static StatusFactory *Instance() {
static StatusFactory instance;
return &instance;
}
void RegisterErrorNo(uint32_t err, const std::string &desc) {
if (err_desc_.find(err) != err_desc_.end()) return;
err_desc_[err] = desc;
}
std::string GetErrDesc(uint32_t err) {
auto iter_find = err_desc_.find(err);
if (iter_find == err_desc_.end()) return "";
return iter_find->second;
}
protected:
StatusFactory() = default;
~StatusFactory() = default;
private:
std::map<uint32_t, std::string> err_desc_;
};
class ErrorNoRegisterar {
public:
ErrorNoRegisterar(uint32_t err, const std::string &desc) { StatusFactory::Instance()->RegisterErrorNo(err, desc); }
~ErrorNoRegisterar() = default;
};
using Status = uint32_t;
} // namespace DataEngineBase
#endif // MINDSPORE_CCSRC_UTILS_LOG_DE_ERROR_CODE_H_

8
mindspore/mindrecord/filewriter.py
View File

@ -240,7 +240,9 @@ class FileWriter:
def set_header_size(self, header_size):
"""
Set the size of header.
Set the size of header which contains shard information, schema information, \
page meta information, etc. The larger the header, the more training data \
a single mindrecord file can store.
Args:
header_size (int): Size of header, between 16KB and 128MB.
@ -256,7 +258,9 @@ class FileWriter:
def set_page_size(self, page_size):
"""
Set the size of Page.
Set the size of page which mainly refers to the block to store training data, \
and the training data will be split into raw page and blob page in mindrecord. \
The larger the page, the more training data a single page can store.
Args:
page_size (int): Size of page, between 32KB and 256MB.

1
tests/ut/cpp/dataset/arena_test.cc
View File

@ -17,7 +17,6 @@
#include <string>
#include "dataset/util/arena.h"
#include "common/common.h"
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
using namespace mindspore::dataset;

1
tests/ut/cpp/dataset/batch_op_test.cc
View File

@ -21,7 +21,6 @@
#include "common/utils.h"
#include "gtest/gtest.h"
#include "dataset/core/global_context.h"
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
#include "securec.h"
#include "dataset/util/status.h"

1
tests/ut/cpp/dataset/btree_test.cc
View File

@ -21,7 +21,6 @@
#include "dataset/util/task_manager.h"
#include "common/common.h"
#include "gtest/gtest.h"
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
using namespace mindspore::dataset;

1
tests/ut/cpp/dataset/celeba_op_test.cc
View File

@ -23,7 +23,6 @@
#include "dataset/core/global_context.h"
#include "dataset/engine/datasetops/source/celeba_op.h"
#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
#include "dataset/util/de_error.h"
#include "dataset/util/status.h"
#include "gtest/gtest.h"
#include "utils/log_adapter.h"

1
tests/ut/cpp/dataset/cifar_op_test.cc
View File

@ -26,7 +26,6 @@
#include "dataset/engine/datasetops/source/sampler/sampler.h"
#include "dataset/engine/datasetops/source/sampler/random_sampler.h"
#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
#include "dataset/util/de_error.h"
#include "dataset/util/path.h"
#include "dataset/util/status.h"
#include "gtest/gtest.h"

1
tests/ut/cpp/dataset/coco_op_test.cc
View File

@ -30,7 +30,6 @@
#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
#include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h"
#include "dataset/util/de_error.h"
#include "dataset/util/path.h"
#include "dataset/util/status.h"
#include "gtest/gtest.h"

1
tests/ut/cpp/dataset/common/common.h
View File

@ -16,7 +16,6 @@
#ifndef TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_
#define TESTS_DATASET_UT_CORE_COMMON_DE_UT_COMMON_H_
#include "dataset/util/de_error.h"
#include "gtest/gtest.h"
#include "utils/log_adapter.h"

6
tests/ut/cpp/dataset/connector_test.cc
View File

@ -140,7 +140,7 @@ Status MindDataTestConnector::Run_test_0() {
auto my_conn = std::make_shared<Connector<uint32_t>>(1, // num of producers
1, // num of consumers
10); // capacity of each queue
DS_ASSERT(my_conn != nullptr);
MS_ASSERT(my_conn != nullptr);
rc = my_conn->Register(tg_.get());
RETURN_IF_NOT_OK(rc);
@ -277,7 +277,7 @@ Status MindDataTestConnector::FirstWorkerPush(
int start_in,
int offset) {
TaskManager::FindMe()->Post();
DS_ASSERT(my_conn != nullptr);
MS_ASSERT(my_conn != nullptr);
Status rc;
for (int i = start_in; i < input_.size(); i += offset) {
rc = my_conn->Push(tid, input_[i]);
@ -294,7 +294,7 @@ Status MindDataTestConnector::MidWorkerJob(
int tid,
std::shared_ptr<Connector<uint32_t> > from_conn,
std::shared_ptr<Connector<uint32_t> > to_conn) {
DS_ASSERT((from_conn != nullptr) && (to_conn != nullptr));
MS_ASSERT((from_conn != nullptr) && (to_conn != nullptr));
Status rc;
TaskManager::FindMe()->Post();
while (1) {

1
tests/ut/cpp/dataset/duplicate_op_test.cc
View File

@ -17,7 +17,6 @@
#include "common/common.h"
#include "gtest/gtest.h"
#include "dataset/core/tensor.h"
#include "dataset/util/de_error.h"
#include "dataset/kernels/data/duplicate_op.h"
using namespace mindspore::dataset;

1
tests/ut/cpp/dataset/execution_tree_test.cc
View File

@ -21,7 +21,6 @@
#include "dataset/engine/datasetops/source/tf_reader_op.h"
#include "common/common.h"
#include "gtest/gtest.h"
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
using namespace mindspore::dataset;

1
tests/ut/cpp/dataset/global_context_test.cc
View File

@ -15,7 +15,6 @@
*/
#include "dataset/core/global_context.h"
#include "common/common.h"
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
using namespace mindspore::dataset;

1
tests/ut/cpp/dataset/image_folder_op_test.cc
View File

@ -29,7 +29,6 @@
#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
#include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h"
#include "dataset/util/de_error.h"
#include "dataset/util/path.h"
#include "dataset/util/status.h"
#include "gtest/gtest.h"

1
tests/ut/cpp/dataset/interrupt_test.cc
View File

@ -14,7 +14,6 @@
* limitations under the License.
*/
#include "common/common.h"
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
#include "dataset/util/services.h"
#include "dataset/util/intrp_service.h"

1
tests/ut/cpp/dataset/manifest_op_test.cc
View File

@ -25,7 +25,6 @@
#include "dataset/engine/datasetops/source/manifest_op.h"
#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
#include "dataset/util/de_error.h"
#include "dataset/util/status.h"
#include "gtest/gtest.h"
#include "utils/log_adapter.h"

1
tests/ut/cpp/dataset/mask_test.cc
View File

@ -22,7 +22,6 @@
#include "dataset/core/tensor.h"
#include "dataset/core/cv_tensor.h"
#include "dataset/core/data_type.h"
#include "dataset/util/de_error.h"
#include "dataset/kernels/data/mask_op.h"
#include "dataset/kernels/data/data_utils.h"

1
tests/ut/cpp/dataset/mnist_op_test.cc
View File

@ -30,7 +30,6 @@
#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
#include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h"
#include "dataset/util/de_error.h"
#include "dataset/util/path.h"
#include "dataset/util/status.h"
#include "gtest/gtest.h"

1
tests/ut/cpp/dataset/path_test.cc
View File

@ -16,7 +16,6 @@
#include "dataset/util/path.h"
#include "common/common.h"
#include "gtest/gtest.h"
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
#include <cstdio>

1
tests/ut/cpp/dataset/stand_alone_samplers_test.cc
View File

@ -21,7 +21,6 @@
#include "dataset/engine/datasetops/source/sampler/random_sampler.h"
#include "dataset/engine/datasetops/source/sampler/sampler.h"
#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "dataset/util/de_error.h"
#include "dataset/util/status.h"
#include "gtest/gtest.h"
#include "utils/log_adapter.h"

1
tests/ut/cpp/dataset/status_test.cc
View File

@ -16,7 +16,6 @@
#include "dataset/util/status.h"
#include "common/common.h"
#include "gtest/gtest.h"
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
using namespace mindspore::dataset;

1
tests/ut/cpp/dataset/tensor_string_test.cc
View File

@ -22,7 +22,6 @@
#include "dataset/core/tensor.h"
#include "dataset/core/cv_tensor.h"
#include "dataset/core/data_type.h"
#include "dataset/util/de_error.h"
using namespace mindspore::dataset;

1
tests/ut/cpp/dataset/tensor_test.cc
View File

@ -22,7 +22,6 @@
#include "dataset/core/tensor.h"
#include "dataset/core/cv_tensor.h"
#include "dataset/core/data_type.h"
#include "dataset/util/de_error.h"
using namespace mindspore::dataset;

1
tests/ut/cpp/dataset/treap_test.cc
View File

@ -16,7 +16,6 @@
#include "dataset/util/treap.h"
#include "common/common.h"
#include "gtest/gtest.h"
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
using namespace mindspore::dataset;

1
tests/ut/cpp/dataset/type_cast_op_test.cc
View File

@ -25,7 +25,6 @@
#include "dataset/core/pybind_support.h"
#include "gtest/gtest.h"
#include "securec.h"
#include "dataset/util/de_error.h"
#define MAX_INT_PRECISION 16777216 // float int precision is 16777216
using namespace mindspore::dataset;

1
tests/ut/cpp/dataset/voc_op_test.cc
View File

@ -30,7 +30,6 @@
#include "dataset/engine/datasetops/source/sampler/sequential_sampler.h"
#include "dataset/engine/datasetops/source/sampler/subset_random_sampler.h"
#include "dataset/engine/datasetops/source/sampler/weighted_random_sampler.h"
#include "dataset/util/de_error.h"
#include "dataset/util/path.h"
#include "dataset/util/status.h"
#include "gtest/gtest.h"

1
tests/ut/cpp/dataset/zip_op_test.cc
View File

@ -32,7 +32,6 @@
#include "dataset/engine/data_buffer.h"
#include "gtest/gtest.h"
#include "dataset/core/global_context.h"
#include "dataset/util/de_error.h"
#include "utils/log_adapter.h"
namespace common = mindspore::common;