remove make_unique.h
This commit is contained in:
parent
9aab1613e7
commit
84d780c1a4
|
@ -23,7 +23,6 @@
|
|||
#include "dataset/engine/datasetops/source/image_folder_op.h"
|
||||
#include "dataset/engine/datasetops/source/mnist_op.h"
|
||||
#include "dataset/engine/datasetops/source/voc_op.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "dataset/core/tensor.h"
|
||||
#include "dataset/engine/dataset_iterator.h"
|
||||
#include "dataset/engine/datasetops/source/manifest_op.h"
|
||||
|
@ -123,7 +122,7 @@ Status DEPipeline::AssignRootNode(const DsOpPtr &dataset_op) { return (tree_->As
|
|||
Status DEPipeline::LaunchTreeExec() {
|
||||
RETURN_IF_NOT_OK(tree_->Prepare());
|
||||
RETURN_IF_NOT_OK(tree_->Launch());
|
||||
iterator_ = make_unique<DatasetIterator>(tree_);
|
||||
iterator_ = std::make_unique<DatasetIterator>(tree_);
|
||||
if (iterator_ == nullptr) RETURN_STATUS_UNEXPECTED("Cannot create an Iterator.");
|
||||
return Status::OK();
|
||||
}
|
||||
|
@ -311,7 +310,7 @@ Status DEPipeline::ParseStorageOp(const py::dict &args, std::shared_ptr<DatasetO
|
|||
if (!args["schema"].is_none()) {
|
||||
(void)builder->SetSchemaFile(ToString(args["schema"]));
|
||||
} else if (!args["schema_json_string"].is_none()) {
|
||||
std::unique_ptr<DataSchema> schema = make_unique<DataSchema>();
|
||||
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
|
||||
std::string s = ToString(args["schema_json_string"]);
|
||||
RETURN_IF_NOT_OK(schema->LoadSchemaString(s, std::vector<std::string>()));
|
||||
(void)builder->SetNumRows(schema->num_rows());
|
||||
|
@ -689,7 +688,7 @@ Status DEPipeline::ParseTFReaderOp(const py::dict &args, std::shared_ptr<Dataset
|
|||
}
|
||||
}
|
||||
if (schema_exists) {
|
||||
std::unique_ptr<DataSchema> schema = make_unique<DataSchema>();
|
||||
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
|
||||
if (args.contains("schema_file_path")) {
|
||||
RETURN_IF_NOT_OK(schema->LoadSchemaFile(ToString(args["schema_file_path"]), columns_to_load));
|
||||
} else {
|
||||
|
|
|
@ -55,9 +55,9 @@ Status GlobalContext::Init() {
|
|||
// For testing we can use Dummy pool instead
|
||||
|
||||
// Create some tensor allocators for the different types and hook them into the pool.
|
||||
tensor_allocator_ = mindspore::make_unique<Allocator<Tensor>>(mem_pool_);
|
||||
cv_tensor_allocator_ = mindspore::make_unique<Allocator<CVTensor>>(mem_pool_);
|
||||
int_allocator_ = mindspore::make_unique<IntAlloc>(mem_pool_);
|
||||
tensor_allocator_ = std::make_unique<Allocator<Tensor>>(mem_pool_);
|
||||
cv_tensor_allocator_ = std::make_unique<Allocator<CVTensor>>(mem_pool_);
|
||||
int_allocator_ = std::make_unique<IntAlloc>(mem_pool_);
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
|
@ -28,7 +28,6 @@
|
|||
#include "dataset/core/global_context.h"
|
||||
#include "dataset/core/pybind_support.h"
|
||||
#include "dataset/core/tensor_shape.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
|
||||
namespace py = pybind11;
|
||||
namespace mindspore {
|
||||
|
@ -53,7 +52,7 @@ namespace dataset {
|
|||
Tensor::Tensor(const TensorShape &shape, const DataType &type) : shape_(shape), type_(type), data_(nullptr) {
|
||||
// grab the mem pool from global context and create the allocator for char data area
|
||||
std::shared_ptr<MemoryPool> global_pool = GlobalContext::Instance()->mem_pool();
|
||||
data_allocator_ = mindspore::make_unique<Allocator<unsigned char>>(global_pool);
|
||||
data_allocator_ = std::make_unique<Allocator<unsigned char>>(global_pool);
|
||||
}
|
||||
|
||||
Tensor::Tensor(const TensorShape &shape, const DataType &type, const unsigned char *data) : Tensor(shape, type) {
|
||||
|
@ -137,7 +136,7 @@ Status Tensor::CreateTensor(std::shared_ptr<Tensor> *ptr, py::array arr) {
|
|||
if ((*ptr)->type_ == DataType::DE_UNKNOWN) RETURN_STATUS_UNEXPECTED("Invalid data type.");
|
||||
|
||||
std::shared_ptr<MemoryPool> global_pool = GlobalContext::Instance()->mem_pool();
|
||||
(*ptr)->data_allocator_ = mindspore::make_unique<Allocator<unsigned char>>(global_pool);
|
||||
(*ptr)->data_allocator_ = std::make_unique<Allocator<unsigned char>>(global_pool);
|
||||
static_cast<void>((*ptr)->StartAddr());
|
||||
int64_t byte_size = (*ptr)->SizeInBytes();
|
||||
unsigned char *data = static_cast<unsigned char *>(arr.request().ptr);
|
||||
|
|
|
@ -40,7 +40,7 @@ Status DataBuffer::CreateDataBuffer(
|
|||
case DatasetType::kTf: {
|
||||
// This type of buffer is for TF record data.
|
||||
// Allocate derived class version for a TF buffers
|
||||
new_data_buffer = mindspore::make_unique<TFBuffer>(id, kDeBFlagNone, storage_client);
|
||||
new_data_buffer = std::make_unique<TFBuffer>(id, kDeBFlagNone, storage_client);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
|
|
|
@ -26,8 +26,8 @@
|
|||
#include "common/utils.h"
|
||||
#include "dataset/util/status.h"
|
||||
#include "dataset/core/tensor_shape.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "utils/log_adapter.h"
|
||||
#include "dataset/util/de_error.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
@ -58,7 +58,7 @@ ColDescriptor::ColDescriptor(const std::string &col_name, DataType col_type, Ten
|
|||
// our shape. Otherwise, set our shape to be empty.
|
||||
if (in_shape != nullptr) {
|
||||
// Create a shape and copy construct it into our column's shape.
|
||||
tensor_shape_ = mindspore::make_unique<TensorShape>(*in_shape);
|
||||
tensor_shape_ = std::make_unique<TensorShape>(*in_shape);
|
||||
} else {
|
||||
tensor_shape_ = nullptr;
|
||||
}
|
||||
|
@ -75,7 +75,7 @@ ColDescriptor::ColDescriptor(const std::string &col_name, DataType col_type, Ten
|
|||
ColDescriptor::ColDescriptor(const ColDescriptor &in_cd)
|
||||
: type_(in_cd.type_), rank_(in_cd.rank_), tensor_impl_(in_cd.tensor_impl_), col_name_(in_cd.col_name_) {
|
||||
// If it has a tensor shape, make a copy of it with our own unique_ptr.
|
||||
tensor_shape_ = in_cd.hasShape() ? mindspore::make_unique<TensorShape>(in_cd.shape()) : nullptr;
|
||||
tensor_shape_ = in_cd.hasShape() ? std::make_unique<TensorShape>(in_cd.shape()) : nullptr;
|
||||
}
|
||||
|
||||
// Assignment overload
|
||||
|
@ -86,7 +86,7 @@ ColDescriptor &ColDescriptor::operator=(const ColDescriptor &in_cd) {
|
|||
tensor_impl_ = in_cd.tensor_impl_;
|
||||
col_name_ = in_cd.col_name_;
|
||||
// If it has a tensor shape, make a copy of it with our own unique_ptr.
|
||||
tensor_shape_ = in_cd.hasShape() ? mindspore::make_unique<TensorShape>(in_cd.shape()) : nullptr;
|
||||
tensor_shape_ = in_cd.hasShape() ? std::make_unique<TensorShape>(in_cd.shape()) : nullptr;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
|
|
@ -59,8 +59,8 @@ Status BatchOp::operator()() {
|
|||
TaskManager::FindMe()->Post();
|
||||
int32_t epoch_num = 0, batch_num = 0, cnt = 0;
|
||||
TensorRow new_row;
|
||||
std::unique_ptr<TensorQTable> table = make_unique<TensorQTable>();
|
||||
child_iterator_ = mindspore::make_unique<ChildIterator>(this, 0, 0);
|
||||
std::unique_ptr<TensorQTable> table = std::make_unique<TensorQTable>();
|
||||
child_iterator_ = std::make_unique<ChildIterator>(this, 0, 0);
|
||||
RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row));
|
||||
column_name_map_ = child_iterator_->col_name_id_map();
|
||||
int32_t cur_batch_size = 0;
|
||||
|
@ -72,7 +72,7 @@ Status BatchOp::operator()() {
|
|||
if (table->size() == static_cast<size_t>(cur_batch_size)) {
|
||||
RETURN_IF_NOT_OK(worker_queues_[cnt++ % num_workers_]->EmplaceBack(
|
||||
std::make_pair(std::move(table), CBatchInfo(epoch_num, batch_num++, cnt - epoch_num))));
|
||||
table = make_unique<TensorQTable>();
|
||||
table = std::make_unique<TensorQTable>();
|
||||
RETURN_IF_NOT_OK(GetBatchSize(&cur_batch_size, CBatchInfo(epoch_num, batch_num, cnt - epoch_num)));
|
||||
}
|
||||
RETURN_IF_NOT_OK(child_iterator_->FetchNextTensorRow(&new_row));
|
||||
|
@ -82,7 +82,7 @@ Status BatchOp::operator()() {
|
|||
RETURN_IF_NOT_OK(worker_queues_[cnt++ % num_workers_]->EmplaceBack(
|
||||
std::make_pair(std::move(table), CBatchInfo(epoch_num, batch_num++, cnt - epoch_num))));
|
||||
}
|
||||
table = make_unique<TensorQTable>(); // this drops when drop == true
|
||||
table = std::make_unique<TensorQTable>(); // this drops when drop == true
|
||||
// end of the current epoch, batch_num should start from 0 again
|
||||
batch_num = 0;
|
||||
epoch_num++;
|
||||
|
@ -153,9 +153,9 @@ Status BatchOp::WorkerEntry(int32_t workerId) {
|
|||
RETURN_IF_NOT_OK(worker_queues_[workerId]->PopFront(&table_pair));
|
||||
while (table_pair.second.ctrl_ != batchCtrl::kQuit) {
|
||||
if (table_pair.second.ctrl_ == batchCtrl::kEOE) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(workerId, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(workerId, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
|
||||
} else if (table_pair.second.ctrl_ == batchCtrl::kEOF) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(workerId, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(workerId, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
|
||||
} else if (table_pair.second.ctrl_ == batchCtrl::kNoCtrl) {
|
||||
std::unique_ptr<DataBuffer> db = nullptr;
|
||||
RETURN_IF_NOT_OK(MakeBatchedBuffer(std::move(table_pair), &db));
|
||||
|
@ -170,8 +170,8 @@ Status BatchOp::MakeBatchedBuffer(std::pair<std::unique_ptr<TensorQTable>, CBatc
|
|||
std::unique_ptr<DataBuffer> *db) {
|
||||
RETURN_UNEXPECTED_IF_NULL(table_pair.first);
|
||||
if (!input_column_names_.empty()) RETURN_IF_NOT_OK(MapColumns(&table_pair)); // pass it through pyfunc
|
||||
(*db) = make_unique<DataBuffer>(table_pair.second.batch_num_, DataBuffer::kDeBFlagNone);
|
||||
std::unique_ptr<TensorQTable> dest_table = make_unique<TensorQTable>();
|
||||
(*db) = std::make_unique<DataBuffer>(table_pair.second.batch_num_, DataBuffer::kDeBFlagNone);
|
||||
std::unique_ptr<TensorQTable> dest_table = std::make_unique<TensorQTable>();
|
||||
RETURN_IF_NOT_OK(BatchRows(&table_pair.first, &dest_table, table_pair.first->size()));
|
||||
(*db)->set_tensor_table(std::move(dest_table));
|
||||
(*db)->set_column_name_map(column_name_map_);
|
||||
|
|
|
@ -80,9 +80,9 @@ void DatasetOp::CreateConnector(int32_t num_producers, int32_t num_consumers) {
|
|||
MS_LOG(INFO) << "Creating connector in tree operator: " << operator_id_ << ". Producer: " << num_producers
|
||||
<< ". Consumer: " << num_consumers << ".";
|
||||
if (oc_queue_size_ > 0) {
|
||||
out_connector_ = mindspore::make_unique<DbConnector>(num_producers, // The number of producers
|
||||
num_consumers, // Only one consumer (the training App)
|
||||
oc_queue_size_);
|
||||
out_connector_ = std::make_unique<DbConnector>(num_producers, // The number of producers
|
||||
num_consumers, // Only one consumer (the training App)
|
||||
oc_queue_size_);
|
||||
} else {
|
||||
// Some op's may choose not to have an output connector
|
||||
MS_LOG(INFO) << "Bypassed connector creation for tree operator: " << operator_id_ << ".";
|
||||
|
@ -149,7 +149,7 @@ Status DatasetOp::GetNextInput(std::unique_ptr<DataBuffer> *p_buffer, int32_t wo
|
|||
// The base class implementation simply flows the eoe message to output. Derived classes
|
||||
// may override if they need to perform special eoe handling.
|
||||
Status DatasetOp::EoeReceived(int32_t worker_id) {
|
||||
std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
return (out_connector_->Add(static_cast<int>(worker_id), std::move(eoe_buffer)));
|
||||
}
|
||||
|
||||
|
@ -157,7 +157,7 @@ Status DatasetOp::EoeReceived(int32_t worker_id) {
|
|||
// The base class implementation simply flows the eof message to output. Derived classes
|
||||
// may override if they need to perform special eof handling.
|
||||
Status DatasetOp::EofReceived(int32_t worker_id) {
|
||||
std::unique_ptr<DataBuffer> eof_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
|
||||
std::unique_ptr<DataBuffer> eof_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
|
||||
return (out_connector_->Add(static_cast<int>(worker_id), std::move(eof_buffer)));
|
||||
}
|
||||
|
||||
|
|
|
@ -225,7 +225,7 @@ Status DeviceQueueOp::SendDataToCPU() {
|
|||
MS_LOG(INFO) << "Device queue, sending data to CPU.";
|
||||
int64_t total_batch = 0;
|
||||
|
||||
std::unique_ptr<ChildIterator> child_iterator = mindspore::make_unique<ChildIterator>(this, 0, 0);
|
||||
std::unique_ptr<ChildIterator> child_iterator = std::make_unique<ChildIterator>(this, 0, 0);
|
||||
while (!(child_iterator->eof_handled())) {
|
||||
TensorRow curr_row;
|
||||
RETURN_IF_NOT_OK(child_iterator->FetchNextTensorRow(&curr_row));
|
||||
|
|
|
@ -179,7 +179,7 @@ Status MapOp::WorkerEntry(int32_t worker_id) {
|
|||
RETURN_IF_NOT_OK(WorkerEntryInit(in_buffer.get(), &keep_input_columns, &to_process_indices, &final_col_name_id_map,
|
||||
&input_columns, &output_columns));
|
||||
|
||||
std::unique_ptr<TensorQTable> new_tensor_table(mindspore::make_unique<TensorQTable>());
|
||||
std::unique_ptr<TensorQTable> new_tensor_table(std::make_unique<TensorQTable>());
|
||||
// Perform the compute function of TensorOp(s) and store the result in new_tensor_table.
|
||||
RETURN_IF_NOT_OK(WorkerCompute(in_buffer.get(), to_process_indices, new_tensor_table.get(), keep_input_columns,
|
||||
&input_columns, &output_columns));
|
||||
|
|
|
@ -48,7 +48,7 @@ Status ParallelOp::CreateWorkerConnector(int32_t worker_connector_size) {
|
|||
// Instantiate the worker connector. This is the internal connector, not the operators
|
||||
// output connector. It has single master consuming from it (num producers is 1), and the number
|
||||
// of workers is the defined count from the op.
|
||||
worker_connector_ = mindspore::make_unique<DbConnector>(num_workers_, num_producers_, worker_connector_size);
|
||||
worker_connector_ = std::make_unique<DbConnector>(num_workers_, num_producers_, worker_connector_size);
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
|
|
@ -79,7 +79,7 @@ Status ProjectOp::Project(std::unique_ptr<DataBuffer> *data_buffer) {
|
|||
new_column_name_mapping[current_column] = i;
|
||||
projected_column_indices.push_back(column_name_mapping[current_column]);
|
||||
}
|
||||
std::unique_ptr<TensorQTable> new_tensor_table = mindspore::make_unique<TensorQTable>();
|
||||
std::unique_ptr<TensorQTable> new_tensor_table = std::make_unique<TensorQTable>();
|
||||
while ((*data_buffer)->NumRows() > 0) {
|
||||
TensorRow current_row;
|
||||
RETURN_IF_NOT_OK((*data_buffer)->PopRow(¤t_row));
|
||||
|
|
|
@ -84,13 +84,13 @@ Status RenameOp::operator()() {
|
|||
|
||||
// we got eoe, now try again until we get eof
|
||||
MS_LOG(INFO) << "Rename operator EOE Received.";
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))));
|
||||
MS_LOG(DEBUG) << "Rename operator fetching buffer after EOE.";
|
||||
RETURN_IF_NOT_OK(GetNextInput(&curr_buffer));
|
||||
} // end of while eof loop
|
||||
|
||||
MS_LOG(INFO) << "Rename opeerator EOF Received.";
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
|
@ -70,7 +70,7 @@ ShuffleOp::ShuffleOp(int32_t shuffle_size, uint32_t shuffle_seed, int32_t op_con
|
|||
rng_(shuffle_seed),
|
||||
buffer_counter_(0),
|
||||
rows_per_buffer_(rows_per_buffer),
|
||||
shuffle_buffer_(mindspore::make_unique<TensorTable>()),
|
||||
shuffle_buffer_(std::make_unique<TensorTable>()),
|
||||
shuffle_last_row_idx_(0),
|
||||
shuffle_buffer_state_(kShuffleStateInit) {}
|
||||
|
||||
|
@ -90,7 +90,7 @@ Status ShuffleOp::SelfReset() {
|
|||
shuffle_seed_ = distribution(random_device);
|
||||
rng_ = std::mt19937_64(shuffle_seed_);
|
||||
}
|
||||
shuffle_buffer_ = mindspore::make_unique<TensorTable>();
|
||||
shuffle_buffer_ = std::make_unique<TensorTable>();
|
||||
buffer_counter_ = 0;
|
||||
shuffle_last_row_idx_ = 0;
|
||||
shuffle_buffer_state_ = kShuffleStateInit;
|
||||
|
@ -142,7 +142,7 @@ Status ShuffleOp::operator()() {
|
|||
// Create the child iterator to fetch our data from.
|
||||
int32_t worker_id = 0;
|
||||
int32_t child_idx = 0;
|
||||
child_iterator_ = mindspore::make_unique<ChildIterator>(this, worker_id, child_idx);
|
||||
child_iterator_ = std::make_unique<ChildIterator>(this, worker_id, child_idx);
|
||||
|
||||
// Main operator loop
|
||||
while (true) {
|
||||
|
@ -161,7 +161,7 @@ Status ShuffleOp::operator()() {
|
|||
// Step 1)
|
||||
// Create an output tensor table if one is not created yet.
|
||||
if (!new_buffer_table) {
|
||||
new_buffer_table = mindspore::make_unique<TensorQTable>();
|
||||
new_buffer_table = std::make_unique<TensorQTable>();
|
||||
}
|
||||
|
||||
// Step 2)
|
||||
|
@ -176,7 +176,7 @@ Status ShuffleOp::operator()() {
|
|||
// and send this buffer on it's way up the pipeline. Special case is if this is the
|
||||
// last row then we also send it.
|
||||
if (new_buffer_table->size() == rows_per_buffer_ || shuffle_last_row_idx_ == 0) {
|
||||
auto new_buffer = mindspore::make_unique<DataBuffer>(buffer_counter_, DataBuffer::kDeBFlagNone);
|
||||
auto new_buffer = std::make_unique<DataBuffer>(buffer_counter_, DataBuffer::kDeBFlagNone);
|
||||
new_buffer->set_tensor_table(std::move(new_buffer_table));
|
||||
new_buffer->set_column_name_map(column_name_map_);
|
||||
buffer_counter_++;
|
||||
|
@ -218,7 +218,7 @@ Status ShuffleOp::operator()() {
|
|||
// Since we overloaded eoeReceived function, we are responsible to flow the EOE up the
|
||||
// pipepline manually now that we are done draining the shuffle buffer
|
||||
MS_LOG(INFO) << "Shuffle operator sending EOE.";
|
||||
auto eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
auto eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer)));
|
||||
|
||||
// Do not wait for any reset to be flown down from operators above us.
|
||||
|
|
|
@ -40,7 +40,7 @@ Status CelebAOp::Builder::Build(std::shared_ptr<CelebAOp> *op) {
|
|||
builder_sampler_ = std::make_shared<SequentialSampler>();
|
||||
}
|
||||
|
||||
builder_schema_ = make_unique<DataSchema>();
|
||||
builder_schema_ = std::make_unique<DataSchema>();
|
||||
RETURN_IF_NOT_OK(
|
||||
builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1)));
|
||||
// label is like this:0 1 0 0 1......
|
||||
|
@ -83,7 +83,7 @@ CelebAOp::CelebAOp(int32_t num_workers, int32_t rows_per_buffer, const std::stri
|
|||
col_name_map_[data_schema_->column(index).name()] = index;
|
||||
}
|
||||
|
||||
attr_info_queue_ = make_unique<Queue<std::vector<std::string>>>(queue_size);
|
||||
attr_info_queue_ = std::make_unique<Queue<std::vector<std::string>>>(queue_size);
|
||||
io_block_queues_.Init(num_workers_, queue_size);
|
||||
}
|
||||
|
||||
|
@ -311,7 +311,7 @@ Status CelebAOp::AddIOBlock(std::unique_ptr<DataBuffer> *data_buffer) {
|
|||
row_count++;
|
||||
if (row_count % rows_per_buffer_ == 0) {
|
||||
RETURN_IF_NOT_OK(io_block_queues_[buff_count++ % num_workers_]->Add(
|
||||
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
keys.clear();
|
||||
}
|
||||
}
|
||||
|
@ -320,21 +320,21 @@ Status CelebAOp::AddIOBlock(std::unique_ptr<DataBuffer> *data_buffer) {
|
|||
|
||||
if (!keys.empty()) {
|
||||
RETURN_IF_NOT_OK(io_block_queues_[(buff_count++) % num_workers_]->Add(
|
||||
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
}
|
||||
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buff_count++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
io_block_queues_[(buff_count++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buff_count++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
|
||||
io_block_queues_[(buff_count++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
|
||||
for (int32_t i = 0; i < num_workers_; i++) {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[i]->Add(std::move(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))));
|
||||
io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
|
||||
}
|
||||
return Status::OK();
|
||||
} else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buff_count++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
io_block_queues_[(buff_count++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks
|
||||
wp_.Clear();
|
||||
RETURN_IF_NOT_OK(sampler_->GetNextBuffer(data_buffer));
|
||||
|
@ -349,17 +349,17 @@ Status CelebAOp::WorkerEntry(int32_t worker_id) {
|
|||
RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block));
|
||||
while (io_block != nullptr) {
|
||||
if (io_block->eoe() == true) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
|
||||
buffer_id = worker_id;
|
||||
} else if (io_block->eof() == true) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
|
||||
} else {
|
||||
std::vector<int64_t> keys;
|
||||
RETURN_IF_NOT_OK(io_block->GetKeys(&keys));
|
||||
if (keys.empty()) {
|
||||
return Status::OK(); // empty key is a quit signal for workers
|
||||
}
|
||||
std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
|
||||
std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
|
||||
RETURN_IF_NOT_OK(LoadBuffer(keys, &db));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db)));
|
||||
buffer_id += num_workers_;
|
||||
|
@ -370,7 +370,7 @@ Status CelebAOp::WorkerEntry(int32_t worker_id) {
|
|||
}
|
||||
|
||||
Status CelebAOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) {
|
||||
std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>();
|
||||
std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>();
|
||||
for (const auto &key : keys) {
|
||||
TensorRow row;
|
||||
RETURN_IF_NOT_OK(LoadTensorRow(image_labels_vec_[key], &row));
|
||||
|
|
|
@ -47,7 +47,7 @@ Status CifarOp::Builder::Build(std::shared_ptr<CifarOp> *ptr) {
|
|||
if (sampler_ == nullptr) {
|
||||
sampler_ = std::make_shared<SequentialSampler>();
|
||||
}
|
||||
schema_ = make_unique<DataSchema>();
|
||||
schema_ = std::make_unique<DataSchema>();
|
||||
TensorShape scalar = TensorShape::CreateScalar();
|
||||
RETURN_IF_NOT_OK(schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1)));
|
||||
if (cifar_type_ == kCifar10) {
|
||||
|
@ -91,7 +91,7 @@ CifarOp::CifarOp(CifarType type, int32_t num_works, int32_t rows_per_buf, const
|
|||
col_name_map_[data_schema_->column(i).name()] = i;
|
||||
}
|
||||
constexpr uint64_t kUtilQueueSize = 512;
|
||||
cifar_raw_data_block_ = make_unique<Queue<std::vector<unsigned char>>>(kUtilQueueSize);
|
||||
cifar_raw_data_block_ = std::make_unique<Queue<std::vector<unsigned char>>>(kUtilQueueSize);
|
||||
io_block_queues_.Init(num_workers_, queue_size);
|
||||
}
|
||||
|
||||
|
@ -114,7 +114,7 @@ Status CifarOp::operator()() {
|
|||
if (row_cnt_ >= num_samples_) break; // enough row read, break for loop
|
||||
if (row_cnt_ % rows_per_buffer_ == 0) {
|
||||
RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add(
|
||||
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
keys.clear();
|
||||
}
|
||||
}
|
||||
|
@ -122,21 +122,21 @@ Status CifarOp::operator()() {
|
|||
}
|
||||
if (keys.empty() == false) {
|
||||
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(
|
||||
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
}
|
||||
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
|
||||
for (int32_t i = 0; i < num_workers_; i++) {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
|
||||
io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
|
||||
}
|
||||
return Status::OK();
|
||||
} else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks
|
||||
wp_.Clear();
|
||||
RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer));
|
||||
|
@ -169,17 +169,17 @@ Status CifarOp::WorkerEntry(int32_t worker_id) {
|
|||
RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block));
|
||||
while (io_block != nullptr) {
|
||||
if (io_block->eoe() == true) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
|
||||
buffer_id = worker_id;
|
||||
} else if (io_block->eof() == true) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
|
||||
} else {
|
||||
std::vector<int64_t> keys;
|
||||
RETURN_IF_NOT_OK(io_block->GetKeys(&keys));
|
||||
if (keys.empty() == true) {
|
||||
return Status::OK(); // empty key is a quit signal for workers
|
||||
}
|
||||
std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
|
||||
std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
|
||||
RETURN_IF_NOT_OK(LoadBuffer(keys, &db));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db)));
|
||||
buffer_id += num_workers_;
|
||||
|
@ -213,7 +213,7 @@ Status CifarOp::LoadTensorRow(uint64_t index, TensorRow *trow) {
|
|||
|
||||
// Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer
|
||||
Status CifarOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) {
|
||||
std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>();
|
||||
std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>();
|
||||
for (const int64_t &key : keys) {
|
||||
TensorRow trow;
|
||||
RETURN_IF_NOT_OK(LoadTensorRow(key, &trow));
|
||||
|
|
|
@ -173,9 +173,9 @@ Status GeneratorOp::operator()() {
|
|||
bool eof = false;
|
||||
while (!eof) {
|
||||
// Create new buffer each iteration
|
||||
fetched_buffer = mindspore::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone);
|
||||
fetched_buffer = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone);
|
||||
fetched_buffer->set_column_name_map(column_names_map_);
|
||||
std::unique_ptr<TensorQTable> fetched_table = mindspore::make_unique<TensorQTable>();
|
||||
std::unique_ptr<TensorQTable> fetched_table = std::make_unique<TensorQTable>();
|
||||
bool eoe = false;
|
||||
{
|
||||
py::gil_scoped_acquire gil_acquire;
|
||||
|
@ -201,12 +201,12 @@ Status GeneratorOp::operator()() {
|
|||
if (eoe) {
|
||||
// Push out EOE upon StopIteration exception from generator
|
||||
MS_LOG(INFO) << "Generator operator sends out EOE.";
|
||||
std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer)));
|
||||
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
|
||||
// If last repeat or not repeated, push out EOF and exit master loop
|
||||
MS_LOG(INFO) << "Generator operator sends out EOF.";
|
||||
std::unique_ptr<DataBuffer> eof_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
|
||||
std::unique_ptr<DataBuffer> eof_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eof_buffer)));
|
||||
MS_LOG(INFO) << "Generator operator main execution loop complete.";
|
||||
eof = true;
|
||||
|
|
|
@ -39,7 +39,7 @@ Status ImageFolderOp::Builder::Build(std::shared_ptr<ImageFolderOp> *ptr) {
|
|||
if (builder_sampler_ == nullptr) {
|
||||
builder_sampler_ = std::make_shared<SequentialSampler>();
|
||||
}
|
||||
builder_schema_ = make_unique<DataSchema>();
|
||||
builder_schema_ = std::make_unique<DataSchema>();
|
||||
TensorShape scalar = TensorShape::CreateScalar();
|
||||
RETURN_IF_NOT_OK(
|
||||
builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1)));
|
||||
|
@ -82,8 +82,8 @@ ImageFolderOp::ImageFolderOp(int32_t num_wkrs, int32_t rows_per_buffer, std::str
|
|||
for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) {
|
||||
col_name_map_[data_schema_->column(i).name()] = i;
|
||||
}
|
||||
folder_name_queue_ = make_unique<Queue<std::string>>(num_wkrs * queue_size);
|
||||
image_name_queue_ = make_unique<Queue<FolderImagesPair>>(num_wkrs * queue_size);
|
||||
folder_name_queue_ = std::make_unique<Queue<std::string>>(num_wkrs * queue_size);
|
||||
image_name_queue_ = std::make_unique<Queue<FolderImagesPair>>(num_wkrs * queue_size);
|
||||
io_block_queues_.Init(num_workers_, queue_size);
|
||||
}
|
||||
|
||||
|
@ -143,7 +143,7 @@ Status ImageFolderOp::operator()() {
|
|||
row_cnt_++;
|
||||
if (row_cnt_ % rows_per_buffer_ == 0) {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[buf_cnt_++ % num_workers_]->Add(make_unique<IOBlock>(keys, IOBlock::kDeIoBlockNone)));
|
||||
io_block_queues_[buf_cnt_++ % num_workers_]->Add(std::make_unique<IOBlock>(keys, IOBlock::kDeIoBlockNone)));
|
||||
keys.clear();
|
||||
}
|
||||
}
|
||||
|
@ -151,21 +151,21 @@ Status ImageFolderOp::operator()() {
|
|||
}
|
||||
if (keys.empty() == false) {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(keys, IOBlock::kDeIoBlockNone)));
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(keys, IOBlock::kDeIoBlockNone)));
|
||||
}
|
||||
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
|
||||
std::unique_ptr<IOBlock> eoe_block = make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe);
|
||||
std::unique_ptr<IOBlock> eof_block = make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof);
|
||||
std::unique_ptr<IOBlock> eoe_block = std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe);
|
||||
std::unique_ptr<IOBlock> eof_block = std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof);
|
||||
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eoe_block)));
|
||||
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eof_block)));
|
||||
for (int32_t i = 0; i < num_workers_; ++i) {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
|
||||
io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
|
||||
}
|
||||
return Status::OK();
|
||||
} else { // not the last repeat. Sleep master thread, wait for the wake-up from reset
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks
|
||||
wp_.Clear();
|
||||
RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer));
|
||||
|
@ -182,15 +182,15 @@ Status ImageFolderOp::WorkerEntry(int32_t worker_id) {
|
|||
RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block));
|
||||
while (io_block != nullptr) {
|
||||
if (io_block->eoe() == true) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
|
||||
buffer_id = worker_id;
|
||||
} else if (io_block->eof() == true) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
|
||||
} else {
|
||||
std::vector<int64_t> keys;
|
||||
RETURN_IF_NOT_OK(io_block->GetKeys(&keys));
|
||||
if (keys.empty() == true) return Status::OK(); // empty key is a quit signal for workers
|
||||
std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
|
||||
std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
|
||||
RETURN_IF_NOT_OK(LoadBuffer(keys, &db));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db)));
|
||||
buffer_id += num_workers_;
|
||||
|
@ -231,7 +231,7 @@ Status ImageFolderOp::LoadTensorRow(ImageLabelPair pairPtr, TensorRow *trow) {
|
|||
|
||||
// Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer
|
||||
Status ImageFolderOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) {
|
||||
std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>();
|
||||
std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>();
|
||||
TensorRow trow;
|
||||
for (const int64_t &key : keys) {
|
||||
RETURN_IF_NOT_OK(this->LoadTensorRow(image_label_pairs_[key], &trow));
|
||||
|
|
|
@ -40,7 +40,7 @@ Status ManifestOp::Builder::Build(std::shared_ptr<ManifestOp> *ptr) {
|
|||
if (builder_sampler_ == nullptr) {
|
||||
builder_sampler_ = std::make_shared<SequentialSampler>();
|
||||
}
|
||||
builder_schema_ = make_unique<DataSchema>();
|
||||
builder_schema_ = std::make_unique<DataSchema>();
|
||||
RETURN_IF_NOT_OK(
|
||||
builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1)));
|
||||
RETURN_IF_NOT_OK(
|
||||
|
@ -105,7 +105,7 @@ Status ManifestOp::AddIoBlock(std::unique_ptr<DataBuffer> *sampler_buffer) {
|
|||
row_cnt_++;
|
||||
if (row_cnt_ % rows_per_buffer_ == 0) {
|
||||
RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add(
|
||||
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
keys.clear();
|
||||
}
|
||||
}
|
||||
|
@ -113,21 +113,21 @@ Status ManifestOp::AddIoBlock(std::unique_ptr<DataBuffer> *sampler_buffer) {
|
|||
}
|
||||
if (keys.empty() == false) {
|
||||
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(
|
||||
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
}
|
||||
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
|
||||
for (int32_t i = 0; i < num_workers_; i++) {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
|
||||
io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
|
||||
}
|
||||
return Status::OK();
|
||||
} else {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks
|
||||
wp_.Clear();
|
||||
RETURN_IF_NOT_OK(sampler_->GetNextBuffer(sampler_buffer));
|
||||
|
@ -160,17 +160,17 @@ Status ManifestOp::WorkerEntry(int32_t worker_id) {
|
|||
RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block));
|
||||
while (io_block != nullptr) {
|
||||
if (io_block->eoe() == true) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
|
||||
buffer_id = worker_id;
|
||||
} else if (io_block->eof() == true) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
|
||||
} else {
|
||||
std::vector<int64_t> keys;
|
||||
RETURN_IF_NOT_OK(io_block->GetKeys(&keys));
|
||||
if (keys.empty()) {
|
||||
return Status::OK(); // empty key is a quit signal for workers
|
||||
}
|
||||
std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
|
||||
std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
|
||||
RETURN_IF_NOT_OK(LoadBuffer(keys, &db));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db)));
|
||||
buffer_id += num_workers_;
|
||||
|
@ -227,7 +227,7 @@ Status ManifestOp::LoadTensorRow(const std::pair<std::string, std::vector<std::s
|
|||
|
||||
// Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer
|
||||
Status ManifestOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) {
|
||||
std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>();
|
||||
std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>();
|
||||
for (const auto &key : keys) {
|
||||
TensorRow trow;
|
||||
RETURN_IF_NOT_OK(LoadTensorRow(image_labelname_[static_cast<size_t>(key)], &trow));
|
||||
|
|
|
@ -30,7 +30,6 @@
|
|||
#include "dataset/engine/datasetops/dataset_op.h"
|
||||
#include "dataset/engine/db_connector.h"
|
||||
#include "dataset/engine/execution_tree.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "utils/log_adapter.h"
|
||||
|
||||
namespace mindspore {
|
||||
|
@ -96,18 +95,18 @@ MindRecordOp::MindRecordOp(int32_t num_mind_record_workers, int32_t rows_per_buf
|
|||
io_blk_queues_.Init(num_workers_, op_connector_queue_size);
|
||||
if (!block_reader_) return;
|
||||
for (int32_t i = 0; i < num_workers_; ++i) {
|
||||
block_buffer_.emplace_back(make_unique<std::vector<ShardTuple>>(std::vector<ShardTuple>{}));
|
||||
block_buffer_.emplace_back(std::make_unique<std::vector<ShardTuple>>(std::vector<ShardTuple>{}));
|
||||
}
|
||||
}
|
||||
|
||||
// Private helper method to encapsulate some common construction/reset tasks
|
||||
Status MindRecordOp::Init() {
|
||||
shard_reader_ = mindspore::make_unique<ShardReader>();
|
||||
shard_reader_ = std::make_unique<ShardReader>();
|
||||
auto rc = shard_reader_->Open(dataset_file_, num_mind_record_workers_, columns_to_load_, operators_, block_reader_);
|
||||
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(rc != MSRStatus::FAILED, "MindRecordOp init failed.");
|
||||
|
||||
data_schema_ = mindspore::make_unique<DataSchema>();
|
||||
data_schema_ = std::make_unique<DataSchema>();
|
||||
|
||||
std::vector<std::shared_ptr<Schema>> schema_vec = shard_reader_->get_shard_header()->get_schemas();
|
||||
// check whether schema exists, if so use the first one
|
||||
|
@ -144,7 +143,7 @@ Status MindRecordOp::Init() {
|
|||
}
|
||||
|
||||
if (!load_all_cols) {
|
||||
std::unique_ptr<DataSchema> tmp_schema = make_unique<DataSchema>();
|
||||
std::unique_ptr<DataSchema> tmp_schema = std::make_unique<DataSchema>();
|
||||
for (std::string colname : columns_to_load_) {
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(colname_to_ind.find(colname) != colname_to_ind.end(), colname + ": doesn't exist");
|
||||
RETURN_IF_NOT_OK(tmp_schema->AddColumn(data_schema_->column(colname_to_ind[colname])));
|
||||
|
@ -298,7 +297,7 @@ Status MindRecordOp::LoadFloat(TensorShape *new_shape, std::unique_ptr<T[]> *arr
|
|||
RETURN_IF_NOT_OK(GetFloat(&value, columns_json[column_name], use_double));
|
||||
|
||||
*new_shape = TensorShape::CreateScalar();
|
||||
*array_data = mindspore::make_unique<T[]>(1);
|
||||
*array_data = std::make_unique<T[]>(1);
|
||||
(*array_data)[0] = value;
|
||||
} else {
|
||||
if (column.hasShape()) {
|
||||
|
@ -309,7 +308,7 @@ Status MindRecordOp::LoadFloat(TensorShape *new_shape, std::unique_ptr<T[]> *arr
|
|||
}
|
||||
|
||||
int idx = 0;
|
||||
*array_data = mindspore::make_unique<T[]>(new_shape->NumOfElements());
|
||||
*array_data = std::make_unique<T[]>(new_shape->NumOfElements());
|
||||
for (auto &element : columns_json[column_name]) {
|
||||
T value = 0;
|
||||
RETURN_IF_NOT_OK(GetFloat(&value, element, use_double));
|
||||
|
@ -350,7 +349,7 @@ Status MindRecordOp::LoadInt(TensorShape *new_shape, std::unique_ptr<T[]> *array
|
|||
RETURN_IF_NOT_OK(GetInt(&value, columns_json[column_name]));
|
||||
|
||||
*new_shape = TensorShape::CreateScalar();
|
||||
*array_data = mindspore::make_unique<T[]>(1);
|
||||
*array_data = std::make_unique<T[]>(1);
|
||||
(*array_data)[0] = value;
|
||||
} else {
|
||||
if (column.hasShape()) {
|
||||
|
@ -361,7 +360,7 @@ Status MindRecordOp::LoadInt(TensorShape *new_shape, std::unique_ptr<T[]> *array
|
|||
}
|
||||
|
||||
int idx = 0;
|
||||
*array_data = mindspore::make_unique<T[]>(new_shape->NumOfElements());
|
||||
*array_data = std::make_unique<T[]>(new_shape->NumOfElements());
|
||||
for (auto &element : columns_json[column_name]) {
|
||||
T value = 0;
|
||||
RETURN_IF_NOT_OK(GetInt(&value, element));
|
||||
|
@ -431,12 +430,14 @@ Status MindRecordOp::WorkerEntry(int32_t worker_id) {
|
|||
RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block));
|
||||
while (io_block != nullptr) {
|
||||
if (io_block->eoe() == true) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))));
|
||||
RETURN_IF_NOT_OK(
|
||||
out_connector_->Add(worker_id, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))));
|
||||
RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block));
|
||||
continue;
|
||||
}
|
||||
if (io_block->eof() == true) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
|
||||
RETURN_IF_NOT_OK(
|
||||
out_connector_->Add(worker_id, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
|
||||
RETURN_IF_NOT_OK(io_blk_queues_[worker_id]->PopFront(&io_block));
|
||||
continue;
|
||||
}
|
||||
|
@ -486,9 +487,9 @@ Status MindRecordOp::WorkerEntry(int32_t worker_id) {
|
|||
|
||||
Status MindRecordOp::GetBufferFromReader(std::unique_ptr<DataBuffer> *fetched_buffer, int64_t buffer_id,
|
||||
int32_t worker_id) {
|
||||
*fetched_buffer = mindspore::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
|
||||
*fetched_buffer = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
|
||||
(*fetched_buffer)->set_column_name_map(column_name_mapping_);
|
||||
std::unique_ptr<TensorQTable> tensor_table = mindspore::make_unique<TensorQTable>();
|
||||
std::unique_ptr<TensorQTable> tensor_table = std::make_unique<TensorQTable>();
|
||||
for (int32_t i = 0; i < rows_per_buffer_; ++i) {
|
||||
ShardTuple tupled_buffer;
|
||||
if (block_reader_) {
|
||||
|
@ -597,22 +598,22 @@ Status MindRecordOp::operator()() {
|
|||
for (int32_t i = 0; i < buffers_needed_; ++i) {
|
||||
if (block_reader_) RETURN_IF_NOT_OK(FetchBlockBuffer(i));
|
||||
std::vector<int64_t> keys(1, i);
|
||||
RETURN_IF_NOT_OK(
|
||||
io_blk_queues_[buf_cnt_++ % num_workers_]->Add(make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
RETURN_IF_NOT_OK(io_blk_queues_[buf_cnt_++ % num_workers_]->Add(
|
||||
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
}
|
||||
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
RETURN_IF_NOT_OK(
|
||||
io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
|
||||
io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
|
||||
for (int32_t i = 0; i < num_workers_; i++) {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_blk_queues_[i]->Add(std::move(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))));
|
||||
RETURN_IF_NOT_OK(io_blk_queues_[i]->Add(
|
||||
std::move(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone))));
|
||||
}
|
||||
return Status::OK();
|
||||
} else { // not the last repeat. Acquire lock, sleeps master thread, wait for the wake-up from reset
|
||||
RETURN_IF_NOT_OK(
|
||||
io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
io_blk_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
|
||||
// reset our buffer count and go to loop again.
|
||||
RETURN_IF_NOT_OK(shard_reader_wait_post_.Wait());
|
||||
|
@ -656,7 +657,7 @@ Status MindRecordOp::LaunchThreadAndInitOp() {
|
|||
}
|
||||
|
||||
Status MindRecordOp::CountTotalRows(const std::string dataset_path, int64_t *count) {
|
||||
std::unique_ptr<ShardReader> shard_reader = mindspore::make_unique<ShardReader>();
|
||||
std::unique_ptr<ShardReader> shard_reader = std::make_unique<ShardReader>();
|
||||
MSRStatus rc = shard_reader->CountTotalRows(dataset_path, count);
|
||||
if (rc == MSRStatus::FAILED) {
|
||||
RETURN_STATUS_UNEXPECTED("MindRecordOp count total rows failed.");
|
||||
|
|
|
@ -43,7 +43,7 @@ Status MnistOp::Builder::Build(std::shared_ptr<MnistOp> *ptr) {
|
|||
if (builder_sampler_ == nullptr) {
|
||||
builder_sampler_ = std::make_shared<SequentialSampler>();
|
||||
}
|
||||
builder_schema_ = make_unique<DataSchema>();
|
||||
builder_schema_ = std::make_unique<DataSchema>();
|
||||
RETURN_IF_NOT_OK(
|
||||
builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kCv, 1)));
|
||||
TensorShape scalar = TensorShape::CreateScalar();
|
||||
|
@ -89,7 +89,7 @@ Status MnistOp::TraversalSampleIds(const std::shared_ptr<Tensor> &sample_ids, st
|
|||
row_cnt_++;
|
||||
if (row_cnt_ % rows_per_buffer_ == 0) {
|
||||
RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add(
|
||||
make_unique<IOBlock>(IOBlock(*keys, IOBlock::kDeIoBlockNone))));
|
||||
std::make_unique<IOBlock>(IOBlock(*keys, IOBlock::kDeIoBlockNone))));
|
||||
keys->clear();
|
||||
}
|
||||
}
|
||||
|
@ -115,21 +115,21 @@ Status MnistOp::operator()() {
|
|||
}
|
||||
if (keys.empty() == false) {
|
||||
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(
|
||||
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
}
|
||||
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof)));
|
||||
for (int32_t i = 0; i < num_workers_; ++i) {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
|
||||
io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
|
||||
}
|
||||
return Status::OK();
|
||||
} else {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
RETURN_IF_NOT_OK(wp_.Wait()); // Master thread goes to sleep after it has made all the IOBlocks
|
||||
wp_.Clear();
|
||||
RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer));
|
||||
|
@ -145,15 +145,15 @@ Status MnistOp::WorkerEntry(int32_t worker_id) {
|
|||
RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&iOBlock));
|
||||
while (iOBlock != nullptr) {
|
||||
if (iOBlock->eoe() == true) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
|
||||
buffer_id = worker_id;
|
||||
} else if (iOBlock->eof() == true) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF)));
|
||||
} else {
|
||||
std::vector<int64_t> keys;
|
||||
RETURN_IF_NOT_OK(iOBlock->GetKeys(&keys));
|
||||
if (keys.empty() == true) return Status::OK(); // empty key is a quit signal for workers
|
||||
std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
|
||||
std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
|
||||
RETURN_IF_NOT_OK(LoadBuffer(keys, &db));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db)));
|
||||
buffer_id += num_workers_;
|
||||
|
@ -178,7 +178,7 @@ Status MnistOp::LoadTensorRow(const MnistLabelPair &mnist_pair, TensorRow *trow)
|
|||
|
||||
// Looping over LoadTensorRow to make 1 DataBuffer. 1 function call produces 1 buffer
|
||||
Status MnistOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) {
|
||||
std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>();
|
||||
std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>();
|
||||
TensorRow trow;
|
||||
for (const int64_t &key : keys) {
|
||||
RETURN_IF_NOT_OK(this->LoadTensorRow(image_label_pairs_[key], &trow));
|
||||
|
@ -309,8 +309,8 @@ Status MnistOp::ReadImageAndLabel(std::ifstream *image_reader, std::ifstream *la
|
|||
CHECK_FAIL_RETURN_UNEXPECTED((num_images == num_labels), "num_images != num_labels");
|
||||
// The image size of the Mnist dataset is fixed at [28,28]
|
||||
int64_t size = kMnistImageRows * kMnistImageCols;
|
||||
auto images_buf = mindspore::make_unique<char[]>(size * num_images);
|
||||
auto labels_buf = mindspore::make_unique<char[]>(num_images);
|
||||
auto images_buf = std::make_unique<char[]>(size * num_images);
|
||||
auto labels_buf = std::make_unique<char[]>(num_images);
|
||||
if (images_buf == nullptr || labels_buf == nullptr) {
|
||||
std::string err_msg = "Fail to allocate memory for MNIST Buffer.";
|
||||
MS_LOG(ERROR) << err_msg.c_str();
|
||||
|
|
|
@ -52,9 +52,9 @@ Status DistributedSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer
|
|||
if (cnt_ > samples_per_buffer_) {
|
||||
RETURN_STATUS_UNEXPECTED("Distributed Sampler Error");
|
||||
} else if (cnt_ == samples_per_buffer_) {
|
||||
(*out_buffer) = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
(*out_buffer) = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
} else {
|
||||
(*out_buffer) = mindspore::make_unique<DataBuffer>(cnt_, DataBuffer::kDeBFlagNone);
|
||||
(*out_buffer) = std::make_unique<DataBuffer>(cnt_, DataBuffer::kDeBFlagNone);
|
||||
std::shared_ptr<Tensor> sample_ids;
|
||||
RETURN_IF_NOT_OK(CreateSamplerTensor(&sample_ids, samples_per_buffer_));
|
||||
int64_t *id_ptr = reinterpret_cast<int64_t *>(sample_ids->StartAddr());
|
||||
|
@ -63,7 +63,7 @@ Status DistributedSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer
|
|||
*(id_ptr++) = shuffle_ ? shuffle_vec_[static_cast<size_t>(next_id)] : next_id;
|
||||
}
|
||||
TensorRow row(1, sample_ids);
|
||||
(*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, row));
|
||||
(*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, row));
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
|
|
@ -53,9 +53,9 @@ Status PKSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) {
|
|||
if (next_id_ > num_pk_samples_ || num_pk_samples_ == 0) {
|
||||
RETURN_STATUS_UNEXPECTED("Index out of bound in PKSampler");
|
||||
} else if (next_id_ == num_pk_samples_) {
|
||||
(*out_buffer) = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
(*out_buffer) = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
} else {
|
||||
(*out_buffer) = mindspore::make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone);
|
||||
(*out_buffer) = std::make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone);
|
||||
std::shared_ptr<Tensor> sample_ids;
|
||||
int64_t last_id =
|
||||
(samples_per_buffer_ + next_id_ > num_pk_samples_) ? num_pk_samples_ : samples_per_buffer_ + next_id_;
|
||||
|
@ -68,7 +68,7 @@ Status PKSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) {
|
|||
*(id_ptr++) = samples[rnd_ind];
|
||||
}
|
||||
TensorRow row(1, sample_ids);
|
||||
(*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, row));
|
||||
(*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, row));
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
|
|
@ -32,9 +32,9 @@ Status RandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) {
|
|||
if (next_id_ > num_samples_) {
|
||||
RETURN_STATUS_UNEXPECTED("RandomSampler Internal Error");
|
||||
} else if (next_id_ == num_samples_) {
|
||||
(*out_buffer) = make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
(*out_buffer) = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
} else {
|
||||
(*out_buffer) = make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone);
|
||||
(*out_buffer) = std::make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone);
|
||||
std::shared_ptr<Tensor> sampleIds;
|
||||
int64_t last_id = samples_per_buffer_ + next_id_ > num_samples_ ? num_samples_ : samples_per_buffer_ + next_id_;
|
||||
RETURN_IF_NOT_OK(CreateSamplerTensor(&sampleIds, last_id - next_id_));
|
||||
|
@ -44,7 +44,7 @@ Status RandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) {
|
|||
}
|
||||
next_id_ = last_id;
|
||||
TensorRow row(1, sampleIds);
|
||||
(*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, row));
|
||||
(*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, row));
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
@ -61,7 +61,7 @@ Status RandomSampler::Init(const RandomAccessOp *op) {
|
|||
}
|
||||
std::shuffle(shuffled_ids_.begin(), shuffled_ids_.end(), rnd_);
|
||||
} else {
|
||||
dist = make_unique<std::uniform_int_distribution<int64_t>>(0, num_rows_ - 1);
|
||||
dist = std::make_unique<std::uniform_int_distribution<int64_t>>(0, num_rows_ - 1);
|
||||
}
|
||||
rnd_.seed(seed_++);
|
||||
return Status::OK();
|
||||
|
|
|
@ -35,7 +35,7 @@ Status Sampler::CreateSamplerTensor(std::shared_ptr<Tensor> *sample_ids, int64_t
|
|||
}
|
||||
if (col_desc_ == nullptr) {
|
||||
// a ColDescriptor for Tensor that holds SampleIds
|
||||
col_desc_ = make_unique<ColDescriptor>("sampleIds", DataType(DataType::DE_INT64), TensorImpl::kFlexible, 1);
|
||||
col_desc_ = std::make_unique<ColDescriptor>("sampleIds", DataType(DataType::DE_INT64), TensorImpl::kFlexible, 1);
|
||||
}
|
||||
TensorShape shape(std::vector<dsize_t>(1, num_elements));
|
||||
RETURN_IF_NOT_OK(Tensor::CreateTensor(sample_ids, col_desc_->tensorImpl(), shape, col_desc_->type()));
|
||||
|
|
|
@ -27,7 +27,6 @@
|
|||
#include "dataset/engine/data_buffer.h"
|
||||
#include "dataset/engine/data_schema.h"
|
||||
#include "dataset/engine/datasetops/dataset_op.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
|
|
@ -25,9 +25,9 @@ Status SequentialSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer)
|
|||
if (next_id_ > num_samples_) {
|
||||
RETURN_STATUS_UNEXPECTED("Sequential Sampler Internal Error");
|
||||
} else if (next_id_ == num_samples_) {
|
||||
(*out_buffer) = make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
(*out_buffer) = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
} else {
|
||||
(*out_buffer) = make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone);
|
||||
(*out_buffer) = std::make_unique<DataBuffer>(next_id_, DataBuffer::kDeBFlagNone);
|
||||
std::shared_ptr<Tensor> sampleIds;
|
||||
int64_t lastId = (samples_per_buffer_ + next_id_ > num_samples_) ? num_samples_ : samples_per_buffer_ + next_id_;
|
||||
RETURN_IF_NOT_OK(CreateSamplerTensor(&sampleIds, lastId - next_id_));
|
||||
|
@ -36,7 +36,7 @@ Status SequentialSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer)
|
|||
*(idPtr++) = next_id_++;
|
||||
}
|
||||
TensorRow row(1, sampleIds);
|
||||
(*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, row));
|
||||
(*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, row));
|
||||
}
|
||||
return Status::OK();
|
||||
}
|
||||
|
|
|
@ -64,9 +64,9 @@ Status SubsetRandomSampler::Reset() {
|
|||
Status SubsetRandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffer) {
|
||||
// All samples have been drawn
|
||||
if (sample_id_ == indices_.size()) {
|
||||
(*out_buffer) = make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagEOE);
|
||||
(*out_buffer) = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagEOE);
|
||||
} else {
|
||||
(*out_buffer) = make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone);
|
||||
(*out_buffer) = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone);
|
||||
std::shared_ptr<Tensor> outputIds;
|
||||
|
||||
int64_t last_id = sample_id_ + samples_per_buffer_;
|
||||
|
@ -92,7 +92,7 @@ Status SubsetRandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buffe
|
|||
}
|
||||
|
||||
// Create a TensorTable from that single tensor and push into DataBuffer
|
||||
(*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, TensorRow(1, outputIds)));
|
||||
(*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, TensorRow(1, outputIds)));
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
|
|
|
@ -46,10 +46,10 @@ Status WeightedRandomSampler::Init(const RandomAccessOp *op) {
|
|||
CHECK_FAIL_RETURN_UNEXPECTED(num_samples_ > 0 && samples_per_buffer_ > 0, "Fail to init WeightedRandomSampler");
|
||||
|
||||
if (!replacement_) {
|
||||
exp_dist_ = mindspore::make_unique<std::exponential_distribution<>>(1);
|
||||
exp_dist_ = std::make_unique<std::exponential_distribution<>>(1);
|
||||
InitOnePassSampling();
|
||||
} else {
|
||||
discrete_dist_ = mindspore::make_unique<std::discrete_distribution<int64_t>>(weights_.begin(), weights_.end());
|
||||
discrete_dist_ = std::make_unique<std::discrete_distribution<int64_t>>(weights_.begin(), weights_.end());
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
|
@ -96,9 +96,9 @@ Status WeightedRandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buf
|
|||
}
|
||||
|
||||
if (sample_id_ == num_samples_) {
|
||||
(*out_buffer) = make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagEOE);
|
||||
(*out_buffer) = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagEOE);
|
||||
} else {
|
||||
(*out_buffer) = make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone);
|
||||
(*out_buffer) = std::make_unique<DataBuffer>(buffer_id_++, DataBuffer::kDeBFlagNone);
|
||||
std::shared_ptr<Tensor> outputIds;
|
||||
|
||||
int64_t last_id = sample_id_ + samples_per_buffer_;
|
||||
|
@ -132,7 +132,7 @@ Status WeightedRandomSampler::GetNextBuffer(std::unique_ptr<DataBuffer> *out_buf
|
|||
}
|
||||
|
||||
// Create a TensorTable from that single tensor and push into DataBuffer
|
||||
(*out_buffer)->set_tensor_table(make_unique<TensorQTable>(1, TensorRow(1, outputIds)));
|
||||
(*out_buffer)->set_tensor_table(std::make_unique<TensorQTable>(1, TensorRow(1, outputIds)));
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
|
|
|
@ -24,7 +24,6 @@
|
|||
#include "dataset/engine/datasetops/source/storage_client.h"
|
||||
#include "dataset/engine/datasetops/source/storage_op.h"
|
||||
#include "dataset/engine/datasetops/source/tf_client.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "dataset/util/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
|
@ -57,7 +56,7 @@ static Status CreateStorageClientSwitch(
|
|||
case DatasetType::kTf: {
|
||||
// Construct the derived class TFClient, stored as base class StorageClient
|
||||
store_op->set_rows_per_buffer(32);
|
||||
*out_client = mindspore::make_unique<TFClient>(std::move(schema), store_op);
|
||||
*out_client = std::make_unique<TFClient>(std::move(schema), store_op);
|
||||
break;
|
||||
}
|
||||
case DatasetType::kUnknown:
|
||||
|
@ -83,7 +82,7 @@ Status StorageClient::CreateStorageClient(
|
|||
std::shared_ptr<StorageClient> *out_client) { // Out: the created storage client
|
||||
// Make a new schema first. This only assigns the dataset type. It does not
|
||||
// create the columns yet.
|
||||
auto new_schema = mindspore::make_unique<DataSchema>();
|
||||
auto new_schema = std::make_unique<DataSchema>();
|
||||
RETURN_IF_NOT_OK(new_schema->LoadDatasetType(dataset_schema_path));
|
||||
RETURN_IF_NOT_OK(CreateStorageClientSwitch(std::move(new_schema), store_op, out_client));
|
||||
return Status::OK();
|
||||
|
@ -99,7 +98,7 @@ Status StorageClient::CreateStorageClient(
|
|||
std::shared_ptr<StorageClient> *out_client) { // Out: the created storage client
|
||||
// The dataset type is passed in by the user. Create an empty schema with only
|
||||
// only the dataset type filled in and then create the client with it.
|
||||
auto new_schema = mindspore::make_unique<DataSchema>();
|
||||
auto new_schema = std::make_unique<DataSchema>();
|
||||
new_schema->set_dataset_type(in_type);
|
||||
RETURN_IF_NOT_OK(CreateStorageClientSwitch(std::move(new_schema), store_op, out_client));
|
||||
return Status::OK();
|
||||
|
@ -147,7 +146,7 @@ Status StorageClient::AssignDatasetLayout(uint32_t num_rows, // In: Th
|
|||
// The current schema was just an empty one with only the dataset field populated.
|
||||
// Let's copy construct a new one that will be a copy of the input schema (releasing the old
|
||||
// one) and then set the number of rows that the user requested.
|
||||
data_schema_ = mindspore::make_unique<DataSchema>(schema);
|
||||
data_schema_ = std::make_unique<DataSchema>(schema);
|
||||
CHECK_FAIL_RETURN_UNEXPECTED(num_rows <= MAX_INTEGER_INT32, "numRows exceeds the boundary numRows>2147483647");
|
||||
num_rows_in_dataset_ = num_rows;
|
||||
|
||||
|
|
|
@ -303,7 +303,7 @@ Status StorageOp::init() {
|
|||
// For simplicity, we'll make both of them 3 so they are the same size.
|
||||
int32_t action_queue_size = (buffers_needed / num_workers_) + 1;
|
||||
for (int32_t i = 0; i < num_workers_; ++i) {
|
||||
auto new_queue = mindspore::make_unique<Queue<int32_t>>(action_queue_size);
|
||||
auto new_queue = std::make_unique<Queue<int32_t>>(action_queue_size);
|
||||
action_queue_.push_back(std::move(new_queue));
|
||||
}
|
||||
}
|
||||
|
@ -483,10 +483,10 @@ Status StorageOp::operator()() {
|
|||
// Post the control message to tell the workers to stop waiting on action queue
|
||||
// because we are done!
|
||||
RETURN_IF_NOT_OK(this->PostEndOfData());
|
||||
std::unique_ptr<DataBuffer> eoeBuffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
std::unique_ptr<DataBuffer> eoeBuffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoeBuffer)));
|
||||
MS_LOG(INFO) << "StorageOp master: Flow end-of-data eof message.";
|
||||
std::unique_ptr<DataBuffer> eofBuffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
|
||||
std::unique_ptr<DataBuffer> eofBuffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eofBuffer)));
|
||||
MS_LOG(INFO) << "StorageOp master: Main execution loop complete.";
|
||||
done = true; // while loop exit
|
||||
|
@ -496,7 +496,7 @@ Status StorageOp::operator()() {
|
|||
// RepeatOp above us somewhere in the tree will re-init us with the data to fetch again
|
||||
// once it gets the end-of-epoch message.
|
||||
MS_LOG(INFO) << "StorageOp master: Flow end-of-epoch eoe message.";
|
||||
std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer)));
|
||||
|
||||
// reset our buffer count and go to loop again.
|
||||
|
|
|
@ -27,7 +27,6 @@
|
|||
#include "dataset/core/data_type.h"
|
||||
#include "dataset/engine/datasetops/source/storage_client.h"
|
||||
#include "dataset/engine/data_schema.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
@ -72,7 +71,7 @@ Status TFBuffer::Load() {
|
|||
}
|
||||
|
||||
// Construct the Tensor table for this buffer.
|
||||
tensor_table_ = mindspore::make_unique<TensorQTable>();
|
||||
tensor_table_ = std::make_unique<TensorQTable>();
|
||||
|
||||
// At each position in the tensor table, instantiate the shared pointer to it's Tensor.
|
||||
uint32_t row = 0;
|
||||
|
@ -272,7 +271,7 @@ Status TFBuffer::LoadFloatList(const ColDescriptor ¤t_col, const dataengin
|
|||
// Identify how many values we have and then create a local array of these
|
||||
// to deserialize into
|
||||
*num_elements = float_list.value_size();
|
||||
*float_array = mindspore::make_unique<float[]>(*num_elements);
|
||||
*float_array = std::make_unique<float[]>(*num_elements);
|
||||
for (int i = 0; i < float_list.value_size(); i++) {
|
||||
(*float_array)[i] = float_list.value(i);
|
||||
}
|
||||
|
@ -294,7 +293,7 @@ Status TFBuffer::LoadIntList(const ColDescriptor ¤t_col, const dataengine:
|
|||
// Identify how many values we have and then create a local array of these
|
||||
// to deserialize into
|
||||
*num_elements = int64_list.value_size();
|
||||
*int_array = mindspore::make_unique<int64_t[]>(*num_elements);
|
||||
*int_array = std::make_unique<int64_t[]>(*num_elements);
|
||||
for (int i = 0; i < int64_list.value_size(); i++) {
|
||||
(*int_array)[i] = int64_list.value(i);
|
||||
}
|
||||
|
|
|
@ -36,7 +36,6 @@
|
|||
#include "dataset/engine/db_connector.h"
|
||||
#include "dataset/engine/execution_tree.h"
|
||||
#include "dataset/engine/jagged_connector.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "dataset/util/path.h"
|
||||
#include "dataset/util/queue.h"
|
||||
#include "dataset/util/random.h"
|
||||
|
@ -54,7 +53,7 @@ TFReaderOp::Builder::Builder()
|
|||
builder_op_connector_size_ = config_manager->op_connector_size();
|
||||
builder_rows_per_buffer_ = config_manager->rows_per_buffer();
|
||||
builder_shuffle_files_ = false;
|
||||
builder_data_schema_ = make_unique<DataSchema>();
|
||||
builder_data_schema_ = std::make_unique<DataSchema>();
|
||||
}
|
||||
|
||||
Status TFReaderOp::Builder::ValidateInputs() const {
|
||||
|
@ -103,7 +102,7 @@ TFReaderOp::TFReaderOp(int32_t num_workers, int32_t worker_connector_size, int64
|
|||
finished_reading_dataset_(false),
|
||||
shuffle_files_(shuffle_files),
|
||||
data_schema_(std::move(data_schema)),
|
||||
filename_index_(make_unique<StringIndex>()),
|
||||
filename_index_(std::make_unique<StringIndex>()),
|
||||
load_io_block_queue_(true),
|
||||
load_jagged_connector_(true),
|
||||
num_rows_(0),
|
||||
|
@ -129,7 +128,7 @@ Status TFReaderOp::Init() {
|
|||
// parallel op base.
|
||||
RETURN_IF_NOT_OK(ParallelOp::CreateWorkerConnector(worker_connector_size_));
|
||||
|
||||
jagged_buffer_connector_ = mindspore::make_unique<JaggedConnector>(num_workers_, 1, worker_connector_size_);
|
||||
jagged_buffer_connector_ = std::make_unique<JaggedConnector>(num_workers_, 1, worker_connector_size_);
|
||||
|
||||
// temporary: make size large enough to hold all files + EOE to avoid hangs
|
||||
int32_t safe_queue_size = static_cast<int32_t>(std::ceil(dataset_files_list_.size() / num_workers_)) + 1;
|
||||
|
@ -229,7 +228,7 @@ Status TFReaderOp::operator()() {
|
|||
}
|
||||
|
||||
// all workers finished reading for this epoch, and we have read all the data from all workers
|
||||
std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE);
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eoe_buffer)));
|
||||
|
||||
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
|
||||
|
@ -241,7 +240,7 @@ Status TFReaderOp::operator()() {
|
|||
}
|
||||
}
|
||||
|
||||
std::unique_ptr<DataBuffer> eof_buffer = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
|
||||
std::unique_ptr<DataBuffer> eof_buffer = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(eof_buffer)));
|
||||
|
||||
RETURN_IF_NOT_OK(PostEndOfData());
|
||||
|
@ -274,7 +273,7 @@ Status TFReaderOp::WorkerEntry(int32_t worker_id) {
|
|||
MS_LOG(INFO) << "TFReader operator worker " << worker_id << " loaded file " << filename << ".";
|
||||
}
|
||||
} else {
|
||||
std::unique_ptr<DataBuffer> eoe_buffer = mindspore::make_unique<DataBuffer>(1, DataBuffer::kDeBFlagEOE);
|
||||
std::unique_ptr<DataBuffer> eoe_buffer = std::make_unique<DataBuffer>(1, DataBuffer::kDeBFlagEOE);
|
||||
RETURN_IF_NOT_OK(jagged_buffer_connector_->Add(worker_id, std::move(eoe_buffer)));
|
||||
}
|
||||
|
||||
|
@ -288,7 +287,7 @@ Status TFReaderOp::WorkerEntry(int32_t worker_id) {
|
|||
// When the worker pops this control indicator, it will shut itself down gracefully.
|
||||
Status TFReaderOp::PostEndOfData() {
|
||||
for (int i = 0; i < num_workers_; ++i) {
|
||||
std::unique_ptr<FilenameBlock> eof = mindspore::make_unique<FilenameBlock>(IOBlock::kDeIoBlockFlagEof);
|
||||
std::unique_ptr<FilenameBlock> eof = std::make_unique<FilenameBlock>(IOBlock::kDeIoBlockFlagEof);
|
||||
RETURN_IF_NOT_OK(PushIoBlockQueue(i, std::move(eof)));
|
||||
}
|
||||
|
||||
|
@ -299,7 +298,7 @@ Status TFReaderOp::PostEndOfData() {
|
|||
// pops this control indicator, it will wait until the next epoch starts and then resume execution.
|
||||
Status TFReaderOp::PostEndOfEpoch(int32_t queue_index) {
|
||||
for (int i = 0; i < num_workers_; ++i) {
|
||||
std::unique_ptr<FilenameBlock> eoe = mindspore::make_unique<FilenameBlock>(IOBlock::kDeIoBlockFlagEoe);
|
||||
std::unique_ptr<FilenameBlock> eoe = std::make_unique<FilenameBlock>(IOBlock::kDeIoBlockFlagEoe);
|
||||
RETURN_IF_NOT_OK(PushIoBlockQueue((queue_index + i) % num_workers_, std::move(eoe)));
|
||||
}
|
||||
|
||||
|
@ -358,7 +357,7 @@ Status TFReaderOp::FillIOBlockShuffle(const std::vector<int64_t> &i_keys) {
|
|||
}
|
||||
if (!equal_rows_per_shard_) {
|
||||
if (key_index++ % num_devices_ == device_id_) {
|
||||
auto ioBlock = make_unique<FilenameBlock>(*it, kInvalidOffset, kInvalidOffset, IOBlock::kDeIoBlockNone);
|
||||
auto ioBlock = std::make_unique<FilenameBlock>(*it, kInvalidOffset, kInvalidOffset, IOBlock::kDeIoBlockNone);
|
||||
RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock)));
|
||||
queue_index = (queue_index + 1) % num_workers_;
|
||||
}
|
||||
|
@ -367,7 +366,7 @@ Status TFReaderOp::FillIOBlockShuffle(const std::vector<int64_t> &i_keys) {
|
|||
auto file_it = filename_index_->Search(*it);
|
||||
std::string file_name = file_it.value();
|
||||
if (NeedPushFileToblockQueue(file_name, &start_offset, &end_offset, pre_count)) {
|
||||
auto ioBlock = make_unique<FilenameBlock>(*it, start_offset, end_offset, IOBlock::kDeIoBlockNone);
|
||||
auto ioBlock = std::make_unique<FilenameBlock>(*it, start_offset, end_offset, IOBlock::kDeIoBlockNone);
|
||||
RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock)));
|
||||
MS_LOG(DEBUG) << "File name " << *it << " start offset " << start_offset << " end_offset " << end_offset;
|
||||
queue_index = (queue_index + 1) % num_workers_;
|
||||
|
@ -404,14 +403,15 @@ Status TFReaderOp::FillIOBlockNoShuffle() {
|
|||
}
|
||||
if (!equal_rows_per_shard_) {
|
||||
if (key_index++ % num_devices_ == device_id_) {
|
||||
auto ioBlock = make_unique<FilenameBlock>(it.key(), kInvalidOffset, kInvalidOffset, IOBlock::kDeIoBlockNone);
|
||||
auto ioBlock =
|
||||
std::make_unique<FilenameBlock>(it.key(), kInvalidOffset, kInvalidOffset, IOBlock::kDeIoBlockNone);
|
||||
RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock)));
|
||||
queue_index = (queue_index + 1) % num_workers_;
|
||||
}
|
||||
} else {
|
||||
std::string file_name = it.value();
|
||||
if (NeedPushFileToblockQueue(file_name, &start_offset, &end_offset, pre_count)) {
|
||||
auto ioBlock = make_unique<FilenameBlock>(it.key(), start_offset, end_offset, IOBlock::kDeIoBlockNone);
|
||||
auto ioBlock = std::make_unique<FilenameBlock>(it.key(), start_offset, end_offset, IOBlock::kDeIoBlockNone);
|
||||
RETURN_IF_NOT_OK(PushIoBlockQueue(queue_index, std::move(ioBlock)));
|
||||
queue_index = (queue_index + 1) % num_workers_;
|
||||
}
|
||||
|
@ -490,14 +490,13 @@ Status TFReaderOp::LoadFile(const std::string &filename, const int64_t start_off
|
|||
|
||||
int64_t rows_read = 0;
|
||||
int64_t rows_total = 0;
|
||||
std::unique_ptr<DataBuffer> current_buffer =
|
||||
mindspore::make_unique<DataBuffer>(0, DataBuffer::BufferFlags::kDeBFlagNone);
|
||||
std::unique_ptr<DataBuffer> current_buffer = std::make_unique<DataBuffer>(0, DataBuffer::BufferFlags::kDeBFlagNone);
|
||||
std::unordered_map<std::string, int32_t> column_name_map;
|
||||
for (int32_t i = 0; i < data_schema_->NumColumns(); ++i) {
|
||||
column_name_map[data_schema_->column(i).name()] = i;
|
||||
}
|
||||
current_buffer->set_column_name_map(column_name_map);
|
||||
std::unique_ptr<TensorQTable> new_tensor_table = make_unique<TensorQTable>();
|
||||
std::unique_ptr<TensorQTable> new_tensor_table = std::make_unique<TensorQTable>();
|
||||
|
||||
while (reader.peek() != EOF) {
|
||||
if (!load_jagged_connector_) {
|
||||
|
@ -532,9 +531,9 @@ Status TFReaderOp::LoadFile(const std::string &filename, const int64_t start_off
|
|||
current_buffer->set_tensor_table(std::move(new_tensor_table));
|
||||
RETURN_IF_NOT_OK(jagged_buffer_connector_->Add(worker_id, std::move(current_buffer)));
|
||||
|
||||
current_buffer = make_unique<DataBuffer>(0, DataBuffer::BufferFlags::kDeBFlagNone);
|
||||
current_buffer = std::make_unique<DataBuffer>(0, DataBuffer::BufferFlags::kDeBFlagNone);
|
||||
current_buffer->set_column_name_map(column_name_map);
|
||||
new_tensor_table = make_unique<TensorQTable>();
|
||||
new_tensor_table = std::make_unique<TensorQTable>();
|
||||
rows_read = 0;
|
||||
}
|
||||
}
|
||||
|
@ -742,7 +741,7 @@ Status TFReaderOp::LoadFloatList(const ColDescriptor ¤t_col, const dataeng
|
|||
// Identify how many values we have and then create a local array of these
|
||||
// to deserialize into
|
||||
*num_elements = float_list.value_size();
|
||||
*float_array = mindspore::make_unique<float[]>(*num_elements);
|
||||
*float_array = std::make_unique<float[]>(*num_elements);
|
||||
for (int i = 0; i < float_list.value_size(); ++i) {
|
||||
(*float_array)[i] = float_list.value(i);
|
||||
}
|
||||
|
|
|
@ -38,7 +38,7 @@ Status VOCOp::Builder::Build(std::shared_ptr<VOCOp> *ptr) {
|
|||
if (builder_sampler_ == nullptr) {
|
||||
builder_sampler_ = std::make_shared<SequentialSampler>();
|
||||
}
|
||||
builder_schema_ = make_unique<DataSchema>();
|
||||
builder_schema_ = std::make_unique<DataSchema>();
|
||||
RETURN_IF_NOT_OK(
|
||||
builder_schema_->AddColumn(ColDescriptor("image", DataType(DataType::DE_UINT8), TensorImpl::kFlexible, 1)));
|
||||
RETURN_IF_NOT_OK(
|
||||
|
@ -85,7 +85,7 @@ Status VOCOp::TraverseSampleIds(const std::shared_ptr<Tensor> &sample_ids, std::
|
|||
row_cnt_++;
|
||||
if (row_cnt_ % rows_per_buffer_ == 0) {
|
||||
RETURN_IF_NOT_OK(io_block_queues_[buf_cnt_++ % num_workers_]->Add(
|
||||
make_unique<IOBlock>(IOBlock(*keys, IOBlock::kDeIoBlockNone))));
|
||||
std::make_unique<IOBlock>(IOBlock(*keys, IOBlock::kDeIoBlockNone))));
|
||||
keys->clear();
|
||||
}
|
||||
}
|
||||
|
@ -110,21 +110,21 @@ Status VOCOp::operator()() {
|
|||
}
|
||||
if (keys.empty() == false) {
|
||||
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(
|
||||
make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
std::make_unique<IOBlock>(IOBlock(keys, IOBlock::kDeIoBlockNone))));
|
||||
}
|
||||
if (!BitTest(op_ctrl_flags_, kDeOpRepeated) || BitTest(op_ctrl_flags_, kDeOpLastRepeat)) {
|
||||
std::unique_ptr<IOBlock> eoe_block = make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe);
|
||||
std::unique_ptr<IOBlock> eof_block = make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof);
|
||||
std::unique_ptr<IOBlock> eoe_block = std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe);
|
||||
std::unique_ptr<IOBlock> eof_block = std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEof);
|
||||
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eoe_block)));
|
||||
RETURN_IF_NOT_OK(io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::move(eof_block)));
|
||||
for (int32_t i = 0; i < num_workers_; i++) {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[i]->Add(make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
|
||||
io_block_queues_[i]->Add(std::make_unique<IOBlock>(std::vector<int64_t>(), IOBlock::kDeIoBlockNone)));
|
||||
}
|
||||
return Status::OK();
|
||||
} else {
|
||||
RETURN_IF_NOT_OK(
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
io_block_queues_[(buf_cnt_++) % num_workers_]->Add(std::make_unique<IOBlock>(IOBlock::kDeIoBlockFlagEoe)));
|
||||
RETURN_IF_NOT_OK(wp_.Wait());
|
||||
wp_.Clear();
|
||||
RETURN_IF_NOT_OK(sampler_->GetNextBuffer(&sampler_buffer));
|
||||
|
@ -164,7 +164,7 @@ Status VOCOp::LoadTensorRow(const std::string &image_id, TensorRow *trow) {
|
|||
}
|
||||
|
||||
Status VOCOp::LoadBuffer(const std::vector<int64_t> &keys, std::unique_ptr<DataBuffer> *db) {
|
||||
std::unique_ptr<TensorQTable> deq = make_unique<TensorQTable>();
|
||||
std::unique_ptr<TensorQTable> deq = std::make_unique<TensorQTable>();
|
||||
TensorRow trow;
|
||||
for (const uint64_t &key : keys) {
|
||||
RETURN_IF_NOT_OK(this->LoadTensorRow(image_ids_[key], &trow));
|
||||
|
@ -182,15 +182,15 @@ Status VOCOp::WorkerEntry(int32_t worker_id) {
|
|||
RETURN_IF_NOT_OK(io_block_queues_[worker_id]->PopFront(&io_block));
|
||||
while (io_block != nullptr) {
|
||||
if (io_block->eoe() == true) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE)));
|
||||
buffer_id = worker_id;
|
||||
} else if (io_block->eof() == true) {
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, (make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, (std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
|
||||
} else {
|
||||
std::vector<int64_t> keys;
|
||||
RETURN_IF_NOT_OK(io_block->GetKeys(&keys));
|
||||
if (keys.empty() == true) return Status::OK();
|
||||
std::unique_ptr<DataBuffer> db = make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
|
||||
std::unique_ptr<DataBuffer> db = std::make_unique<DataBuffer>(buffer_id, DataBuffer::kDeBFlagNone);
|
||||
RETURN_IF_NOT_OK(LoadBuffer(keys, &db));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(worker_id, std::move(db)));
|
||||
buffer_id += num_workers_;
|
||||
|
|
|
@ -65,13 +65,13 @@ Status ZipOp::operator()() {
|
|||
// initialize the iterators
|
||||
for (int32_t i = 0; i < children_num_; ++i) {
|
||||
// magic number 0 since Zip is not a parallel Op
|
||||
child_iterators_.push_back(mindspore::make_unique<ChildIterator>(this, 0, i));
|
||||
child_iterators_.push_back(std::make_unique<ChildIterator>(this, 0, i));
|
||||
}
|
||||
|
||||
// Loop until eof is true
|
||||
while (!eof_) {
|
||||
// Create tensor table and prepare it by fetching and packing the first zipped row into it.
|
||||
std::unique_ptr<TensorQTable> curr_table = mindspore::make_unique<TensorQTable>();
|
||||
std::unique_ptr<TensorQTable> curr_table = std::make_unique<TensorQTable>();
|
||||
RETURN_IF_NOT_OK(prepare(curr_table.get()));
|
||||
|
||||
// If an eof got picked up during the above prepare, then we're done
|
||||
|
@ -81,7 +81,7 @@ Status ZipOp::operator()() {
|
|||
while (!draining_) {
|
||||
// 1. If a previous loop iteration sent the current table out, then create a new one.
|
||||
if (curr_table == nullptr) {
|
||||
curr_table = mindspore::make_unique<TensorQTable>();
|
||||
curr_table = std::make_unique<TensorQTable>();
|
||||
}
|
||||
|
||||
// 2 fill the table. Note: draining mode might get turned on if any of the child inputs were done
|
||||
|
@ -89,8 +89,7 @@ Status ZipOp::operator()() {
|
|||
|
||||
// 3 create and update buffer and send it to the out connector
|
||||
if (!curr_table->empty()) {
|
||||
std::unique_ptr<DataBuffer> curr_buffer =
|
||||
mindspore::make_unique<DataBuffer>(buffer_id_, DataBuffer::kDeBFlagNone);
|
||||
std::unique_ptr<DataBuffer> curr_buffer = std::make_unique<DataBuffer>(buffer_id_, DataBuffer::kDeBFlagNone);
|
||||
curr_buffer->set_tensor_table(std::move(curr_table));
|
||||
curr_buffer->set_column_name_map(col_name_id_map_);
|
||||
MS_LOG(DEBUG) << "Zip operator finished one buffer, pushing, rows " << curr_buffer->NumRows() << ", cols "
|
||||
|
@ -105,15 +104,14 @@ Status ZipOp::operator()() {
|
|||
MS_LOG(DEBUG) << "Zip operator is now draining child inputs.";
|
||||
RETURN_IF_NOT_OK(drainPipeline());
|
||||
// Now that we have drained child inputs, send the eoe up.
|
||||
RETURN_IF_NOT_OK(
|
||||
out_connector_->Add(0, std::move(mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOE))));
|
||||
}
|
||||
}
|
||||
|
||||
// 5 handle eof
|
||||
// propagate eof here.
|
||||
MS_LOG(INFO) << "Zip operator got EOF, propagating.";
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
|
||||
RETURN_IF_NOT_OK(out_connector_->Add(0, std::move(std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF))));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
|
@ -65,7 +65,7 @@ class DbConnector : public Connector<std::unique_ptr<DataBuffer>> {
|
|||
RETURN_IF_NOT_OK(cv_.Wait(&lk, [this, worker_id]() { return expect_consumer_ == worker_id; }));
|
||||
// Once an EOF message is encountered this flag will be set and we can return early.
|
||||
if (end_of_file_) {
|
||||
*result = mindspore::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
|
||||
*result = std::make_unique<DataBuffer>(0, DataBuffer::kDeBFlagEOF);
|
||||
} else {
|
||||
RETURN_IF_NOT_OK(queues_[pop_from_]->PopFront(result));
|
||||
if (*result == nullptr) {
|
||||
|
|
|
@ -24,7 +24,7 @@ namespace mindspore {
|
|||
namespace dataset {
|
||||
// Constructor
|
||||
ExecutionTree::ExecutionTree() : id_count_(0) {
|
||||
tg_ = mindspore::make_unique<TaskGroup>();
|
||||
tg_ = std::make_unique<TaskGroup>();
|
||||
tree_state_ = kDeTStateInit;
|
||||
prepare_flags_ = kDePrepNone;
|
||||
}
|
||||
|
|
|
@ -24,7 +24,6 @@
|
|||
#include "dataset/core/cv_tensor.h"
|
||||
#include "dataset/core/tensor.h"
|
||||
#include "dataset/core/tensor_shape.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "dataset/util/random.h"
|
||||
|
||||
#define MAX_INT_PRECISION 16777216 // float int precision is 16777216
|
||||
|
@ -376,7 +375,7 @@ Status HwcToChw(std::shared_ptr<Tensor> input, std::shared_ptr<Tensor> *output)
|
|||
int width = input_cv->shape()[1];
|
||||
int num_channels = input_cv->shape()[2];
|
||||
|
||||
auto output_cv = mindspore::make_unique<CVTensor>(TensorShape{num_channels, height, width}, input_cv->type());
|
||||
auto output_cv = std::make_unique<CVTensor>(TensorShape{num_channels, height, width}, input_cv->type());
|
||||
for (int i = 0; i < num_channels; ++i) {
|
||||
cv::Mat mat;
|
||||
RETURN_IF_NOT_OK(output_cv->Mat({i}, &mat));
|
||||
|
|
|
@ -20,7 +20,6 @@
|
|||
|
||||
#include "dataset/core/tensor.h"
|
||||
#include "dataset/kernels/tensor_op.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "dataset/util/status.h"
|
||||
|
||||
namespace mindspore {
|
||||
|
|
|
@ -16,7 +16,6 @@
|
|||
#include "dataset/util/arena.h"
|
||||
#include <unistd.h>
|
||||
#include <utility>
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "dataset/util/system_pool.h"
|
||||
#include "dataset/util/de_error.h"
|
||||
#include "./securec.h"
|
||||
|
|
|
@ -18,10 +18,8 @@
|
|||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <utility>
|
||||
|
||||
#include "./securec.h"
|
||||
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "dataset/util/de_error.h"
|
||||
#include "dataset/util/system_pool.h"
|
||||
|
||||
namespace mindspore {
|
||||
|
|
|
@ -16,6 +16,13 @@
|
|||
#ifndef DATASET_UTIL_DE_ERROR_H_
|
||||
#define DATASET_UTIL_DE_ERROR_H_
|
||||
|
||||
#ifdef DEBUG
|
||||
#include <cassert>
|
||||
#define DS_ASSERT(f) assert(f)
|
||||
#else
|
||||
#define DS_ASSERT(f) ((void)0)
|
||||
#endif
|
||||
|
||||
#include <map>
|
||||
#include "utils/error_code.h"
|
||||
|
||||
|
|
|
@ -18,8 +18,7 @@
|
|||
|
||||
#include <iostream>
|
||||
#include <iterator>
|
||||
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "dataset/util/de_error.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
|
|
@ -14,6 +14,7 @@
|
|||
* limitations under the License.
|
||||
*/
|
||||
#include "dataset/util/lock.h"
|
||||
#include "dataset/util/de_error.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
|
|
@ -19,7 +19,6 @@
|
|||
#include <atomic>
|
||||
#include <condition_variable>
|
||||
#include <mutex>
|
||||
#include "dataset/util/make_unique.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace dataset {
|
||||
|
|
|
@ -1,37 +0,0 @@
|
|||
/**
|
||||
* Copyright 2019 Huawei Technologies Co., Ltd
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
#ifndef DATASET_UTIL_MAKE_UNIQUE_H_
|
||||
#define DATASET_UTIL_MAKE_UNIQUE_H_
|
||||
|
||||
#ifdef DEBUG
|
||||
#include <cassert>
|
||||
#define DS_ASSERT(f) assert(f)
|
||||
#else
|
||||
#define DS_ASSERT(f) ((void)0)
|
||||
#endif
|
||||
|
||||
#include <memory>
|
||||
#include <type_traits>
|
||||
#include <utility>
|
||||
#include "dataset/util/de_error.h"
|
||||
#include "utils/log_adapter.h"
|
||||
|
||||
namespace mindspore {
|
||||
using std::make_unique;
|
||||
} // namespace mindspore
|
||||
|
||||
#endif // DATASET_UTIL_MAKE_UNIQUE_H_
|
|
@ -212,7 +212,7 @@ class QueueList {
|
|||
void Init(int num_queues, int capacity) {
|
||||
queue_list_.reserve(num_queues);
|
||||
for (int i = 0; i < num_queues; i++) {
|
||||
queue_list_.emplace_back(mindspore::make_unique<Queue<T>>(capacity));
|
||||
queue_list_.emplace_back(std::make_unique<Queue<T>>(capacity));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -27,7 +27,6 @@
|
|||
#include <string>
|
||||
#include <thread>
|
||||
#include "dataset/util/de_error.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "dataset/util/intrp_resource.h"
|
||||
#include "dataset/util/list.h"
|
||||
#include "dataset/util/memory_pool.h"
|
||||
|
|
|
@ -17,7 +17,6 @@
|
|||
#include "device/gpu/blocking_queue.h"
|
||||
#include <chrono>
|
||||
#include "device/gpu/gpu_common.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "common/utils.h"
|
||||
|
||||
namespace mindspore {
|
||||
|
@ -32,7 +31,7 @@ GpuQueue::GpuQueue(void *addr, size_t feature_size, size_t label_size, size_t ca
|
|||
stream_(0),
|
||||
node_info_(nullptr) {
|
||||
CHECK_CUDA_RET_WITH_ERROR(cudaStreamCreate(&stream_), "Cuda Create Stream Failed");
|
||||
node_info_ = mindspore::make_unique<NodeInfo[]>(capacity);
|
||||
node_info_ = std::make_unique<NodeInfo[]>(capacity);
|
||||
}
|
||||
|
||||
GpuQueue::~GpuQueue() { buffer_ = nullptr; }
|
||||
|
|
|
@ -23,7 +23,6 @@
|
|||
#include <vector>
|
||||
#include "kernel/gpu/gpu_kernel.h"
|
||||
#include "kernel/gpu/gpu_kernel_factory.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "kernel/gpu/kernel_constants.h"
|
||||
|
||||
namespace mindspore {
|
||||
|
@ -74,8 +73,8 @@ class BiasAddGpuKernel : public GpuKernel {
|
|||
|
||||
// Expand to 4 dims for cudnnSetTensorNdDescriptorEx.
|
||||
auto cudnn_dims = std::max(num_dims, 4UL);
|
||||
std::unique_ptr<int[]> x_dims = mindspore::make_unique<int[]>(cudnn_dims);
|
||||
std::unique_ptr<int[]> b_dims = mindspore::make_unique<int[]>(cudnn_dims);
|
||||
std::unique_ptr<int[]> x_dims = std::make_unique<int[]>(cudnn_dims);
|
||||
std::unique_ptr<int[]> b_dims = std::make_unique<int[]>(cudnn_dims);
|
||||
for (size_t i = 0; i < cudnn_dims; i++) {
|
||||
x_dims[i] = (i < num_dims) ? SizeToInt(x_shape[i]) : 1;
|
||||
b_dims[i] = (i == pos) ? SizeToInt(x_shape[i]) : 1;
|
||||
|
|
|
@ -26,7 +26,6 @@
|
|||
#include "kernel/gpu/gpu_kernel.h"
|
||||
#include "kernel/gpu/gpu_kernel_factory.h"
|
||||
#include "kernel/gpu/kernel_constants.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace kernel {
|
||||
|
@ -84,8 +83,8 @@ class BiasAddGradGpuKernel : public GpuKernel {
|
|||
|
||||
// Expand to 4 dims for cudnnSetTensorNdDescriptorEx.
|
||||
auto cudnn_dims = std::max(num_dims, 4UL);
|
||||
std::unique_ptr<int[]> dy_dims = mindspore::make_unique<int[]>(cudnn_dims);
|
||||
std::unique_ptr<int[]> db_dims = mindspore::make_unique<int[]>(cudnn_dims);
|
||||
std::unique_ptr<int[]> dy_dims = std::make_unique<int[]>(cudnn_dims);
|
||||
std::unique_ptr<int[]> db_dims = std::make_unique<int[]>(cudnn_dims);
|
||||
for (size_t i = 0; i < cudnn_dims; i++) {
|
||||
dy_dims[i] = (i < num_dims) ? SizeToInt(dy_shape[i]) : 1;
|
||||
db_dims[i] = (i == pos) ? SizeToInt(dy_shape[i]) : 1;
|
||||
|
|
|
@ -22,7 +22,6 @@
|
|||
#include <memory>
|
||||
#include "kernel/gpu/gpu_kernel.h"
|
||||
#include "kernel/gpu/gpu_kernel_factory.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "kernel/gpu/kernel_constants.h"
|
||||
|
||||
namespace mindspore {
|
||||
|
@ -144,8 +143,8 @@ class LstmGpuKernel : public GpuKernel {
|
|||
int x_dims[3]{batch_size_, input_size_, 1};
|
||||
int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1};
|
||||
|
||||
x_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
|
||||
y_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
|
||||
x_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
|
||||
y_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
|
||||
|
||||
for (size_t i = 0; i < IntToSize(seq_len_); ++i) {
|
||||
CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_[i]), "create x_desc failed");
|
||||
|
|
|
@ -23,7 +23,6 @@
|
|||
#include "kernel/gpu/gpu_kernel.h"
|
||||
#include "kernel/gpu/gpu_kernel_factory.h"
|
||||
#include "kernel/gpu/kernel_constants.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
|
||||
namespace mindspore {
|
||||
namespace kernel {
|
||||
|
@ -212,9 +211,9 @@ class LstmGradDataGpuKernel : public GpuKernel {
|
|||
int x_dims[3]{batch_size_, input_size_, 1};
|
||||
int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1};
|
||||
|
||||
dx_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
|
||||
y_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
|
||||
dy_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
|
||||
dx_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
|
||||
y_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
|
||||
dy_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
|
||||
|
||||
for (size_t i = 0; i < IntToSize(seq_len_); ++i) {
|
||||
CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&dx_desc_[i]), "create x_desc failed");
|
||||
|
|
|
@ -22,7 +22,6 @@
|
|||
#include <memory>
|
||||
#include "kernel/gpu/gpu_kernel.h"
|
||||
#include "kernel/gpu/gpu_kernel_factory.h"
|
||||
#include "dataset/util/make_unique.h"
|
||||
#include "kernel/gpu/kernel_constants.h"
|
||||
namespace mindspore {
|
||||
namespace kernel {
|
||||
|
@ -169,8 +168,8 @@ class LstmGradWeightGpuKernel : public GpuKernel {
|
|||
int x_dims[3]{batch_size_, input_size_, 1};
|
||||
int y_dims[3]{batch_size_, hidden_size_ * (bidirectional_ ? 2 : 1), 1};
|
||||
|
||||
x_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
|
||||
y_desc_ = mindspore::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
|
||||
x_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
|
||||
y_desc_ = std::make_unique<cudnnTensorDescriptor_t[]>(seq_len_);
|
||||
|
||||
for (size_t i = 0; i < IntToSize(seq_len_); ++i) {
|
||||
CHECK_CUDNN_RET_WITH_EXCEPT(cudnnCreateTensorDescriptor(&x_desc_[i]), "create x_desc failed");
|
||||
|
|
|
@ -116,7 +116,7 @@ TEST_F(MindDataTestCelebaDataset, TestCelebaRepeat) {
|
|||
|
||||
TEST_F(MindDataTestCelebaDataset, TestSubsetRandomSamplerCeleba) {
|
||||
std::vector<int64_t> indices({1});
|
||||
std::unique_ptr<Sampler> sampler = mindspore::make_unique<SubsetRandomSampler>(indices);
|
||||
std::unique_ptr<Sampler> sampler = std::make_unique<SubsetRandomSampler>(indices);
|
||||
uint32_t expect_labels[1][40] = {{0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1}};
|
||||
std::string dir = datasets_root_path_ + "/testCelebAData/";
|
||||
uint32_t count = 0;
|
||||
|
|
|
@ -92,7 +92,7 @@ TEST_F(MindDataTestCifarOp, TestSequentialSamplerCifar10) {
|
|||
TEST_F(MindDataTestCifarOp, TestRandomSamplerCifar10) {
|
||||
uint32_t original_seed = GlobalContext::config_manager()->seed();
|
||||
GlobalContext::config_manager()->set_seed(0);
|
||||
std::unique_ptr<Sampler> sampler = mindspore::make_unique<RandomSampler>(true, 12);
|
||||
std::unique_ptr<Sampler> sampler = std::make_unique<RandomSampler>(true, 12);
|
||||
std::string folder_path = datasets_root_path_ + "/testCifar10Data/";
|
||||
auto tree = Build({Cifarop(16, 2, 32, folder_path, std::move(sampler), 100)});
|
||||
tree->Prepare();
|
||||
|
|
|
@ -138,7 +138,7 @@ TEST_F(MindDataTestImageFolderSampler, TestRandomImageFolder) {
|
|||
TEST_F(MindDataTestImageFolderSampler, TestRandomSamplerImageFolder) {
|
||||
int32_t original_seed = GlobalContext::config_manager()->seed();
|
||||
GlobalContext::config_manager()->set_seed(0);
|
||||
std::unique_ptr<Sampler> sampler = mindspore::make_unique<RandomSampler>(true, 12);
|
||||
std::unique_ptr<Sampler> sampler = std::make_unique<RandomSampler>(true, 12);
|
||||
int32_t res[] = {2, 2, 2, 3, 2, 3, 2, 3, 1, 2, 2, 1}; // ground truth label
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data";
|
||||
auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))});
|
||||
|
@ -200,7 +200,7 @@ TEST_F(MindDataTestImageFolderSampler, TestSequentialImageFolderWithRepeatBatch)
|
|||
TEST_F(MindDataTestImageFolderSampler, TestSubsetRandomSamplerImageFolder) {
|
||||
// id range 0 - 10 is label 0, and id range 11 - 21 is label 1
|
||||
std::vector<int64_t> indices({0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 11});
|
||||
std::unique_ptr<Sampler> sampler = mindspore::make_unique<SubsetRandomSampler>(indices);
|
||||
std::unique_ptr<Sampler> sampler = std::make_unique<SubsetRandomSampler>(indices);
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data";
|
||||
// Expect 6 samples for label 0 and 1
|
||||
int res[2] = {6, 6};
|
||||
|
@ -238,7 +238,7 @@ TEST_F(MindDataTestImageFolderSampler, TestWeightedRandomSamplerImageFolder) {
|
|||
|
||||
// create sampler with replacement = replacement
|
||||
std::unique_ptr<Sampler> sampler =
|
||||
mindspore::make_unique<WeightedRandomSampler>(weights, num_samples, true, samples_per_buffer);
|
||||
std::make_unique<WeightedRandomSampler>(weights, num_samples, true, samples_per_buffer);
|
||||
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data";
|
||||
auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))});
|
||||
|
@ -295,7 +295,7 @@ TEST_F(MindDataTestImageFolderSampler, TestImageFolderClassIndex) {
|
|||
}
|
||||
|
||||
TEST_F(MindDataTestImageFolderSampler, TestDistributedSampler) {
|
||||
std::unique_ptr<Sampler> sampler = mindspore::make_unique<DistributedSampler>(11, 10, false);
|
||||
std::unique_ptr<Sampler> sampler = std::make_unique<DistributedSampler>(11, 10, false);
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data";
|
||||
auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler)), Repeat(4)});
|
||||
tree->Prepare();
|
||||
|
@ -322,7 +322,7 @@ TEST_F(MindDataTestImageFolderSampler, TestDistributedSampler) {
|
|||
}
|
||||
|
||||
TEST_F(MindDataTestImageFolderSampler, TestPKSamplerImageFolder) {
|
||||
std::unique_ptr<Sampler> sampler = mindspore::make_unique<PKSampler>(3, false, 4);
|
||||
std::unique_ptr<Sampler> sampler = std::make_unique<PKSampler>(3, false, 4);
|
||||
int32_t res[] = {0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3}; // ground truth label
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data";
|
||||
auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler))});
|
||||
|
@ -431,7 +431,7 @@ TEST_F(MindDataTestImageFolderSampler, TestImageFolderDatasetSize) {
|
|||
}
|
||||
|
||||
TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding1) {
|
||||
std::unique_ptr<Sampler> sampler = mindspore::make_unique<DistributedSampler>(4, 0, false);
|
||||
std::unique_ptr<Sampler> sampler = std::make_unique<DistributedSampler>(4, 0, false);
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data";
|
||||
// numWrks, rows, conns, path, shuffle, sampler, map, numSamples, decode
|
||||
auto tree = Build({ImageFolder(16, 2, 32, folder_path, false, std::move(sampler), {}, 5)});
|
||||
|
@ -460,7 +460,7 @@ TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding1) {
|
|||
}
|
||||
|
||||
TEST_F(MindDataTestImageFolderSampler, TestImageFolderSharding2) {
|
||||
std::unique_ptr<Sampler> sampler = mindspore::make_unique<DistributedSampler>(4, 3, false);
|
||||
std::unique_ptr<Sampler> sampler = std::make_unique<DistributedSampler>(4, 3, false);
|
||||
std::string folder_path = datasets_root_path_ + "/testPK/data";
|
||||
// numWrks, rows, conns, path, shuffle, sampler, map, numSamples, decode
|
||||
auto tree = Build({ImageFolder(16, 16, 32, folder_path, false, std::move(sampler), {}, 12)});
|
||||
|
|
|
@ -86,7 +86,7 @@ TEST_F(MindDataTestManifest, TestSequentialManifestWithRepeat) {
|
|||
|
||||
TEST_F(MindDataTestManifest, TestSubsetRandomSamplerManifest) {
|
||||
std::vector<int64_t> indices({1});
|
||||
std::unique_ptr<Sampler> sampler = mindspore::make_unique<SubsetRandomSampler>(indices);
|
||||
std::unique_ptr<Sampler> sampler = std::make_unique<SubsetRandomSampler>(indices);
|
||||
std::string file = datasets_root_path_ + "/testManifestData/cpp.json";
|
||||
// Expect 6 samples for label 0 and 1
|
||||
auto tree = Build({Manifest(16, 2, 32, file, "train", std::move(sampler))});
|
||||
|
|
|
@ -45,7 +45,7 @@ TEST_F(MindDataTestProjectOp, TestProjectProject) {
|
|||
.SetRowsPerBuffer(16)
|
||||
.SetWorkerConnectorSize(16)
|
||||
.SetNumWorkers(16);
|
||||
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
|
||||
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
|
||||
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
|
||||
builder.SetDataSchema(std::move(schema));
|
||||
Status rc = builder.Build(&my_tfreader_op); ASSERT_TRUE(rc.IsOk());
|
||||
|
|
|
@ -74,7 +74,7 @@ TEST_F(MindDataTestStandAloneSampler, TestDistributedSampler) {
|
|||
std::unique_ptr<DataBuffer> db;
|
||||
std::shared_ptr<Tensor> tensor;
|
||||
for (int i = 0; i < 6; i++) {
|
||||
std::unique_ptr<Sampler> sampler = mindspore::make_unique<DistributedSampler>(3, i % 3, (i < 3 ? false : true));
|
||||
std::unique_ptr<Sampler> sampler = std::make_unique<DistributedSampler>(3, i % 3, (i < 3 ? false : true));
|
||||
sampler->Init(&mock);
|
||||
sampler->GetNextBuffer(&db);
|
||||
db->GetTensor(&tensor, 0, 0);
|
||||
|
|
|
@ -48,7 +48,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderBasic1) {
|
|||
builder.SetDatasetFilesList({dataset_path})
|
||||
.SetRowsPerBuffer(16)
|
||||
.SetNumWorkers(16);
|
||||
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
|
||||
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
|
||||
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
|
||||
builder.SetDataSchema(std::move(schema));
|
||||
Status rc = builder.Build(&my_tfreader_op);
|
||||
|
@ -102,7 +102,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderLargeRowsPerBuffer) {
|
|||
builder.SetDatasetFilesList({dataset_path})
|
||||
.SetRowsPerBuffer(500)
|
||||
.SetNumWorkers(16);
|
||||
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
|
||||
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
|
||||
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
|
||||
builder.SetDataSchema(std::move(schema));
|
||||
Status rc = builder.Build(&my_tfreader_op);
|
||||
|
@ -156,7 +156,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderSmallRowsPerBuffer) {
|
|||
builder.SetDatasetFilesList({dataset_path})
|
||||
.SetRowsPerBuffer(1)
|
||||
.SetNumWorkers(16);
|
||||
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
|
||||
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
|
||||
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
|
||||
builder.SetDataSchema(std::move(schema));
|
||||
Status rc = builder.Build(&my_tfreader_op);
|
||||
|
@ -211,7 +211,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderLargeQueueSize) {
|
|||
.SetWorkerConnectorSize(1)
|
||||
.SetRowsPerBuffer(16)
|
||||
.SetNumWorkers(16);
|
||||
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
|
||||
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
|
||||
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
|
||||
builder.SetDataSchema(std::move(schema));
|
||||
Status rc = builder.Build(&my_tfreader_op);
|
||||
|
@ -265,7 +265,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderOneThread) {
|
|||
builder.SetDatasetFilesList({dataset_path})
|
||||
.SetRowsPerBuffer(16)
|
||||
.SetNumWorkers(1);
|
||||
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
|
||||
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
|
||||
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
|
||||
builder.SetDataSchema(std::move(schema));
|
||||
Status rc = builder.Build(&my_tfreader_op);
|
||||
|
@ -321,7 +321,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderRepeat) {
|
|||
.SetRowsPerBuffer(16)
|
||||
.SetWorkerConnectorSize(16)
|
||||
.SetNumWorkers(16);
|
||||
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
|
||||
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
|
||||
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema.json", {});
|
||||
builder.SetDataSchema(std::move(schema));
|
||||
Status rc= builder.Build(&my_tfreader_op);
|
||||
|
@ -379,7 +379,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderSchemaConstructor) {
|
|||
std::string dataset_path;
|
||||
dataset_path = datasets_root_path_ + "/testTFTestAllTypes";
|
||||
|
||||
std::unique_ptr<DataSchema> data_schema = mindspore::make_unique<DataSchema>();
|
||||
std::unique_ptr<DataSchema> data_schema = std::make_unique<DataSchema>();
|
||||
std::vector<std::string> columns_to_load;
|
||||
columns_to_load.push_back("col_sint32");
|
||||
columns_to_load.push_back("col_binary");
|
||||
|
@ -445,7 +445,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake1Row) {
|
|||
std::shared_ptr<TFReaderOp> my_tfreader_op;
|
||||
TFReaderOp::Builder builder;
|
||||
builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16);
|
||||
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
|
||||
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
|
||||
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema1Row.json", {});
|
||||
builder.SetDataSchema(std::move(schema));
|
||||
|
||||
|
@ -503,7 +503,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake1Buffer) {
|
|||
std::shared_ptr<TFReaderOp> my_tfreader_op;
|
||||
TFReaderOp::Builder builder;
|
||||
builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16);
|
||||
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
|
||||
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
|
||||
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema5Rows.json", {});
|
||||
builder.SetDataSchema(std::move(schema));
|
||||
|
||||
|
@ -561,7 +561,7 @@ TEST_F(MindDataTestTFReaderOp, TestTFReaderTake7Rows) {
|
|||
std::shared_ptr<TFReaderOp> my_tfreader_op;
|
||||
TFReaderOp::Builder builder;
|
||||
builder.SetDatasetFilesList({dataset_path + "/test.data"}).SetRowsPerBuffer(5).SetNumWorkers(16);
|
||||
std::unique_ptr<DataSchema> schema = mindspore::make_unique<DataSchema>();
|
||||
std::unique_ptr<DataSchema> schema = std::make_unique<DataSchema>();
|
||||
schema->LoadSchemaFile(datasets_root_path_ + "/testTFTestAllTypes/datasetSchema7Rows.json", {});
|
||||
builder.SetDataSchema(std::move(schema));
|
||||
|
||||
|
|
Loading…
Reference in New Issue