Nigel
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Added dependency removal for data_helper

This commit is contained in:
Eric 2020-11-03 23:28:46 -05:00
parent fd16535017
commit 85f0a9bc8f
4 changed files with 9 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mindspore/ccsrc/minddata/dataset/engine/opt/pre/removal_pass.cc
View File

@ -66,7 +66,7 @@ Status RemovalPass::RunOnTree(ExecutionTree *tree, bool *modified) {
// Then, execute the removal of any nodes that were set up for removal
for (auto node : removal_nodes->nodes_to_remove()) {
node->Remove();
RETURN_IF_NOT_OK(node->Remove());
}
MS_LOG(INFO) << "Pre pass: removal pass complete.";
return Status::OK();

10
mindspore/ccsrc/minddata/dataset/util/data_helper.cc
View File

@ -23,8 +23,6 @@
#include <sstream>
#include <nlohmann/json.hpp>
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/util/log_adapter.h"
#include "minddata/dataset/util/path.h"
#include "minddata/dataset/util/status.h"
@ -122,12 +120,8 @@ Status DataHelper::RemoveKey(const std::string &in_file, const std::string &key,
return Status::OK();
}
size_t DataHelper::DumpTensor(const std::shared_ptr<Tensor> &input, void *addr, const size_t &buffer_size) {
// get tensor size
size_t tensor_size = input->SizeInBytes();
// iterate over entire tensor
const unsigned char *tensor_addr = input->GetBuffer();
// tensor iterator print
size_t DataHelper::DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr,
const size_t &buffer_size) {
// write to address, input order is: destination, source
errno_t ret = memcpy_s(addr, buffer_size, tensor_addr, tensor_size);
if (ret != 0) {

11
mindspore/ccsrc/minddata/dataset/util/data_helper.h
View File

@ -25,11 +25,7 @@
#include <unordered_map>
#include <vector>
#include <nlohmann/json.hpp>
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/data_type.h"
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "./securec.h"
#include "minddata/dataset/util/log_adapter.h"
#include "minddata/dataset/util/path.h"
#include "minddata/dataset/util/status.h"
@ -181,11 +177,12 @@ class DataHelper {
/// \brief Helper function to copy content of a tensor to buffer
/// \note This function iterates over the tensor in bytes, since
/// \param[in] input The tensor to copy value from
/// \param[in] tensor_addr The memory held by a tensor, e.g. tensor->GetBuffer()
/// \param[in] tensor_size The amount of data in bytes in tensor_addr, e.g. tensor->SizeInBytes()
/// \param[out] addr The address to copy tensor data to
/// \param[in] buffer_size The buffer size of addr
/// \return The size of the tensor (bytes copied
size_t DumpTensor(const std::shared_ptr<Tensor> &input, void *addr, const size_t &buffer_size);
size_t DumpData(const unsigned char *tensor_addr, const size_t &tensor_size, void *addr, const size_t &buffer_size);
/// \brief Helper function to delete key in json file
/// note This function will return okay even if key not found

4
tests/ut/cpp/dataset/data_helper_test.cc
View File

@ -153,7 +153,7 @@ TEST_F(MindDataTestDataHelper, MindDataTestTensorWriteFloat) {
// create buffer using system mempool
DataHelper dh;
void *data = malloc(t->SizeInBytes());
auto bytes_copied = dh.DumpTensor(std::move(t), data, t->SizeInBytes());
auto bytes_copied = dh.DumpData(t->GetBuffer(), t->SizeInBytes(), data, t->SizeInBytes());
if (bytes_copied != t->SizeInBytes()) {
EXPECT_TRUE(false);
}
@ -177,7 +177,7 @@ TEST_F(MindDataTestDataHelper, MindDataTestTensorWriteUInt) {
// create buffer using system mempool
DataHelper dh;
void *data = malloc(t->SizeInBytes());
auto bytes_copied = dh.DumpTensor(t, data, t->SizeInBytes());
auto bytes_copied = dh.DumpData(t->GetBuffer(), t->SizeInBytes(), data, t->SizeInBytes());
if (bytes_copied != t->SizeInBytes()) {
EXPECT_TRUE(false);
}