p83571920
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

!13088 Add Error message 

From: @lizhenglong1992
Reviewed-by: @pandoublefeng,@heleiwang
Signed-off-by: @pandoublefeng
This commit is contained in:
mindspore-ci-bot 2021-03-11 10:42:48 +08:00 committed by Gitee
parent 3fc486883e 762c63af6f
commit b4c485e255
11 changed files with 68 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
mindspore/ccsrc/minddata/dataset/api/execute.cc
View File

@ -297,11 +297,6 @@ Status Execute::operator()(const mindspore::MSTensor &input, mindspore::MSTensor
} }
CHECK_FAIL_RETURN_UNEXPECTED(device_input->HasDeviceData(), "Apply transform failed, output tensor has no data"); CHECK_FAIL_RETURN_UNEXPECTED(device_input->HasDeviceData(), "Apply transform failed, output tensor has no data");
// TODO(lizhenglong) waiting for computing department development, hence we pop data onto host temporarily.
// std::shared_ptr<mindspore::dataset::Tensor> host_output;
// RETURN_IF_NOT_OK(device_resource_->Pop(device_input, &host_output));
// *output = mindspore::MSTensor(std::make_shared<DETensor>(host_output));
*output = mindspore::MSTensor(std::make_shared<DETensor>(device_input, true)); *output = mindspore::MSTensor(std::make_shared<DETensor>(device_input, true));
#endif #endif
} }

4
mindspore/ccsrc/minddata/dataset/core/ascend_resource.cc
View File

@ -61,7 +61,9 @@ Status AscendResource::Sink(const mindspore::MSTensor &host_input, std::shared_p
(const uchar *)(host_input.Data().get()), &de_input); (const uchar *)(host_input.Data().get()), &de_input);
RETURN_IF_NOT_OK(rc); RETURN_IF_NOT_OK(rc);
if (!IsNonEmptyJPEG(de_input)) { if (!IsNonEmptyJPEG(de_input)) {
RETURN_STATUS_UNEXPECTED("Dvpp operators can only support processing JPEG image"); std::string err_msg = "Dvpp operators can only support processing JPEG image";
MS_LOG(ERROR) << err_msg;
RETURN_STATUS_UNEXPECTED(err_msg);
} }
APP_ERROR ret = processor_->H2D_Sink(de_input, *device_input); APP_ERROR ret = processor_->H2D_Sink(de_input, *device_input);

20
mindspore/ccsrc/minddata/dataset/core/de_tensor.cc
View File

@ -23,10 +23,10 @@
#include "utils/hashing.h" #include "utils/hashing.h"
#ifndef ENABLE_ANDROID #ifndef ENABLE_ANDROID
#include "utils/log_adapter.h" #include "utils/log_adapter.h"
#define ASSERT_NULL(ptr) MS_EXCEPTION_IF_NULL(ptr) #define EXCEPTION_IF_NULL(ptr) MS_EXCEPTION_IF_NULL(ptr)
#else #else
#include "mindspore/lite/src/common/log_adapter.h" #include "mindspore/lite/src/common/log_adapter.h"
#define ASSERT_NULL(ptr) MS_ASSERT((ptr) != nullptr) #define EXCEPTION_IF_NULL(ptr) MS_ASSERT((ptr) != nullptr)
#endif #endif
namespace mindspore { namespace mindspore {
@ -63,22 +63,22 @@ const std::string &DETensor::Name() const { return name_; }
enum mindspore::DataType DETensor::DataType() const { enum mindspore::DataType DETensor::DataType() const {
#ifndef ENABLE_ANDROID #ifndef ENABLE_ANDROID
if (is_device_) { if (is_device_) {
ASSERT_NULL(device_tensor_impl_); EXCEPTION_IF_NULL(device_tensor_impl_);
return static_cast<mindspore::DataType>(DETypeToMSType(device_tensor_impl_->DeviceDataType())); return static_cast<mindspore::DataType>(DETypeToMSType(device_tensor_impl_->DeviceDataType()));
} }
#endif #endif
ASSERT_NULL(tensor_impl_); EXCEPTION_IF_NULL(tensor_impl_);
return static_cast<mindspore::DataType>(DETypeToMSType(tensor_impl_->type())); return static_cast<mindspore::DataType>(DETypeToMSType(tensor_impl_->type()));
} }
size_t DETensor::DataSize() const { size_t DETensor::DataSize() const {
#ifndef ENABLE_ANDROID #ifndef ENABLE_ANDROID
if (is_device_) { if (is_device_) {
ASSERT_NULL(device_tensor_impl_); EXCEPTION_IF_NULL(device_tensor_impl_);
return device_tensor_impl_->DeviceDataSize(); return device_tensor_impl_->DeviceDataSize();
} }
#endif #endif
ASSERT_NULL(tensor_impl_); EXCEPTION_IF_NULL(tensor_impl_);
return tensor_impl_->SizeInBytes(); return tensor_impl_->SizeInBytes();
} }
@ -87,7 +87,7 @@ const std::vector<int64_t> &DETensor::Shape() const { return shape_; }
std::shared_ptr<const void> DETensor::Data() const { std::shared_ptr<const void> DETensor::Data() const {
#ifndef ENABLE_ANDROID #ifndef ENABLE_ANDROID
if (is_device_) { if (is_device_) {
ASSERT_NULL(device_tensor_impl_); EXCEPTION_IF_NULL(device_tensor_impl_);
return std::shared_ptr<const void>(device_tensor_impl_->GetHostBuffer(), [](const void *) {}); return std::shared_ptr<const void>(device_tensor_impl_->GetHostBuffer(), [](const void *) {});
} }
#endif #endif
@ -97,11 +97,11 @@ std::shared_ptr<const void> DETensor::Data() const {
void *DETensor::MutableData() { void *DETensor::MutableData() {
#ifndef ENABLE_ANDROID #ifndef ENABLE_ANDROID
if (is_device_) { if (is_device_) {
ASSERT_NULL(device_tensor_impl_); EXCEPTION_IF_NULL(device_tensor_impl_);
return static_cast<void *>(device_tensor_impl_->GetDeviceMutableBuffer()); return static_cast<void *>(device_tensor_impl_->GetDeviceMutableBuffer());
} }
#endif #endif
ASSERT_NULL(tensor_impl_); EXCEPTION_IF_NULL(tensor_impl_);
return static_cast<void *>(tensor_impl_->GetMutableBuffer()); return static_cast<void *>(tensor_impl_->GetMutableBuffer());
} }
@ -110,7 +110,7 @@ bool DETensor::IsDevice() const { return is_device_; }
std::shared_ptr<mindspore::MSTensor::Impl> DETensor::Clone() const { std::shared_ptr<mindspore::MSTensor::Impl> DETensor::Clone() const {
#ifndef ENABLE_ANDROID #ifndef ENABLE_ANDROID
if (is_device_) { if (is_device_) {
ASSERT_NULL(device_tensor_impl_); EXCEPTION_IF_NULL(device_tensor_impl_);
return std::make_shared<DETensor>(device_tensor_impl_, is_device_); return std::make_shared<DETensor>(device_tensor_impl_, is_device_);
} }
#endif #endif

7
mindspore/ccsrc/minddata/dataset/core/device_tensor.cc
View File

@ -98,7 +98,7 @@ const unsigned char *DeviceTensor::GetHostBuffer() {
return host_data_tensor_->GetBuffer(); return host_data_tensor_->GetBuffer();
} }
uint8_t *DeviceTensor::GetDeviceBuffer() { return device_data_; } const uint8_t *DeviceTensor::GetDeviceBuffer() { return device_data_; }
uint8_t *DeviceTensor::GetDeviceMutableBuffer() { return device_data_; } uint8_t *DeviceTensor::GetDeviceMutableBuffer() { return device_data_; }
@ -136,6 +136,7 @@ Status DeviceTensor::DataPop_(std::shared_ptr<Tensor> *host_tensor) {
MS_LOG(ERROR) << "Failed to allocate memory from host ret = " << ret; MS_LOG(ERROR) << "Failed to allocate memory from host ret = " << ret;
return Status(StatusCode::kMDNoSpace); return Status(StatusCode::kMDNoSpace);
} }
std::shared_ptr<void> outBuf(resHostBuf, aclrtFreeHost); std::shared_ptr<void> outBuf(resHostBuf, aclrtFreeHost);
auto processedInfo_ = outBuf; auto processedInfo_ = outBuf;
// Memcpy the output data from device to host // Memcpy the output data from device to host
@ -145,6 +146,7 @@ Status DeviceTensor::DataPop_(std::shared_ptr<Tensor> *host_tensor) {
MS_LOG(ERROR) << "Failed to copy memory from device to host, ret = " << ret; MS_LOG(ERROR) << "Failed to copy memory from device to host, ret = " << ret;
return Status(StatusCode::kMDOutOfMemory); return Status(StatusCode::kMDOutOfMemory);
} }
auto data = std::static_pointer_cast<unsigned char>(processedInfo_); auto data = std::static_pointer_cast<unsigned char>(processedInfo_);
unsigned char *ret_ptr = data.get(); unsigned char *ret_ptr = data.get();
@ -155,11 +157,14 @@ Status DeviceTensor::DataPop_(std::shared_ptr<Tensor> *host_tensor) {
uint32_t _output_height_ = this->GetYuvStrideShape()[2]; uint32_t _output_height_ = this->GetYuvStrideShape()[2];
uint32_t _output_heightStride_ = this->GetYuvStrideShape()[3]; uint32_t _output_heightStride_ = this->GetYuvStrideShape()[3];
const mindspore::dataset::DataType dvpp_data_type(mindspore::dataset::DataType::DE_UINT8); const mindspore::dataset::DataType dvpp_data_type(mindspore::dataset::DataType::DE_UINT8);
mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, host_tensor); mindspore::dataset::Tensor::CreateFromMemory(dvpp_shape, dvpp_data_type, ret_ptr, host_tensor);
(*host_tensor)->SetYuvShape(_output_width_, _output_widthStride_, _output_height_, _output_heightStride_); (*host_tensor)->SetYuvShape(_output_width_, _output_widthStride_, _output_height_, _output_heightStride_);
if (!(*host_tensor)->HasData()) { if (!(*host_tensor)->HasData()) {
return Status(StatusCode::kMCDeviceError); return Status(StatusCode::kMCDeviceError);
} }
MS_LOG(INFO) << "Successfully pop DeviceTensor data onto host"; MS_LOG(INFO) << "Successfully pop DeviceTensor data onto host";
return Status::OK(); return Status::OK();
} }

2
mindspore/ccsrc/minddata/dataset/core/device_tensor.h
View File

@ -45,7 +45,7 @@ class DeviceTensor : public Tensor {
const unsigned char *GetHostBuffer(); const unsigned char *GetHostBuffer();
uint8_t *GetDeviceBuffer(); const uint8_t *GetDeviceBuffer();
uint8_t *GetDeviceMutableBuffer(); uint8_t *GetDeviceMutableBuffer();

3
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_crop_jpeg_op.cc
View File

@ -61,6 +61,9 @@ Status DvppCropJpegOp::Compute(const std::shared_ptr<DeviceTensor> &input, std::
Status DvppCropJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) { Status DvppCropJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output); IO_CHECK(input, output);
if (!IsNonEmptyJPEG(input)) {
RETURN_STATUS_UNEXPECTED("DvppCropJpegOp only support process jpeg image.");
}
try { try {
CHECK_FAIL_RETURN_UNEXPECTED(input->GetBuffer() != nullptr, "The input image buffer is empty."); CHECK_FAIL_RETURN_UNEXPECTED(input->GetBuffer() != nullptr, "The input image buffer is empty.");
unsigned char *buffer = const_cast<unsigned char *>(input->GetBuffer()); unsigned char *buffer = const_cast<unsigned char *>(input->GetBuffer());

1
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_jpeg_op.cc
View File

@ -55,6 +55,7 @@ Status DvppDecodeJpegOp::Compute(const std::shared_ptr<DeviceTensor> &input, std
} }
return Status::OK(); return Status::OK();
} }
// Compute() will be called when context=="CPU" // Compute() will be called when context=="CPU"
Status DvppDecodeJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) { Status DvppDecodeJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output); IO_CHECK(input, output);

2
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_decode_resize_crop_jpeg_op.cc
View File

@ -59,7 +59,7 @@ Status DvppDecodeResizeCropJpegOp::Compute(const std::shared_ptr<DeviceTensor> &
Status DvppDecodeResizeCropJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) { Status DvppDecodeResizeCropJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output); IO_CHECK(input, output);
if (!IsNonEmptyJPEG(input)) { if (!IsNonEmptyJPEG(input)) {
RETURN_STATUS_UNEXPECTED("DvppDecodeReiszeJpegOp only support process jpeg image."); RETURN_STATUS_UNEXPECTED("DvppDecodeReiszeCropJpegOp only support process jpeg image.");
} }
try { try {
CHECK_FAIL_RETURN_UNEXPECTED(input->GetBuffer() != nullptr, "The input image buffer is empty."); CHECK_FAIL_RETURN_UNEXPECTED(input->GetBuffer() != nullptr, "The input image buffer is empty.");

4
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_normalize_op.cc
View File

@ -24,8 +24,8 @@ Status DvppNormalizeOp::Compute(const std::shared_ptr<DeviceTensor> &input, std:
const DataType dvpp_data_type(DataType::DE_UINT8); const DataType dvpp_data_type(DataType::DE_UINT8);
mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output); mindspore::dataset::DeviceTensor::CreateEmpty(dvpp_shape, dvpp_data_type, output);
std::vector<uint32_t> yuv_shape = input->GetYuvStrideShape(); std::vector<uint32_t> yuv_shape = input->GetYuvStrideShape();
(*output)->SetAttributes(input->GetDeviceBuffer(), input->DeviceDataSize(), yuv_shape[0], yuv_shape[1], yuv_shape[2], (*output)->SetAttributes(input->GetDeviceMutableBuffer(), input->DeviceDataSize(), yuv_shape[0], yuv_shape[1],
yuv_shape[3]); yuv_shape[2], yuv_shape[3]);
if (!((*output)->HasDeviceData())) { if (!((*output)->HasDeviceData())) {
std::string error = "[ERROR] Fail to get the output result from device memory!"; std::string error = "[ERROR] Fail to get the output result from device memory!";
RETURN_STATUS_UNEXPECTED(error); RETURN_STATUS_UNEXPECTED(error);

3
mindspore/ccsrc/minddata/dataset/kernels/image/dvpp/dvpp_resize_jpeg_op.cc
View File

@ -62,6 +62,9 @@ Status DvppResizeJpegOp::Compute(const std::shared_ptr<DeviceTensor> &input, std
Status DvppResizeJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) { Status DvppResizeJpegOp::Compute(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output) {
IO_CHECK(input, output); IO_CHECK(input, output);
if (!IsNonEmptyJPEG(input)) {
RETURN_STATUS_UNEXPECTED("DvppReiszeJpegOp only support process jpeg image.");
}
try { try {
CHECK_FAIL_RETURN_UNEXPECTED(input->GetBuffer() != nullptr, "The input image buffer is empty."); CHECK_FAIL_RETURN_UNEXPECTED(input->GetBuffer() != nullptr, "The input image buffer is empty.");
unsigned char *buffer = const_cast<unsigned char *>(input->GetBuffer()); unsigned char *buffer = const_cast<unsigned char *>(input->GetBuffer());

52
tests/st/cpp/dataset/test_de.cc
View File

@ -13,6 +13,7 @@
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
#include <iostream>
#include <string> #include <string>
#include <vector> #include <vector>
#include "common/common_test.h" #include "common/common_test.h"
@ -37,11 +38,35 @@ class TestDE : public ST::Common {
TestDE() {} TestDE() {}
}; };
mindspore::MSTensor ReadFileToTensor(const std::string &file) {
if (file.empty()) {
std::cout << "[ERROR]Pointer file is nullptr, return an empty Tensor." << std::endl;
return mindspore::MSTensor();
}
std::ifstream ifs(file);
if (!ifs.good()) {
std::cout << "[ERROR]File: " << file << " does not exist, return an empty Tensor." << std::endl;
return mindspore::MSTensor();
}
if (!ifs.is_open()) {
std::cout << "[ERROR]File: " << file << "open failed, return an empty Tensor." << std::endl;
return mindspore::MSTensor();
}
ifs.seekg(0, std::ios::end);
size_t size = ifs.tellg();
mindspore::MSTensor buf("file", mindspore::DataType::kNumberTypeUInt8, {static_cast<int64_t>(size)}, nullptr, size);
ifs.seekg(0, std::ios::beg);
ifs.read(reinterpret_cast<char *>(buf.MutableData()), size);
ifs.close();
return buf;
}
TEST_F(TestDE, TestResNetPreprocess) { TEST_F(TestDE, TestResNetPreprocess) {
// Read images // Read images
std::shared_ptr<mindspore::dataset::Tensor> de_tensor; auto image = ReadFileToTensor("./data/dataset/apple.jpg");
mindspore::dataset::Tensor::CreateFromFile("./data/dataset/apple.jpg", &de_tensor);
auto image = mindspore::MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor));
// Define transform operations // Define transform operations
std::shared_ptr<TensorTransform> decode(new vision::Decode()); std::shared_ptr<TensorTransform> decode(new vision::Decode());
@ -66,10 +91,15 @@ TEST_F(TestDE, TestResNetPreprocess) {
TEST_F(TestDE, TestDvpp) { TEST_F(TestDE, TestDvpp) {
#ifdef ENABLE_ACL #ifdef ENABLE_ACL
// Read images from target directory // Read images from target directory
/* Old internal method, we deprecate it
std::shared_ptr<mindspore::dataset::Tensor> de_tensor; std::shared_ptr<mindspore::dataset::Tensor> de_tensor;
Status rc = mindspore::dataset::Tensor::CreateFromFile("./data/dataset/apple.jpg", &de_tensor); Status rc = mindspore::dataset::Tensor::CreateFromFile("./data/dataset/apple.jpg", &de_tensor);
ASSERT_TRUE(rc.IsOk()); ASSERT_TRUE(rc.IsOk());
auto image = MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor)); auto image = MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor));
*/
auto image = ReadFileToTensor("./data/dataset/apple.jpg");
// Define dvpp transform // Define dvpp transform
std::vector<uint32_t> crop_paras = {224, 224}; std::vector<uint32_t> crop_paras = {224, 224};
@ -78,7 +108,7 @@ TEST_F(TestDE, TestDvpp) {
mindspore::dataset::Execute Transform(decode_resize_crop, MapTargetDevice::kAscend310); mindspore::dataset::Execute Transform(decode_resize_crop, MapTargetDevice::kAscend310);
// Apply transform on images // Apply transform on images
rc = Transform(image, &image); Status rc = Transform(image, &image);
std::string aipp_cfg = Transform.AippCfgGenerator(); std::string aipp_cfg = Transform.AippCfgGenerator();
ASSERT_EQ(aipp_cfg, "./aipp.cfg"); ASSERT_EQ(aipp_cfg, "./aipp.cfg");
@ -116,10 +146,7 @@ TEST_F(TestDE, TestDvpp) {
TEST_F(TestDE, TestDvppSinkMode) { TEST_F(TestDE, TestDvppSinkMode) {
#ifdef ENABLE_ACL #ifdef ENABLE_ACL
// Read images from target directory // Read images from target directory
std::shared_ptr<mindspore::dataset::Tensor> de_tensor; auto image = ReadFileToTensor("./data/dataset/apple.jpg");
Status rc = mindspore::dataset::Tensor::CreateFromFile("./data/dataset/apple.jpg", &de_tensor);
ASSERT_TRUE(rc.IsOk());
auto image = MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor));
// Define dvpp transform // Define dvpp transform
std::vector<int32_t> crop_paras = {224, 224}; std::vector<int32_t> crop_paras = {224, 224};
@ -131,7 +158,7 @@ TEST_F(TestDE, TestDvppSinkMode) {
mindspore::dataset::Execute Transform(trans_list, MapTargetDevice::kAscend310); mindspore::dataset::Execute Transform(trans_list, MapTargetDevice::kAscend310);
// Apply transform on images // Apply transform on images
rc = Transform(image, &image); Status rc = Transform(image, &image);
// Check image info // Check image info
ASSERT_TRUE(rc.IsOk()); ASSERT_TRUE(rc.IsOk());
@ -159,10 +186,7 @@ TEST_F(TestDE, TestDvppSinkMode) {
TEST_F(TestDE, TestDvppDecodeResizeCropNormalize) { TEST_F(TestDE, TestDvppDecodeResizeCropNormalize) {
#ifdef ENABLE_ACL #ifdef ENABLE_ACL
std::shared_ptr<mindspore::dataset::Tensor> de_tensor; auto image = ReadFileToTensor("./data/dataset/apple.jpg");
Status rc = mindspore::dataset::Tensor::CreateFromFile("./data/dataset/apple.jpg", &de_tensor);
ASSERT_TRUE(rc.IsOk());
auto image = MSTensor(std::make_shared<mindspore::dataset::DETensor>(de_tensor));
// Define dvpp transform // Define dvpp transform
std::vector<int32_t> crop_paras = {416}; std::vector<int32_t> crop_paras = {416};
@ -182,7 +206,7 @@ TEST_F(TestDE, TestDvppDecodeResizeCropNormalize) {
ASSERT_EQ(aipp_cfg, "./aipp.cfg"); ASSERT_EQ(aipp_cfg, "./aipp.cfg");
// Apply transform on images // Apply transform on images
rc = Transform(image, &image); Status rc = Transform(image, &image);
// Check image info // Check image info
ASSERT_TRUE(rc.IsOk()); ASSERT_TRUE(rc.IsOk());