p83571920
/
mindspore
forked from mindspore-Ecosystem/mindspore
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fix error log issue

This commit is contained in:
xiefangqi 2021-02-02 21:51:18 +08:00
parent 832fa1d9fe
commit 7bc35b01f2
5 changed files with 102 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mindspore/ccsrc/minddata/dataset/core/data_type.cc
View File

@ -51,7 +51,9 @@ uint8_t DataType::AsCVType() const {
}
if (res == kCVInvalidType) {
MS_LOG(ERROR) << "Cannot convert to OpenCV type. Return invalid type!";
std::string err_msg = "Cannot convert [" + std::string(kTypeInfo[type_].name_) + "] to OpenCV type.";
err_msg += " Currently unsupported data type: [uint32, int64, uint64, string]";
MS_LOG(ERROR) << err_msg;
}
return res;

75
mindspore/ccsrc/minddata/dataset/core/tensor.cc
View File

@ -269,10 +269,11 @@ Status Tensor::CreateFromFile(const std::string &path, std::shared_ptr<Tensor> *
CHECK_FAIL_RETURN_UNEXPECTED(!fs.fail(), "Fail to open file: " + path);
int64_t num_bytes = fs.seekg(0, std::ios::end).tellg();
CHECK_FAIL_RETURN_UNEXPECTED(num_bytes <= kDeMaxDim, "Invalid file to allocate tensor memory, check path: " + path);
CHECK_FAIL_RETURN_UNEXPECTED(fs.seekg(0, std::ios::beg).good(), "Fail to find size of file");
CHECK_FAIL_RETURN_UNEXPECTED(fs.seekg(0, std::ios::beg).good(), "Fail to find size of file, check path: " + path);
RETURN_IF_NOT_OK(Tensor::CreateEmpty(TensorShape{num_bytes}, DataType(DataType::DE_UINT8), out));
int64_t written_bytes = fs.read(reinterpret_cast<char *>((*out)->GetMutableBuffer()), num_bytes).gcount();
CHECK_FAIL_RETURN_UNEXPECTED(written_bytes == num_bytes && fs.good(), "Error in writing to tensor");
CHECK_FAIL_RETURN_UNEXPECTED(written_bytes == num_bytes && fs.good(),
"Error in writing to tensor, check path: " + path);
fs.close();
return Status::OK();
}
@ -873,6 +874,42 @@ Status Tensor::CopyLastDimAt(const std::shared_ptr<Tensor> &src, const std::vect
return Status::OK();
}
Status Tensor::GetSliceOption(const SliceOption &slice_option, const int32_t &slice_index,
SliceOption *slice_option_ptr) {
if (slice_option.indices_.empty() && !slice_option.slice_.valid()) {
RETURN_STATUS_UNEXPECTED("Both indices and slices can not be empty.");
}
if (!slice_option.indices_.empty() && slice_option.slice_.valid()) {
RETURN_STATUS_UNEXPECTED("Both indices and slices can not be given.");
}
// if slice object was provided, indices should be empty. Generate indices from the slice object.
if (slice_option.indices_.empty()) {
// check if slice is valid
mindspore::dataset::Slice slice_copy = slice_option.slice_;
slice_copy.start_ = HandleNeg(slice_option.slice_.start_, shape_[slice_index]);
slice_copy.stop_ = HandleNeg(slice_option.slice_.stop_, shape_[slice_index]);
slice_copy.start_ = slice_copy.start_ < 0 ? 0 : slice_copy.start_;
slice_copy.stop_ = slice_copy.stop_ < 0 ? 0 : slice_copy.stop_;
dsize_t max_idx = shape_[slice_index];
slice_copy.start_ = slice_copy.start_ > max_idx ? max_idx : slice_copy.start_;
slice_copy.stop_ = slice_copy.stop_ > max_idx ? max_idx : slice_copy.stop_;
*slice_option_ptr = SliceOption(slice_copy);
} else {
// indices validation
std::vector<dsize_t> indices_copy;
for (int j = 0; j < slice_option.indices_.size(); j++) {
dsize_t index = HandleNeg(slice_option.indices_[j], shape_[slice_index]);
CHECK_FAIL_RETURN_UNEXPECTED(index < shape_[slice_index] && index >= 0,
"Index " + std::to_string(index) + " is out of bounds.");
indices_copy.emplace_back(index);
}
*slice_option_ptr = SliceOption(indices_copy);
}
return Status::OK();
}
Status Tensor::Slice(std::shared_ptr<Tensor> *out, const std::vector<SliceOption> slice_options_) {
std::vector<SliceOption> converted_slice_objects;
@ -885,37 +922,9 @@ Status Tensor::Slice(std::shared_ptr<Tensor> *out, const std::vector<SliceOption
continue;
}
if (slice_option.indices_.empty() && !slice_option.slice_.valid()) {
RETURN_STATUS_UNEXPECTED("Both indices and slices can not be empty.");
}
if (!slice_option.indices_.empty() && slice_option.slice_.valid()) {
RETURN_STATUS_UNEXPECTED("Both indices and slices can not be given.");
}
// if slice object was provided, indices should be empty. Generate indices from the slice object.
if (slice_option.indices_.empty()) {
// check if slice is valid
mindspore::dataset::Slice slice_copy = slice_option.slice_;
slice_copy.start_ = HandleNeg(slice_option.slice_.start_, shape_[i]);
slice_copy.stop_ = HandleNeg(slice_option.slice_.stop_, shape_[i]);
slice_copy.start_ = slice_copy.start_ < 0 ? 0 : slice_copy.start_;
slice_copy.stop_ = slice_copy.stop_ < 0 ? 0 : slice_copy.stop_;
dsize_t max_idx = shape_[i];
slice_copy.start_ = slice_copy.start_ > max_idx ? max_idx : slice_copy.start_;
slice_copy.stop_ = slice_copy.stop_ > max_idx ? max_idx : slice_copy.stop_;
converted_slice_objects.emplace_back(SliceOption(slice_copy));
} else {
// indices validation
std::vector<dsize_t> indices_copy;
for (int j = 0; j < slice_option.indices_.size(); j++) {
dsize_t index = HandleNeg(slice_option.indices_[j], shape_[i]);
CHECK_FAIL_RETURN_UNEXPECTED(index < shape_[i] && index >= 0,
"Index " + std::to_string(index) + " is out of bounds.");
indices_copy.emplace_back(index);
}
converted_slice_objects.emplace_back(SliceOption(indices_copy));
}
SliceOption slice_option_item(false);
RETURN_IF_NOT_OK(GetSliceOption(slice_option, i, &slice_option_item));
converted_slice_objects.emplace_back(slice_option_item);
}
// if a string with partial slices, pass in the rest

7
mindspore/ccsrc/minddata/dataset/core/tensor.h
View File

@ -405,6 +405,13 @@ class Tensor {
/// \return Status error code
Status Slice(TensorPtr *out, const std::vector<mindspore::dataset::SliceOption> slice_options);
/// Get slice_option according to shape and index.
/// \param[in] slice_option input SliceOption object
/// \param[in] slice_index index of SliceOption object
/// \param[out] output slice_option with shape info
/// \return Status error code
Status GetSliceOption(const SliceOption &slice_option, const int32_t &slice_index, SliceOption *slice_option_ptr);
#ifdef ENABLE_PYTHON
/// Constructs numpy array from input tensor
/// \param[in] data this data is the location of python data

54
mindspore/ccsrc/minddata/dataset/include/tensor.h
View File

@ -41,6 +41,7 @@
#include "include/api/status.h"
#include "minddata/dataset/core/constants.h"
#include "minddata/dataset/core/data_type.h"
#include "minddata/dataset/core/tensor_helpers.h"
#include "minddata/dataset/core/tensor_shape.h"
#include "minddata/dataset/core/de_tensor.h"
#ifndef ENABLE_ANDROID
@ -171,6 +172,18 @@ class Tensor {
return CreateFromVector(items, TensorShape({static_cast<dsize_t>(items.size())}), out);
}
/// Create a 1D boolean Tensor from a given list of boolean values.
/// \param[in] items elements of the tensor
/// \param[in] shape shape of the output tensor
/// \param[out] out output argument to hold the created Tensor
/// \return Status Code
static Status CreateFromVector(const std::vector<bool> &items, const TensorShape &shape, TensorPtr *out) {
std::vector<uint8_t> temp(items.begin(), items.end());
RETURN_IF_NOT_OK(CreateFromVector(temp, shape, out));
(*out)->type_ = DataType(DataType::DE_BOOL);
return Status::OK();
}
/// Create a numeric scalar Tensor from the given value.
/// \tparam T type of value
/// \param[in] item value
@ -282,7 +295,7 @@ class Tensor {
const TensorShape &shape() const { return shape_; }
/// Check if tensor has data
/// \return bool - true if tensor is empty
/// \return bool - true if tensor is not empty
bool HasData() const { return data_ != nullptr; }
/// Reshape the tensor. The given shape should have the same number of elements in the Tensor
@ -367,20 +380,37 @@ class Tensor {
}
/// Handle negative indices.
/// \param[out] out modified index
/// \param[in] index
/// \param[in] length axis length used to modify index
/// \return dsize_t modified index
static inline dsize_t HandleNeg(dsize_t index, dsize_t length) { return (index < 0) ? (index + length) : index; }
/// Slice tensor bases on the given indices. Copy the sliced data into out tensor. Only rank1 tensors are supported.
/// Handle negative indices for a vector of indices.
/// \param[out] out modified vector of indices
/// \param[in] index_vector vector of indices
/// \return std::vector<dsize_t> modified vector of indices
static inline std::vector<dsize_t> HandleNegIndices(std::vector<dsize_t> index_vector, std::vector<dsize_t> length) {
std::vector<dsize_t> indices(index_vector.size(), 0);
for (int i = 0; i < index_vector.size(); i++) {
indices[i] = HandleNeg(index_vector[i], length[i]);
}
return indices;
}
/// Slice tensor bases on the given indices. Copy the sliced data into out tensor.
/// Based on the type of tensor, SliceNumeric or SliceString will be called
/// \param[out] out Tensor
/// \param[in] indices vector of indices
/// \param[in] slice_options vector of SliceOption objects
/// \return Status error code
Status Slice(TensorPtr *out, const std::vector<dsize_t> &indices);
Status Slice(TensorPtr *out, const std::vector<mindspore::dataset::SliceOption> slice_options);
/// Slice numeric tensors.
Status SliceNumeric(TensorPtr *out, const std::vector<dsize_t> &indices);
/// Slice string tensors
Status SliceString(TensorPtr *out, const std::vector<dsize_t> &indices);
/// Get slice_option according to shape and index.
/// \param[in] slice_option input SliceOption object
/// \param[in] slice_index index of SliceOption object
/// \param[out] output slice_option with shape info
/// \return Status error code
Status GetSliceOption(const SliceOption &slice_option, const int32_t &slice_index, SliceOption *slice_option_ptr);
#ifdef ENABLE_PYTHON
/// Constructs numpy array from input tensor
@ -659,6 +689,12 @@ class Tensor {
private:
friend class DETensor;
/// Slice numeric tensors.
Status SliceNumeric(TensorPtr *out, const std::vector<std::vector<dsize_t>> &indices, const TensorShape &shape);
/// Slice string tensors
Status SliceString(TensorPtr *out, const std::vector<std::vector<dsize_t>> &indices, const TensorShape &shape);
/// Copy raw data of a array based on shape and strides to the destination pointer
/// \param dst [out] Pointer to the destination array where the content is to be copied
/// \param[in] src Pointer to the source of strided array to be copied

5
mindspore/dataset/vision/py_transforms_util.py
View File

@ -56,6 +56,10 @@ def normalize(img, mean, std, pad_channel=False, dtype="float32"):
Returns:
img (numpy.ndarray), Normalized image.
"""
if np.issubdtype(img.dtype, np.integer):
raise NotImplementedError("Unsupported image datatype: [{}], pls execute [ToTensor] before [Normalize]."
.format(img.dtype))
if not is_numpy(img):
raise TypeError("img should be NumPy image. Got {}.".format(type(img)))
@ -74,6 +78,7 @@ def normalize(img, mean, std, pad_channel=False, dtype="float32"):
mean = np.array(mean, dtype=img.dtype)
std = np.array(std, dtype=img.dtype)
image = (img - mean[:, None, None]) / std[:, None, None]
if pad_channel:
zeros = np.zeros([1, image.shape[1], image.shape[2]], dtype=np.float32)