mindspore-Ecosystem
/
mindspore
21
0
Fork 27
Code Issues Pull Requests 6 Projects Releases Wiki Activity

adding bbox class

This commit is contained in:
islam_amin 2020-08-20 16:53:25 -04:00
parent 13d1738ff3
commit 641e751d5e
13 changed files with 388 additions and 218 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
mindspore/ccsrc/minddata/dataset/kernels/image/CMakeLists.txt
View File

@ -4,6 +4,7 @@ add_subdirectory(soft_dvpp)
add_library(kernels-image OBJECT
affine_op.cc
auto_contrast_op.cc
bounding_box.cc
center_crop_op.cc
crop_op.cc
cut_out_op.cc

195
mindspore/ccsrc/minddata/dataset/kernels/image/bounding_box.cc Normal file
View File

@ -0,0 +1,195 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#include "minddata/dataset/kernels/image/bounding_box.h"
#include <algorithm>
#include <vector>
namespace mindspore {
namespace dataset {
const uint8_t kNumOfCols = 4;
BoundingBox::BoundingBox(bbox_float x, bbox_float y, bbox_float width, bbox_float height)
: x_(x), y_(y), width_(width), height_(height) {}
Status BoundingBox::ReadFromTensor(const TensorPtr &bbox_tensor, dsize_t index_of_bbox,
std::shared_ptr<BoundingBox> *bbox_out) {
bbox_float x;
bbox_float y;
bbox_float width;
bbox_float height;
RETURN_IF_NOT_OK(bbox_tensor->GetItemAt<bbox_float>(&x, {index_of_bbox, 0}));
RETURN_IF_NOT_OK(bbox_tensor->GetItemAt<bbox_float>(&y, {index_of_bbox, 1}));
RETURN_IF_NOT_OK(bbox_tensor->GetItemAt<bbox_float>(&width, {index_of_bbox, 2}));
RETURN_IF_NOT_OK(bbox_tensor->GetItemAt<bbox_float>(&height, {index_of_bbox, 3}));
*bbox_out = std::make_shared<BoundingBox>(x, y, width, height);
return Status::OK();
}
Status BoundingBox::ValidateBoundingBoxes(const TensorRow &image_and_bbox) {
if (image_and_bbox.size() != 2) {
return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__,
"Requires Image and Bounding Boxes, likely missed bounding boxes.");
}
if (image_and_bbox[1]->shape().Size() < 2) {
return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__,
"Bounding boxes shape should have at least two dimensions.");
}
uint32_t num_of_features = image_and_bbox[1]->shape()[1];
if (num_of_features < 4) {
return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__,
"Bounding boxes should be have at least 4 features.");
}
std::vector<std::shared_ptr<BoundingBox>> bbox_list;
RETURN_IF_NOT_OK(GetListOfBoundingBoxes(image_and_bbox[1], &bbox_list));
uint32_t img_h = image_and_bbox[0]->shape()[0];
uint32_t img_w = image_and_bbox[0]->shape()[1];
for (auto &bbox : bbox_list) {
if ((bbox->x() + bbox->width() > img_w) || (bbox->y() + bbox->height() > img_h)) {
return Status(StatusCode::kBoundingBoxOutOfBounds, __LINE__, __FILE__,
"At least one of the bounding boxes is out of bounds of the image.");
}
if (static_cast<int>(bbox->x()) < 0 || static_cast<int>(bbox->y()) < 0) {
return Status(StatusCode::kBoundingBoxOutOfBounds, __LINE__, __FILE__,
"At least one of the bounding boxes has negative min_x or min_y.");
}
}
return Status::OK();
}
Status BoundingBox::WriteToTensor(const TensorPtr &bbox_tensor, dsize_t index_of_bbox) {
RETURN_IF_NOT_OK(bbox_tensor->SetItemAt<bbox_float>({index_of_bbox, 0}, x_));
RETURN_IF_NOT_OK(bbox_tensor->SetItemAt<bbox_float>({index_of_bbox, 1}, y_));
RETURN_IF_NOT_OK(bbox_tensor->SetItemAt<bbox_float>({index_of_bbox, 2}, width_));
RETURN_IF_NOT_OK(bbox_tensor->SetItemAt<bbox_float>({index_of_bbox, 3}, height_));
return Status::OK();
}
Status BoundingBox::GetListOfBoundingBoxes(const TensorPtr &bbox_tensor,
std::vector<std::shared_ptr<BoundingBox>> *bbox_out) {
dsize_t num_of_boxes = bbox_tensor->shape()[0];
for (dsize_t i = 0; i < num_of_boxes; i++) {
std::shared_ptr<BoundingBox> bbox;
RETURN_IF_NOT_OK(ReadFromTensor(bbox_tensor, i, &bbox));
bbox_out->push_back(bbox);
}
return Status::OK();
}
Status BoundingBox::CreateTensorFromBoundingBoxList(const std::vector<std::shared_ptr<BoundingBox>> &bboxes,
TensorPtr *tensor_out) {
dsize_t num_of_boxes = bboxes.size();
std::vector<bbox_float> bboxes_for_tensor;
for (dsize_t i = 0; i < num_of_boxes; i++) {
bbox_float b_data[kNumOfCols] = {bboxes[i]->x(), bboxes[i]->y(), bboxes[i]->width(), bboxes[i]->height()};
bboxes_for_tensor.insert(bboxes_for_tensor.end(), b_data, b_data + kNumOfCols);
}
RETURN_IF_NOT_OK(Tensor::CreateFromVector(bboxes_for_tensor, TensorShape{num_of_boxes, kNumOfCols}, tensor_out));
return Status::OK();
}
Status BoundingBox::PadBBoxes(const TensorPtr *bbox_list, size_t bbox_count, int32_t pad_top, int32_t pad_left) {
for (dsize_t i = 0; i < bbox_count; i++) {
std::shared_ptr<BoundingBox> bbox;
RETURN_IF_NOT_OK(ReadFromTensor(*bbox_list, i, &bbox));
bbox->SetX(bbox->x() + pad_left);
bbox->SetY(bbox->y() + pad_top);
RETURN_IF_NOT_OK(bbox->WriteToTensor(*bbox_list, i));
}
return Status::OK();
}
Status BoundingBox::UpdateBBoxesForCrop(TensorPtr *bbox_list, size_t *bbox_count, int32_t CB_Xmin, int32_t CB_Ymin,
int32_t CB_Xmax, int32_t CB_Ymax) {
// PASS LIST, COUNT OF BOUNDING BOXES
// Also PAss X/Y Min/Max of image cropped region - normally obtained from 'GetCropBox' functions
std::vector<dsize_t> correct_ind;
std::vector<bbox_float> copyVals;
bool retFlag = false; // true unless overlap found
dsize_t bboxDim = (*bbox_list)->shape()[1];
for (dsize_t i = 0; i < *bbox_count; i++) {
std::shared_ptr<BoundingBox> bbox;
RETURN_IF_NOT_OK(ReadFromTensor(*bbox_list, i, &bbox));
bbox_float bb_Xmax = bbox->x() + bbox->width();
bbox_float bb_Ymax = bbox->y() + bbox->height();
// check for image / BB overlap
if (((bbox->x() > CB_Xmax) || (bbox->y() > CB_Ymax)) || ((bb_Xmax < CB_Xmin) || (bb_Ymax < CB_Ymin))) {
continue; // no overlap found
}
// Update this bbox and select it to move to the final output tensor
correct_ind.push_back(i);
// adjust BBox corners by bringing into new CropBox if beyond
// Also reseting/adjusting for boxes to lie within CropBox instead of Image - subtract CropBox Xmin/YMin
bbox_float bb_Xmin = bbox->x() - std::min(static_cast<bbox_float>(0.0), (bbox->x() - CB_Xmin)) - CB_Xmin;
bbox_float bb_Ymin = bbox->y() - std::min(static_cast<bbox_float>(0.0), (bbox->y() - CB_Ymin)) - CB_Ymin;
bb_Xmax = bb_Xmax - std::max(static_cast<bbox_float>(0.0), (bb_Xmax - CB_Xmax)) - CB_Xmin;
bb_Ymax = bb_Ymax - std::max(static_cast<bbox_float>(0.0), (bb_Ymax - CB_Ymax)) - CB_Ymin;
// bound check for float values
bb_Xmin = std::max(bb_Xmin, static_cast<bbox_float>(0));
bb_Ymin = std::max(bb_Ymin, static_cast<bbox_float>(0));
bb_Xmax = std::min(bb_Xmax, static_cast<bbox_float>(CB_Xmax - CB_Xmin)); // find max value relative to new image
bb_Ymax = std::min(bb_Ymax, static_cast<bbox_float>(CB_Ymax - CB_Ymin));
// reset min values and calculate width/height from Box corners
bbox->SetX(bb_Xmin);
bbox->SetY(bb_Ymin);
bbox->SetWidth(bb_Xmax - bb_Xmin);
bbox->SetHeight(bb_Ymax - bb_Ymin);
RETURN_IF_NOT_OK(bbox->WriteToTensor(*bbox_list, i));
}
// create new tensor and copy over bboxes still valid to the image
// bboxes outside of new cropped region are ignored - empty tensor returned in case of none
*bbox_count = correct_ind.size();
bbox_float temp = 0.0;
for (auto slice : correct_ind) { // for every index in the loop
for (dsize_t ix = 0; ix < bboxDim; ix++) {
RETURN_IF_NOT_OK((*bbox_list)->GetItemAt<bbox_float>(&temp, {slice, ix}));
copyVals.push_back(temp);
}
}
std::shared_ptr<Tensor> retV;
RETURN_IF_NOT_OK(
Tensor::CreateFromVector(copyVals, TensorShape({static_cast<dsize_t>(*bbox_count), bboxDim}), &retV));
(*bbox_list) = retV; // reset pointer
return Status::OK();
}
Status BoundingBox::UpdateBBoxesForResize(const TensorPtr &bbox_list, size_t bbox_count, int32_t target_width,
int32_t target_height, int32_t orig_width, int32_t orig_height) {
// cast to float to preserve fractional
bbox_float W_aspRatio = (target_width * 1.0) / (orig_width * 1.0);
bbox_float H_aspRatio = (target_height * 1.0) / (orig_height * 1.0);
for (dsize_t i = 0; i < bbox_count; i++) {
// for each bounding box
std::shared_ptr<BoundingBox> bbox;
RETURN_IF_NOT_OK(ReadFromTensor(bbox_list, i, &bbox));
// update positions and widths
bbox->SetX(bbox->x() * W_aspRatio);
bbox->SetY(bbox->y() * H_aspRatio);
bbox->SetWidth(bbox->width() * W_aspRatio);
bbox->SetHeight(bbox->height() * H_aspRatio);
// reset bounding box values
bbox->WriteToTensor(bbox_list, i);
}
return Status::OK();
}
} // namespace dataset
} // namespace mindspore

137
mindspore/ccsrc/minddata/dataset/kernels/image/bounding_box.h Normal file
View File

@ -0,0 +1,137 @@
/**
* Copyright 2020 Huawei Technologies Co., Ltd
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_BOUNDING_BOX_H_
#define MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_BOUNDING_BOX_H_
#include <memory>
#include <vector>
#include <string>
#include "minddata/dataset/core/tensor.h"
#include "minddata/dataset/core/tensor_row.h"
#include "minddata/dataset/util/status.h"
namespace mindspore {
namespace dataset {
class BoundingBox {
public:
typedef float_t bbox_float;
/// \brief Constructor for BoundingBox
/// \param[in] x horizontal axis coordinate of bounding box
/// \param[in] y vertical axis coordinate of bounding box
/// \param[in] width width of bounding box on horizontal axis
/// \param[in] height height of bounding box on vertical axis
BoundingBox(bbox_float x, bbox_float y, bbox_float width, bbox_float height);
~BoundingBox() = default;
/// \brief Provide stream operator for displaying a bounding box.
friend std::ostream &operator<<(std::ostream &out, const BoundingBox &bbox) {
out << "Bounding Box with (X,Y,W,H): (" << bbox.x_ << "," << bbox.y_ << "," << bbox.width_ << "," << bbox.height_
<< ")";
return out;
}
/// Getters
bbox_float x() { return x_; }
bbox_float y() { return y_; }
bbox_float width() { return width_; }
bbox_float height() { return height_; }
/// Setters
void SetX(bbox_float x) { x_ = x; }
void SetY(bbox_float y) { y_ = y; }
void SetWidth(bbox_float w) { width_ = w; }
void SetHeight(bbox_float h) { height_ = h; }
/// \brief Set the bounding box data to bbox at a certain index in the tensor.
/// \param[in] bbox_tensor tensor containing a list of bounding boxes of shape (m, n) where n >= 4
/// and first 4 items of each row are x,y,w,h of the bounding box
/// \param[in] index_of_bbox index of bounding box to set to tensor
/// \returns Status status of bounding box set
Status WriteToTensor(const TensorPtr &bbox_tensor, dsize_t index_of_bbox = 0);
/// \brief Create a bounding box object from an item at a certain index in a tensor.
/// \param[in] bbox_tensor tensor containing a list of bounding boxes of shape (m, n) where n >= 4
/// and first 4 items of each row are x,y,w,h of the bounding box
/// \param[in] index_of_bbox index of bounding box to fetch from the tensor
/// \param[out] bbox_out output bounding box
/// \returns Status status of bounding box fetch
static Status ReadFromTensor(const TensorPtr &bbox_tensor, dsize_t index_of_bbox,
std::shared_ptr<BoundingBox> *bbox_out);
/// \brief Validate a list of bounding boxes with respect to an image.
/// \param[in] image_and_bbox tensor containing a list of bounding boxes of shape (m, n) where n >= 4
/// and first 4 items of each row are x,y,w,h of the bounding box and an image of shape (H, W, C) or (H, W)
/// \returns Status status of bounding box fetch
static Status ValidateBoundingBoxes(const TensorRow &image_and_bbox);
/// \brief Get a list of bounding boxes from a tensor.
/// \param[in] bbox_tensor tensor containing a list of bounding boxes of shape (m, n) where n >= 4
/// and first 4 items of each row are x,y,w,h of the bounding box
/// \param[out] bbox_out output vector of bounding boxes
/// \returns Status status of bounding box list fetch
static Status GetListOfBoundingBoxes(const TensorPtr &bbox_tensor,
std::vector<std::shared_ptr<BoundingBox>> *bbox_out);
/// \brief Creates a tensor from a list of bounding boxes.
/// \param[in] bboxes list of bounding boxes
/// \param[out] tensor_out output tensor
/// \returns Status status of tensor creation
static Status CreateTensorFromBoundingBoxList(const std::vector<std::shared_ptr<BoundingBox>> &bboxes,
TensorPtr *tensor_out);
/// \brief Updates bounding boxes with required Top and Left padding
/// \note Top and Left padding amounts required to adjust bboxs min X,Y values according to padding 'push'
/// Top/Left since images 0,0 coordinate is taken from top left
/// \param bboxList: A tensor contaning bounding box tensors
/// \param bboxCount: total Number of bounding boxes - required within caller function to run update loop
/// \param pad_top: Total amount of padding applied to image top
/// \param pad_left: Total amount of padding applied to image left side
static Status PadBBoxes(const TensorPtr *bbox_list, size_t bbox_count, int32_t pad_top, int32_t pad_left);
/// \brief Updates and checks bounding boxes for new cropped region of image
/// \param bbox_list: A tensor contaning bounding box tensors
/// \param bbox_count: total Number of bounding boxes - required within caller function to run update loop
/// \param CB_Xmin: Image's CropBox Xmin coordinate
/// \param CB_Xmin: Image's CropBox Ymin coordinate
/// \param CB_Xmax: Image's CropBox Xmax coordinate - (Xmin + width)
/// \param CB_Xmax: Image's CropBox Ymax coordinate - (Ymin + height)
static Status UpdateBBoxesForCrop(TensorPtr *bbox_list, size_t *bbox_count, int32_t CB_Xmin, int32_t CB_Ymin,
int32_t CB_Xmax, int32_t CB_Ymax);
/// \brief Updates bounding boxes for an Image Resize Operation - Takes in set of valid BBoxes
/// For e.g those that remain after a crop
/// \param bbox_list: A tensor contaning bounding box tensors
/// \param bbox_count: total Number of bounding boxes - required within caller function to run update loop
/// \param target_width: required width of image post resize
/// \param target_height: required height of image post resize
/// \param orig_width: current width of image pre resize
/// \param orig_height: current height of image pre resize
static Status UpdateBBoxesForResize(const TensorPtr &bbox_list, size_t bbox_count, int32_t target_width,
int32_t target_height, int32_t orig_width, int32_t orig_height);
private:
bbox_float x_;
bbox_float y_;
bbox_float width_;
bbox_float height_;
};
} // namespace dataset
} // namespace mindspore
#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_KERNELS_IMAGE_BOUNDING_BOX_H_

27
mindspore/ccsrc/minddata/dataset/kernels/image/bounding_box_augment_op.cc
View File

@ -17,6 +17,7 @@
#include <vector>
#include <utility>
#include "minddata/dataset/kernels/image/bounding_box_augment_op.h"
#include "minddata/dataset/kernels/image/bounding_box.h"
#include "minddata/dataset/kernels/image/resize_op.h"
#include "minddata/dataset/kernels/image/image_utils.h"
#include "minddata/dataset/core/cv_tensor.h"
@ -32,7 +33,7 @@ BoundingBoxAugmentOp::BoundingBoxAugmentOp(std::shared_ptr<TensorOp> transform,
Status BoundingBoxAugmentOp::Compute(const TensorRow &input, TensorRow *output) {
IO_CHECK_VECTOR(input, output);
BOUNDING_BOX_CHECK(input); // check if bounding boxes are valid
RETURN_IF_NOT_OK(BoundingBox::ValidateBoundingBoxes(input));
uint32_t num_of_boxes = input[1]->shape()[0];
std::shared_ptr<Tensor> crop_out;
std::shared_ptr<Tensor> res_out;
@ -40,31 +41,25 @@ Status BoundingBoxAugmentOp::Compute(const TensorRow &input, TensorRow *output)
for (uint32_t i = 0; i < num_of_boxes; i++) {
// using a uniform distribution to ensure op happens with probability ratio_
if (uniform_(rnd_) < ratio_) {
float min_x = 0;
float min_y = 0;
float b_w = 0;
float b_h = 0;
// get the required items
RETURN_IF_NOT_OK(input[1]->GetItemAt<float>(&min_x, {i, 0}));
RETURN_IF_NOT_OK(input[1]->GetItemAt<float>(&min_y, {i, 1}));
RETURN_IF_NOT_OK(input[1]->GetItemAt<float>(&b_w, {i, 2}));
RETURN_IF_NOT_OK(input[1]->GetItemAt<float>(&b_h, {i, 3}));
RETURN_IF_NOT_OK(Crop(input_restore, &crop_out, static_cast<int>(min_x), static_cast<int>(min_y),
static_cast<int>(b_w), static_cast<int>(b_h)));
std::shared_ptr<BoundingBox> bbox;
RETURN_IF_NOT_OK(BoundingBox::ReadFromTensor(input[1], i, &bbox));
RETURN_IF_NOT_OK(Crop(input_restore, &crop_out, static_cast<int>(bbox->x()), static_cast<int>(bbox->y()),
static_cast<int>(bbox->width()), static_cast<int>(bbox->height())));
// transform the cropped bbox region
RETURN_IF_NOT_OK(transform_->Compute(crop_out, &res_out));
// place the transformed region back in the restored input
std::shared_ptr<CVTensor> res_img = CVTensor::AsCVTensor(res_out);
// check if transformed crop is out of bounds of the box
if (res_img->mat().cols > b_w || res_img->mat().rows > b_h || res_img->mat().cols < b_w ||
res_img->mat().rows < b_h) {
if (res_img->mat().cols > bbox->width() || res_img->mat().rows > bbox->height() ||
res_img->mat().cols < bbox->width() || res_img->mat().rows < bbox->height()) {
// if so, resize to fit in the box
std::shared_ptr<TensorOp> resize_op =
std::make_shared<ResizeOp>(static_cast<int32_t>(b_h), static_cast<int32_t>(b_w));
std::make_shared<ResizeOp>(static_cast<int32_t>(bbox->height()), static_cast<int32_t>(bbox->width()));
RETURN_IF_NOT_OK(resize_op->Compute(std::static_pointer_cast<Tensor>(res_img), &res_out));
res_img = CVTensor::AsCVTensor(res_out);
}
res_img->mat().copyTo(input_restore->mat()(cv::Rect(min_x, min_y, res_img->mat().cols, res_img->mat().rows)));
res_img->mat().copyTo(
input_restore->mat()(cv::Rect(bbox->x(), bbox->y(), res_img->mat().cols, res_img->mat().rows)));
}
}
(*output).push_back(std::move(std::static_pointer_cast<Tensor>(input_restore)));

97
mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.cc
View File

@ -950,103 +950,6 @@ Status RgbaToBgr(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *
RETURN_STATUS_UNEXPECTED("Unexpected error in RgbaToBgr.");
}
}
// -------- BBOX OPERATIONS -------- //
Status UpdateBBoxesForCrop(std::shared_ptr<Tensor> *bboxList, size_t *bboxCount, int CB_Xmin, int CB_Ymin, int CB_Xmax,
int CB_Ymax) {
// PASS LIST, COUNT OF BOUNDING BOXES
// Also PAss X/Y Min/Max of image cropped region - normally obtained from 'GetCropBox' functions
float bb_Xmin = 0.0, bb_Ymin = 0.0, bb_Xmax = 0.0, bb_Ymax = 0.0;
std::vector<int> correct_ind;
std::vector<float> copyVals;
dsize_t bboxDim = (*bboxList)->shape()[1];
bool retFlag = false; // true unless overlap found
for (int i = 0; i < *bboxCount; i++) {
RETURN_IF_NOT_OK((*bboxList)->GetItemAt<float>(&bb_Xmin, {i, 0}));
RETURN_IF_NOT_OK((*bboxList)->GetItemAt<float>(&bb_Ymin, {i, 1}));
RETURN_IF_NOT_OK((*bboxList)->GetItemAt<float>(&bb_Xmax, {i, 2}));
RETURN_IF_NOT_OK((*bboxList)->GetItemAt<float>(&bb_Ymax, {i, 3}));
bb_Xmax = bb_Xmin + bb_Xmax;
bb_Ymax = bb_Ymin + bb_Ymax;
// check for image / BB overlap
if (((bb_Xmin > CB_Xmax) || (bb_Ymin > CB_Ymax)) || ((bb_Xmax < CB_Xmin) || (bb_Ymax < CB_Ymin))) {
continue; // no overlap found
}
// Update this bbox and select it to move to the final output tensor
correct_ind.push_back(i);
// adjust BBox corners by bringing into new CropBox if beyond
// Also reseting/adjusting for boxes to lie within CropBox instead of Image - subtract CropBox Xmin/YMin
bb_Xmin = bb_Xmin - std::min(static_cast<float>(0.0), (bb_Xmin - CB_Xmin)) - CB_Xmin;
bb_Xmax = bb_Xmax - std::max(static_cast<float>(0.0), (bb_Xmax - CB_Xmax)) - CB_Xmin;
bb_Ymin = bb_Ymin - std::min(static_cast<float>(0.0), (bb_Ymin - CB_Ymin)) - CB_Ymin;
bb_Ymax = bb_Ymax - std::max(static_cast<float>(0.0), (bb_Ymax - CB_Ymax)) - CB_Ymin;
// bound check for float values
bb_Xmin = std::max(bb_Xmin, static_cast<float>(0));
bb_Ymin = std::max(bb_Ymin, static_cast<float>(0));
bb_Xmax = std::min(bb_Xmax, static_cast<float>(CB_Xmax - CB_Xmin)); // find max value relative to new image
bb_Ymax = std::min(bb_Ymax, static_cast<float>(CB_Ymax - CB_Ymin));
// reset min values and calculate width/height from Box corners
RETURN_IF_NOT_OK((*bboxList)->SetItemAt({i, 0}, bb_Xmin));
RETURN_IF_NOT_OK((*bboxList)->SetItemAt({i, 1}, bb_Ymin));
RETURN_IF_NOT_OK((*bboxList)->SetItemAt({i, 2}, bb_Xmax - bb_Xmin));
RETURN_IF_NOT_OK((*bboxList)->SetItemAt({i, 3}, bb_Ymax - bb_Ymin));
}
// create new tensor and copy over bboxes still valid to the image
// bboxes outside of new cropped region are ignored - empty tensor returned in case of none
*bboxCount = correct_ind.size();
float temp = 0.0;
for (auto slice : correct_ind) { // for every index in the loop
for (int ix = 0; ix < bboxDim; ix++) {
RETURN_IF_NOT_OK((*bboxList)->GetItemAt<float>(&temp, {slice, ix}));
copyVals.push_back(temp);
}
}
std::shared_ptr<Tensor> retV;
RETURN_IF_NOT_OK(Tensor::CreateFromVector(copyVals, TensorShape({static_cast<dsize_t>(*bboxCount), bboxDim}), &retV));
(*bboxList) = retV; // reset pointer
return Status::OK();
}
Status PadBBoxes(const std::shared_ptr<Tensor> *bboxList, const size_t &bboxCount, int32_t pad_top, int32_t pad_left) {
for (int i = 0; i < bboxCount; i++) {
float xMin = 0.0, yMin = 0.0;
RETURN_IF_NOT_OK((*bboxList)->GetItemAt<float>(&xMin, {i, 0}));
RETURN_IF_NOT_OK((*bboxList)->GetItemAt<float>(&yMin, {i, 1}));
xMin += pad_left;
yMin += pad_top;
RETURN_IF_NOT_OK((*bboxList)->SetItemAt({i, 0}, xMin));
RETURN_IF_NOT_OK((*bboxList)->SetItemAt({i, 1}, yMin));
}
return Status::OK();
}
Status UpdateBBoxesForResize(const std::shared_ptr<Tensor> &bboxList, const size_t &bboxCount, int32_t target_width_,
int32_t target_height_, int orig_width, int orig_height) {
float bb_Xmin = 0, bb_Ymin = 0, bb_Xwidth = 0, bb_Ywidth = 0;
// cast to float to preserve fractional
float W_aspRatio = (target_width_ * 1.0) / (orig_width * 1.0);
float H_aspRatio = (target_height_ * 1.0) / (orig_height * 1.0);
for (int i = 0; i < bboxCount; i++) {
// for each bounding box
RETURN_IF_NOT_OK(bboxList->GetItemAt<float>(&bb_Xmin, {i, 0}));
RETURN_IF_NOT_OK(bboxList->GetItemAt<float>(&bb_Ymin, {i, 1}));
RETURN_IF_NOT_OK(bboxList->GetItemAt<float>(&bb_Xwidth, {i, 2}));
RETURN_IF_NOT_OK(bboxList->GetItemAt<float>(&bb_Ywidth, {i, 3}));
// update positions and widths
bb_Xmin = bb_Xmin * W_aspRatio;
bb_Ymin = bb_Ymin * H_aspRatio;
bb_Xwidth = bb_Xwidth * W_aspRatio;
bb_Ywidth = bb_Ywidth * H_aspRatio;
// reset bounding box values
RETURN_IF_NOT_OK(bboxList->SetItemAt({i, 0}, bb_Xmin));
RETURN_IF_NOT_OK(bboxList->SetItemAt({i, 1}, bb_Ymin));
RETURN_IF_NOT_OK(bboxList->SetItemAt({i, 2}, bb_Xwidth));
RETURN_IF_NOT_OK(bboxList->SetItemAt({i, 3}, bb_Ywidth));
}
return Status::OK();
}
Status GetJpegImageInfo(const std::shared_ptr<Tensor> &input, int *img_width, int *img_height) {
struct jpeg_decompress_struct cinfo {};

32
mindspore/ccsrc/minddata/dataset/kernels/image/image_utils.h
View File

@ -262,38 +262,6 @@ Status RgbaToRgb(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *
/// \return Status code
Status RgbaToBgr(const std::shared_ptr<Tensor> &input, std::shared_ptr<Tensor> *output);
/// -------- BBOX OPERATIONS -------- ///
/// \brief Updates and checks bounding boxes for new cropped region of image
/// \param bboxList: A tensor contaning bounding box tensors
/// \param bboxCount: total Number of bounding boxes - required within caller function to run update loop
/// \param CB_Xmin: Image's CropBox Xmin coordinate
/// \param CB_Xmin: Image's CropBox Ymin coordinate
/// \param CB_Xmax: Image's CropBox Xmax coordinate - (Xmin + width)
/// \param CB_Xmax: Image's CropBox Ymax coordinate - (Ymin + height)
Status UpdateBBoxesForCrop(std::shared_ptr<Tensor> *bboxList, size_t *bboxCount, int CB_Xmin, int CB_Ymin, int CB_Xmax,
int CB_Ymax);
/// \brief Updates bounding boxes with required Top and Left padding
/// \note Top and Left padding amounts required to adjust bboxs min X,Y values according to padding 'push'
/// Top/Left since images 0,0 coordinate is taken from top left
/// \param bboxList: A tensor contaning bounding box tensors
/// \param bboxCount: total Number of bounding boxes - required within caller function to run update loop
/// \param pad_top: Total amount of padding applied to image top
/// \param pad_left: Total amount of padding applied to image left side
Status PadBBoxes(const std::shared_ptr<Tensor> *bboxList, const size_t &bboxCount, int32_t pad_top, int32_t pad_left);
/// \brief Updates bounding boxes for an Image Resize Operation - Takes in set of valid BBoxes
/// For e.g those that remain after a crop
/// \param bboxList: A tensor contaning bounding box tensors
/// \param bboxCount: total Number of bounding boxes - required within caller function to run update loop
/// \param bboxList: A tensor contaning bounding box tensors
/// \param target_width_: required width of image post resize
/// \param target_width_: required height of image post resize
/// \param orig_width: current width of image pre resize
/// \param orig_height: current height of image pre resize
Status UpdateBBoxesForResize(const std::shared_ptr<Tensor> &bboxList, const size_t &bboxCount, int32_t target_width_,
int32_t target_height_, int orig_width, int orig_height);
/// \brief Get jpeg image width and height
/// \param input: CVTensor containing the not decoded image 1D bytes
/// \param img_width: the jpeg image width

11
mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_and_resize_with_bbox_op.cc
View File

@ -19,6 +19,7 @@
#include "minddata/dataset/util/random.h"
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/kernels/image/bounding_box.h"
#include "minddata/dataset/kernels/image/image_utils.h"
#include "minddata/dataset/kernels/image/random_crop_and_resize_with_bbox_op.h"
@ -27,8 +28,8 @@ namespace dataset {
Status RandomCropAndResizeWithBBoxOp::Compute(const TensorRow &input, TensorRow *output) {
IO_CHECK_VECTOR(input, output);
BOUNDING_BOX_CHECK(input);
CHECK_FAIL_RETURN_UNEXPECTED(input[0]->shape().Size() >= 2, "The shape of input is abnormal");
RETURN_IF_NOT_OK(BoundingBox::ValidateBoundingBoxes(input));
CHECK_FAIL_RETURN_UNEXPECTED(input[0]->shape().Size() >= 2, "The shape of input is not >= 2");
output->resize(2);
(*output)[1] = std::move(input[1]); // move boxes over to output
@ -46,12 +47,12 @@ Status RandomCropAndResizeWithBBoxOp::Compute(const TensorRow &input, TensorRow
int maxX = x + crop_width; // max dims of selected CropBox on image
int maxY = y + crop_height;
RETURN_IF_NOT_OK(UpdateBBoxesForCrop(&(*output)[1], &bboxCount, x, y, maxX, maxY)); // IMAGE_UTIL
RETURN_IF_NOT_OK(BoundingBox::UpdateBBoxesForCrop(&(*output)[1], &bboxCount, x, y, maxX, maxY)); // IMAGE_UTIL
RETURN_IF_NOT_OK(CropAndResize(input[0], &(*output)[0], x, y, crop_height, crop_width, target_height_, target_width_,
interpolation_));
RETURN_IF_NOT_OK(
UpdateBBoxesForResize((*output)[1], bboxCount, target_width_, target_height_, crop_width, crop_height));
RETURN_IF_NOT_OK(BoundingBox::UpdateBBoxesForResize((*output)[1], bboxCount, target_width_, target_height_,
crop_width, crop_height));
return Status::OK();
}
} // namespace dataset

7
mindspore/ccsrc/minddata/dataset/kernels/image/random_crop_with_bbox_op.cc
View File

@ -19,6 +19,7 @@
#include <utility>
#include "minddata/dataset/kernels/image/random_crop_with_bbox_op.h"
#include "minddata/dataset/kernels/image/bounding_box.h"
#include "minddata/dataset/kernels/image/image_utils.h"
#include "minddata/dataset/util/random.h"
#include "minddata/dataset/util/status.h"
@ -27,7 +28,7 @@ namespace mindspore {
namespace dataset {
Status RandomCropWithBBoxOp::Compute(const TensorRow &input, TensorRow *output) {
IO_CHECK_VECTOR(input, output);
BOUNDING_BOX_CHECK(input);
RETURN_IF_NOT_OK(BoundingBox::ValidateBoundingBoxes(input));
std::shared_ptr<Tensor> pad_image;
int32_t t_pad_top, t_pad_bottom, t_pad_left, t_pad_right;
@ -46,7 +47,7 @@ Status RandomCropWithBBoxOp::Compute(const TensorRow &input, TensorRow *output)
// update bounding boxes with new values based on relevant image padding
if (t_pad_left || t_pad_bottom) {
RETURN_IF_NOT_OK(PadBBoxes(&(*output)[1], boxCount, t_pad_left, t_pad_top));
RETURN_IF_NOT_OK(BoundingBox::PadBBoxes(&(*output)[1], boxCount, t_pad_left, t_pad_top));
}
if (!crop_further) {
// no further cropping required
@ -59,7 +60,7 @@ Status RandomCropWithBBoxOp::Compute(const TensorRow &input, TensorRow *output)
RandomCropOp::GenRandomXY(&x, &y, padded_image_w, padded_image_h);
int maxX = x + RandomCropOp::crop_width_; // max dims of selected CropBox on image
int maxY = y + RandomCropOp::crop_height_;
RETURN_IF_NOT_OK(UpdateBBoxesForCrop(&(*output)[1], &boxCount, x, y, maxX, maxY));
RETURN_IF_NOT_OK(BoundingBox::UpdateBBoxesForCrop(&(*output)[1], &boxCount, x, y, maxX, maxY));
return Crop(pad_image, &(*output)[0], x, y, RandomCropOp::crop_width_, RandomCropOp::crop_height_);
}
} // namespace dataset

20
mindspore/ccsrc/minddata/dataset/kernels/image/random_horizontal_flip_with_bbox_op.cc
View File

@ -15,6 +15,7 @@
*/
#include <utility>
#include "minddata/dataset/kernels/image/random_horizontal_flip_with_bbox_op.h"
#include "minddata/dataset/kernels/image/bounding_box.h"
#include "minddata/dataset/kernels/image/image_utils.h"
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/core/cv_tensor.h"
@ -25,22 +26,21 @@ const float RandomHorizontalFlipWithBBoxOp::kDefProbability = 0.5;
Status RandomHorizontalFlipWithBBoxOp::Compute(const TensorRow &input, TensorRow *output) {
IO_CHECK_VECTOR(input, output);
BOUNDING_BOX_CHECK(input);
RETURN_IF_NOT_OK(BoundingBox::ValidateBoundingBoxes(input));
if (distribution_(rnd_)) {
// To test bounding boxes algorithm, create random bboxes from image dims
size_t num_of_boxes = input[1]->shape()[0]; // set to give number of bboxes
float img_center = (input[0]->shape()[1] / 2.); // get the center of the image
for (int i = 0; i < num_of_boxes; i++) {
float b_w = 0; // bounding box width
float min_x = 0;
// get the required items
RETURN_IF_NOT_OK(input[1]->GetItemAt<float>(&min_x, {i, 0}));
RETURN_IF_NOT_OK(input[1]->GetItemAt<float>(&b_w, {i, 2}));
std::shared_ptr<BoundingBox> bbox;
RETURN_IF_NOT_OK(BoundingBox::ReadFromTensor(input[1], i, &bbox));
// do the flip
float diff = img_center - min_x; // get distance from min_x to center
float refl_min_x = diff + img_center; // get reflection of min_x
float new_min_x = refl_min_x - b_w; // subtract from the reflected min_x to get the new one
RETURN_IF_NOT_OK(input[1]->SetItemAt<float>({i, 0}, new_min_x));
BoundingBox::bbox_float diff = img_center - bbox->x(); // get distance from min_x to center
BoundingBox::bbox_float refl_min_x = diff + img_center; // get reflection of min_x
BoundingBox::bbox_float new_min_x =
refl_min_x - bbox->width(); // subtract from the reflected min_x to get the new one
bbox->SetX(new_min_x);
RETURN_IF_NOT_OK(bbox->WriteToTensor(input[1], i));
}
(*output).resize(2);
// move input to output pointer of bounding boxes

14
mindspore/ccsrc/minddata/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc
View File

@ -17,6 +17,7 @@
#include <utility>
#include "minddata/dataset/util/status.h"
#include "minddata/dataset/kernels/image/bounding_box.h"
#include "minddata/dataset/kernels/image/image_utils.h"
#include "minddata/dataset/kernels/image/random_vertical_flip_with_bbox_op.h"
@ -25,7 +26,7 @@ namespace dataset {
const float RandomVerticalFlipWithBBoxOp::kDefProbability = 0.5;
Status RandomVerticalFlipWithBBoxOp::Compute(const TensorRow &input, TensorRow *output) {
IO_CHECK_VECTOR(input, output);
BOUNDING_BOX_CHECK(input);
RETURN_IF_NOT_OK(BoundingBox::ValidateBoundingBoxes(input));
if (distribution_(rnd_)) {
dsize_t imHeight = input[0]->shape()[0];
@ -34,14 +35,13 @@ Status RandomVerticalFlipWithBBoxOp::Compute(const TensorRow &input, TensorRow *
// one time allocation -> updated in the loop
// type defined based on VOC test dataset
for (int i = 0; i < boxCount; i++) {
float boxCorner_y = 0.0, boxHeight = 0.0;
float newBoxCorner_y = 0.0;
RETURN_IF_NOT_OK(input[1]->GetItemAt<float>(&boxCorner_y, {i, 1})); // get min y of bbox
RETURN_IF_NOT_OK(input[1]->GetItemAt<float>(&boxHeight, {i, 3})); // get height of bbox
std::shared_ptr<BoundingBox> bbox;
RETURN_IF_NOT_OK(BoundingBox::ReadFromTensor(input[1], i, &bbox));
// subtract (curCorner + height) from (max) for new Corner position
newBoxCorner_y = (imHeight - 1.0) - ((boxCorner_y + boxHeight) - 1.0);
RETURN_IF_NOT_OK(input[1]->SetItemAt({i, 1}, newBoxCorner_y));
BoundingBox::bbox_float newBoxCorner_y = (imHeight - 1.0) - ((bbox->y() + bbox->height()) - 1.0);
bbox->SetY(newBoxCorner_y);
RETURN_IF_NOT_OK(bbox->WriteToTensor(input[1], i));
}
output->resize(2);

5
mindspore/ccsrc/minddata/dataset/kernels/image/resize_with_bbox_op.cc
View File

@ -18,6 +18,7 @@
#include <utility>
#include <memory>
#include "minddata/dataset/kernels/image/resize_op.h"
#include "minddata/dataset/kernels/image/bounding_box.h"
#include "minddata/dataset/kernels/image/image_utils.h"
#include "minddata/dataset/core/cv_tensor.h"
#include "minddata/dataset/core/tensor.h"
@ -29,7 +30,7 @@ namespace dataset {
Status ResizeWithBBoxOp::Compute(const TensorRow &input, TensorRow *output) {
IO_CHECK_VECTOR(input, output);
BOUNDING_BOX_CHECK(input);
RETURN_IF_NOT_OK(BoundingBox::ValidateBoundingBoxes(input));
int32_t input_h = input[0]->shape()[0];
int32_t input_w = input[0]->shape()[1];
@ -45,7 +46,7 @@ Status ResizeWithBBoxOp::Compute(const TensorRow &input, TensorRow *output) {
int32_t output_w = (*output)[0]->shape()[1]; // output width if ResizeWithBBox
size_t bboxCount = input[1]->shape()[0]; // number of rows in bbox tensor
RETURN_IF_NOT_OK(UpdateBBoxesForResize((*output)[1], bboxCount, output_w, output_h, input_w, input_h));
RETURN_IF_NOT_OK(BoundingBox::UpdateBBoxesForResize((*output)[1], bboxCount, output_w, output_h, input_w, input_h));
return Status::OK();
}
} // namespace dataset

40
mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h
View File

@ -43,46 +43,6 @@
} \
} while (false)
#define BOUNDING_BOX_CHECK(input) \
do { \
if (input.size() != 2) { \
return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__, \
"Requires Image and Bounding Boxes, likely missed bounding boxes."); \
} \
if (input[1]->shape().Size() < 2) { \
return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__, \
"Bounding boxes shape should have at least two dimensions."); \
} \
uint32_t num_of_features = input[1]->shape()[1]; \
if (num_of_features < 4) { \
return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__, \
"Bounding boxes should be have at least 4 features."); \
} \
uint32_t num_of_boxes = input[1]->shape()[0]; \
uint32_t img_h = input[0]->shape()[0]; \
uint32_t img_w = input[0]->shape()[1]; \
for (uint32_t i = 0; i < num_of_boxes; i++) { \
float min_x = 0.0, min_y = 0.0, b_w = 0.0, b_h = 0.0; \
bool passing_data_fetch = true; \
passing_data_fetch &= input[1]->GetItemAt<float>(&min_x, {i, 0}).IsOk(); \
passing_data_fetch &= input[1]->GetItemAt<float>(&min_y, {i, 1}).IsOk(); \
passing_data_fetch &= input[1]->GetItemAt<float>(&b_w, {i, 2}).IsOk(); \
passing_data_fetch &= input[1]->GetItemAt<float>(&b_h, {i, 3}).IsOk(); \
if (!passing_data_fetch) { \
return Status(StatusCode::kUnexpectedError, __LINE__, __FILE__, \
"Fetching BBox values failed in BOUNDING_BOX_CHECK."); \
} \
if ((min_x + b_w > img_w) || (min_y + b_h > img_h)) { \
return Status(StatusCode::kBoundingBoxOutOfBounds, __LINE__, __FILE__, \
"At least one of the bounding boxes is out of bounds of the image."); \
} \
if (static_cast<int>(min_x) < 0 || static_cast<int>(min_y) < 0) { \
return Status(StatusCode::kBoundingBoxOutOfBounds, __LINE__, __FILE__, \
"At least one of the bounding boxes has negative min_x or min_y."); \
} \
} \
} while (false)
namespace mindspore {
namespace dataset {

20
tests/ut/python/dataset/test_random_posterize.py
View File

@ -15,11 +15,12 @@
"""
Testing RandomPosterize op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
from mindspore import log as logger
from util import visualize_list, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers
config_get_set_seed, config_get_set_num_parallel_workers, diff_mse
GENERATE_GOLDEN = False
@ -27,9 +28,10 @@ DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json"
def skip_test_random_posterize_op_c(plot=False, run_golden=True):
def test_random_posterize_op_c(plot=False, run_golden=False):
"""
Test RandomPosterize in C transformations
Test RandomPosterize in C transformations (uses assertion on mse as using md5 could have jpeg decoding
inconsistencies)
"""
logger.info("test_random_posterize_op_c")
@ -57,6 +59,12 @@ def skip_test_random_posterize_op_c(plot=False, run_golden=True):
image_posterize.append(image1)
image_original.append(image2)
# check mse as md5 can be inconsistent.
# mse = 2.9668956 is calculated from
# a thousand runs of diff_mse(np.array(image_original), np.array(image_posterize)) that all produced the same mse.
# allow for an error of 0.0000005
assert abs(2.9668956 - diff_mse(np.array(image_original), np.array(image_posterize))) <= 0.0000005
if run_golden:
# check results with md5 comparison
filename = "random_posterize_01_result_c.npz"
@ -70,7 +78,7 @@ def skip_test_random_posterize_op_c(plot=False, run_golden=True):
ds.config.set_num_parallel_workers(original_num_parallel_workers)
def skip_test_random_posterize_op_fixed_point_c(plot=False, run_golden=True):
def test_random_posterize_op_fixed_point_c(plot=False, run_golden=True):
"""
Test RandomPosterize in C transformations with fixed point
"""
@ -144,6 +152,6 @@ def test_random_posterize_exception_bit():
if __name__ == "__main__":
skip_test_random_posterize_op_c(plot=True)
skip_test_random_posterize_op_fixed_point_c(plot=True)
test_random_posterize_op_c(plot=False, run_golden=False)
test_random_posterize_op_fixed_point_c(plot=False)
test_random_posterize_exception_bit()