updated tests + added support for ToTensor C API for DataFrame

New API

fixed some spacing issues + reverted to pass by ref in 2 cases

lint fixes + comment fix for C API

updated test files + comments

improved comments

lint fix

mindspore/ccsrc/minddata/dataset/api/vision.cc

@@ -1136,11 +1136,14 @@ std::shared_ptr<TensorOperation> SwapRedBlue::Parse() { return std::make_shared<
 
 // ToTensor Transform Operation.
 struct ToTensor::Data {
-  explicit Data(std::string &output_type) : output_type_(output_type) {}
-  std::string output_type_;
+  explicit Data(std::string &output_type) : output_type_(DataType(output_type)) {}
+  explicit Data(DataType::Type output_type) : output_type_(output_type) {}
+  DataType output_type_{};
 };
 
+ToTensor::ToTensor() : data_(std::make_shared<Data>(DataType::Type::DE_FLOAT32)) {}
 ToTensor::ToTensor(std::string output_type) : data_(std::make_shared<Data>(output_type)) {}
+ToTensor::ToTensor(DataType::Type output_type) : data_(std::make_shared<Data>(output_type)) {}
 
 std::shared_ptr<TensorOperation> ToTensor::Parse() { return std::make_shared<ToTensorOperation>(data_->output_type_); }
 

mindspore/ccsrc/minddata/dataset/include/dataset/vision.h

@@ -28,6 +28,7 @@
 #include "include/dataset/constants.h"
 #include "include/dataset/transforms.h"
 #include "include/dataset/vision_lite.h"
+#include "minddata/dataset/kernels/data/data_utils.h"
 
 namespace mindspore {
 namespace dataset {
@@ -1600,7 +1601,8 @@ class MS_API SwapRedBlue final : public TensorTransform {
 class MS_API ToTensor final : public TensorTransform {
  public:
   /// \brief Constructor.
-  /// \param[in] output_type The type of the output tensor (default="float32").
+  /// \param[in] output_type The type of the output tensor of type mindspore::DataType or String
+  /// (default=mindspore::dataset::DataType::DE_FLOAT32).
   /// \par Example
   /// \code
   ///     /* Define operations */
@@ -1610,7 +1612,9 @@ class MS_API ToTensor final : public TensorTransform {
   ///     dataset = dataset->Map({ToTensor},  // operations
   ///                            {"image"});  // input columns
   /// \endcode
-  explicit ToTensor(std::string output_type = "float32");
+  ToTensor();
+  explicit ToTensor(std::string output_type);
+  explicit ToTensor(DataType::Type output_type);
 
   /// \brief Destructor.
   ~ToTensor() = default;

mindspore/ccsrc/minddata/dataset/kernels/image/to_tensor_op.h

@@ -27,8 +27,7 @@ namespace mindspore {
 namespace dataset {
 class ToTensorOp : public TensorOp {
  public:
-  explicit ToTensorOp(const DataType &output_type) : output_type_(output_type) {}
-
+  explicit ToTensorOp(const DataType::Type &output_type) : output_type_(output_type) {}
   explicit ToTensorOp(const std::string &output_type) { output_type_ = DataType(output_type); }
 
   ~ToTensorOp() override = default;

mindspore/ccsrc/minddata/dataset/kernels/ir/vision/to_tensor_ir.cc

@@ -28,6 +28,8 @@ ToTensorOperation::ToTensorOperation(const std::string &output_type) {
   output_type_ = temp_output_type;
 }
 
+ToTensorOperation::ToTensorOperation(const DataType &output_type) { output_type_ = output_type; }
+
 ToTensorOperation::~ToTensorOperation() = default;
 
 std::string ToTensorOperation::Name() const { return kToTensorOperation; }

mindspore/ccsrc/minddata/dataset/kernels/ir/vision/to_tensor_ir.h

@@ -34,6 +34,7 @@ constexpr char kToTensorOperation[] = "ToTensor";
 class ToTensorOperation : public TensorOperation {
  public:
   explicit ToTensorOperation(const std::string &output_type);
+  explicit ToTensorOperation(const DataType &output_type);
 
   ~ToTensorOperation();
 

tests/ut/cpp/dataset/c_api_vision_a_to_q_test.cc

@@ -1421,7 +1421,57 @@ TEST_F(MindDataTestPipeline, TestNormalizePadDefault) {
 
   // Create objects for the tensor ops
   auto normalizepad = std::make_shared<vision::NormalizePad>(std::vector<float>{121.0, 115.0, 100.0},
-                                                             std::vector<float>{70.0, 68.0, 71.0}, "float32");
+                                                             std::vector<float>{70.0, 68.0, 71.0});
+  // Note: No need to check for output after calling API class constructor
+
+  // Create a Map operation on ds
+  ds = ds->Map({normalizepad});
+  EXPECT_NE(ds, nullptr);
+
+  // Create an iterator over the result of the above dataset
+  // This will trigger the creation of the Execution Tree and launch it.
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  EXPECT_NE(iter, nullptr);
+
+  // Iterate the dataset and get each row
+  std::unordered_map<std::string, mindspore::MSTensor> row;
+  ASSERT_OK(iter->GetNextRow(&row));
+
+  uint64_t i = 0;
+  while (row.size() != 0) {
+    i++;
+    auto image = row["image"];
+    MS_LOG(INFO) << "Tensor image shape: " << image.Shape();
+    EXPECT_EQ(image.Shape()[2], 4);
+
+    ASSERT_OK(iter->GetNextRow(&row));
+  }
+
+  EXPECT_EQ(i, 20);
+
+  // Manually terminate the pipeline
+  iter->Stop();
+}
+
+/// Feature: NormalizePad op
+/// Description: Test NormalizePad op basic usage with default HWC parameter and float16 type
+/// Expectation: Output is equal to the expected output
+TEST_F(MindDataTestPipeline, TestNormalizePadDefaultFloat16AndHWCDefault) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestNormalizePadDefaultFloat16AndHWCDefault.";
+
+  // Create an ImageFolder Dataset
+  std::string folder_path = datasets_root_path_ + "/testPK/data/";
+  std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 10));
+  EXPECT_NE(ds, nullptr);
+
+  // Create a Repeat operation on ds
+  int32_t repeat_num = 2;
+  ds = ds->Repeat(repeat_num);
+  EXPECT_NE(ds, nullptr);
+
+  // Create objects for the tensor ops
+  auto normalizepad = std::make_shared<vision::NormalizePad>(std::vector<float>{121.0, 115.0, 100.0},
+                                                             std::vector<float>{70.0, 68.0, 71.0}, "float16");
   // Note: No need to check for output after calling API class constructor
 
   // Create a Map operation on ds

tests/ut/cpp/dataset/c_api_vision_r_to_z_test.cc

@@ -744,10 +744,10 @@ TEST_F(MindDataTestPipeline, TestToTensorOpDefault) {
 }
 
 /// Feature: ToTensor op
-/// Description: Test ToTensor op with float64 type
+/// Description: Test ToTensor op with float64 type passed as string
 /// Expectation: Tensor type is changed to float64 and all rows iterated correctly
-TEST_F(MindDataTestPipeline, TestToTensorOpFloat64) {
-  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestToTensorOpFloat64.";
+TEST_F(MindDataTestPipeline, TestToTensorOpFloat64String) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestToTensorOpFloat64String.";
 
   std::string MindDataPath = "data/dataset";
   std::string folder_path = MindDataPath + "/testImageNetData/train/";
@@ -773,8 +773,37 @@ TEST_F(MindDataTestPipeline, TestToTensorOpFloat64) {
 }
 
 /// Feature: ToTensor op
-/// Description: Test ToTensor op with default float64 type
+/// Description: Test ToTensor op with float64 type passed as DataType
 /// Expectation: Tensor type is changed to float64 and all rows iterated correctly
+TEST_F(MindDataTestPipeline, TestToTensorOpFloat64DataType) {
+  MS_LOG(INFO) << "Doing MindDataTestPipeline-TestToTensorOpFloat64DataType.";
+
+  std::string MindDataPath = "data/dataset";
+  std::string folder_path = MindDataPath + "/testImageNetData/train/";
+  std::shared_ptr<Dataset> ds = ImageFolder(folder_path, true, std::make_shared<RandomSampler>(false, 2));
+  EXPECT_NE(ds, nullptr);
+
+  auto to_tensor_op = vision::ToTensor(DataType::DE_FLOAT64);
+  ds = ds->Map({to_tensor_op}, {"image"});
+  EXPECT_NE(ds, nullptr);
+
+  std::shared_ptr<Iterator> iter = ds->CreateIterator();
+  std::unordered_map<std::string, mindspore::MSTensor> row;
+  iter->GetNextRow(&row);
+  uint64_t i = 0;
+  while (row.size() != 0) {
+    i++;
+    auto image = row["image"];
+    MS_LOG(INFO) << "Tensor image type: " << image.DataType();
+    iter->GetNextRow(&row);
+  }
+  EXPECT_EQ(i, 2);
+  iter->Stop();
+}
+
+/// Feature: ToTensor op
+/// Description: Test ToTensor op with float64 type and invalid uint32 type input data
+/// Expectation: Error is caught as given invalid input data type
 TEST_F(MindDataTestPipeline, TestToTensorOpInvalidInput) {
   MS_LOG(INFO) << "Doing MindDataTestPipeline-TestToTensorOpInvalidInput.";
 

tests/ut/cpp/dataset/to_tensor_op_test.cc

@@ -35,10 +35,10 @@ class MindDataTestToTensorOp : public UT::CVOP::CVOpCommon {
 };
 
 /// Feature: ToTensor Op
-/// Description: Check type changing with ToTensor C op to int8
+/// Description: Check type changing with ToTensor C op to int8 (string param)
 /// Expectation: Run successfully
-TEST_F(MindDataTestToTensorOp, TestToTensorOpInt8) {
-  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpInt8.";
+TEST_F(MindDataTestToTensorOp, TestToTensorOpInt8String) {
+  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpInt8String.";
   auto to_tensor_op = std::make_unique<ToTensorOp>("int8");
   std::shared_ptr<Tensor> output_tensor;
   Status s = to_tensor_op->Compute(input_tensor_, &output_tensor);
@@ -46,10 +46,21 @@ TEST_F(MindDataTestToTensorOp, TestToTensorOpInt8) {
 }
 
 /// Feature: ToTensor Op
-/// Description: Check type changing with ToTensor C op to int16
+/// Description: Check type changing with ToTensor C op to int8 (DataType param)
 /// Expectation: Run successfully
-TEST_F(MindDataTestToTensorOp, TestToTensorOpInt16) {
-  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpInt16.";
+TEST_F(MindDataTestToTensorOp, TestToTensorOpInt8DataType) {
+  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpInt8DataType.";
+  auto to_tensor_op = std::make_unique<ToTensorOp>(DataType::DE_INT8);
+  std::shared_ptr<Tensor> output_tensor;
+  Status s = to_tensor_op->Compute(input_tensor_, &output_tensor);
+  ASSERT_TRUE(DataType("int8") == output_tensor->type());
+}
+
+/// Feature: ToTensor Op
+/// Description: Check type changing with ToTensor C op to int16 (string param)
+/// Expectation: Run successfully
+TEST_F(MindDataTestToTensorOp, TestToTensorOpInt16String) {
+  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpInt16String.";
   auto to_tensor_op = std::make_unique<ToTensorOp>("int16");
   std::shared_ptr<Tensor> output_tensor;
   Status s = to_tensor_op->Compute(input_tensor_, &output_tensor);
@@ -57,10 +68,21 @@ TEST_F(MindDataTestToTensorOp, TestToTensorOpInt16) {
 }
 
 /// Feature: ToTensor Op
-/// Description: Check type changing with ToTensor C op to int32
+/// Description: Check type changing with ToTensor C op to int16 (DataType param)
 /// Expectation: Run successfully
-TEST_F(MindDataTestToTensorOp, TestToTensorOpInt32) {
-  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpInt32.";
+TEST_F(MindDataTestToTensorOp, TestToTensorOpInt16DataType) {
+  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpInt16DataType.";
+  auto to_tensor_op = std::make_unique<ToTensorOp>(DataType::DE_INT16);
+  std::shared_ptr<Tensor> output_tensor;
+  Status s = to_tensor_op->Compute(input_tensor_, &output_tensor);
+  ASSERT_TRUE(DataType("int16") == output_tensor->type());
+}
+
+/// Feature: ToTensor Op
+/// Description: Check type changing with ToTensor C op to int32 (String Param)
+/// Expectation: Run successfully
+TEST_F(MindDataTestToTensorOp, TestToTensorOpInt32String) {
+  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpInt32String.";
   auto to_tensor_op = std::make_unique<ToTensorOp>("int32");
   std::shared_ptr<Tensor> output_tensor;
   Status s = to_tensor_op->Compute(input_tensor_, &output_tensor);
@@ -68,10 +90,21 @@ TEST_F(MindDataTestToTensorOp, TestToTensorOpInt32) {
 }
 
 /// Feature: ToTensor Op
-/// Description: Check type changing with ToTensor C op to int64
+/// Description: Check type changing with ToTensor C op to int32 (DataType param)
 /// Expectation: Run successfully
-TEST_F(MindDataTestToTensorOp, TestToTensorOpInt64) {
-  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpInt64.";
+TEST_F(MindDataTestToTensorOp, TestToTensorOpInt32DataType) {
+  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpInt32DataType.";
+  auto to_tensor_op = std::make_unique<ToTensorOp>(DataType::DE_INT32);
+  std::shared_ptr<Tensor> output_tensor;
+  Status s = to_tensor_op->Compute(input_tensor_, &output_tensor);
+  ASSERT_TRUE(DataType("int32") == output_tensor->type());
+}
+
+/// Feature: ToTensor Op
+/// Description: Check type changing with ToTensor C op to int64 (String param)
+/// Expectation: Run successfully
+TEST_F(MindDataTestToTensorOp, TestToTensorOpInt64String) {
+  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpInt64String.";
   auto to_tensor_op = std::make_unique<ToTensorOp>("int64");
   std::shared_ptr<Tensor> output_tensor;
   Status s = to_tensor_op->Compute(input_tensor_, &output_tensor);
@@ -79,10 +112,21 @@ TEST_F(MindDataTestToTensorOp, TestToTensorOpInt64) {
 }
 
 /// Feature: ToTensor Op
-/// Description: Check type changing with ToTensor C op to float16
+/// Description: Check type changing with ToTensor C op to int64 (DataType param)
 /// Expectation: Run successfully
-TEST_F(MindDataTestToTensorOp, TestToTensorOpFloat16) {
-  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpFloat16.";
+TEST_F(MindDataTestToTensorOp, TestToTensorOpInt64DataType) {
+  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpInt64DataType.";
+  auto to_tensor_op = std::make_unique<ToTensorOp>(DataType::DE_INT64);
+  std::shared_ptr<Tensor> output_tensor;
+  Status s = to_tensor_op->Compute(input_tensor_, &output_tensor);
+  ASSERT_TRUE(DataType("int64") == output_tensor->type());
+}
+
+/// Feature: ToTensor Op
+/// Description: Check type changing with ToTensor C op to float16 (String param)
+/// Expectation: Run successfully
+TEST_F(MindDataTestToTensorOp, TestToTensorOpFloat16String) {
+  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpFloat16String.";
   auto to_tensor_op = std::make_unique<ToTensorOp>("float16");
   std::shared_ptr<Tensor> output_tensor;
   Status s = to_tensor_op->Compute(input_tensor_, &output_tensor);
@@ -90,10 +134,21 @@ TEST_F(MindDataTestToTensorOp, TestToTensorOpFloat16) {
 }
 
 /// Feature: ToTensor Op
-/// Description: Check type changing with ToTensor C op to float32
+/// Description: Check type changing with ToTensor C op to float16 (Datatype Param)
 /// Expectation: Run successfully
-TEST_F(MindDataTestToTensorOp, TestToTensorOpFloat32) {
-  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpFloat32.";
+TEST_F(MindDataTestToTensorOp, TestToTensorOpFloat16DataType) {
+  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpFloat16DataType.";
+  auto to_tensor_op = std::make_unique<ToTensorOp>(DataType::DE_FLOAT16);
+  std::shared_ptr<Tensor> output_tensor;
+  Status s = to_tensor_op->Compute(input_tensor_, &output_tensor);
+  ASSERT_TRUE(DataType("float16") == output_tensor->type());
+}
+
+/// Feature: ToTensor Op
+/// Description: Check type changing with ToTensor C op to float32 (String param)
+/// Expectation: Run successfully
+TEST_F(MindDataTestToTensorOp, TestToTensorOpFloat32String) {
+  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpFloat32String.";
   auto to_tensor_op = std::make_unique<ToTensorOp>("float32");
   std::shared_ptr<Tensor> output_tensor;
   Status s = to_tensor_op->Compute(input_tensor_, &output_tensor);
@@ -101,16 +156,38 @@ TEST_F(MindDataTestToTensorOp, TestToTensorOpFloat32) {
 }
 
 /// Feature: ToTensor Op
-/// Description: Check type changing with ToTensor C op to float64
+/// Description: Check type changing with ToTensor C op to float32 (DataType param)
 /// Expectation: Run successfully
-TEST_F(MindDataTestToTensorOp, TestToTensorOpFloat64) {
-  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpFloat64.";
+TEST_F(MindDataTestToTensorOp, TestToTensorOpFloat32DataType) {
+  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpFloat32DataType.";
+  auto to_tensor_op = std::make_unique<ToTensorOp>(DataType::DE_FLOAT32);
+  std::shared_ptr<Tensor> output_tensor;
+  Status s = to_tensor_op->Compute(input_tensor_, &output_tensor);
+  ASSERT_TRUE(DataType("float32") == output_tensor->type());
+}
+
+/// Feature: ToTensor Op
+/// Description: Check type changing with ToTensor C op to float64 (String param)
+/// Expectation: Run successfully
+TEST_F(MindDataTestToTensorOp, TestToTensorOpFloat64String) {
+  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpFloat64String.";
   auto to_tensor_op = std::make_unique<ToTensorOp>("float64");
   std::shared_ptr<Tensor> output_tensor;
   Status s = to_tensor_op->Compute(input_tensor_, &output_tensor);
   ASSERT_TRUE(DataType("float64") == output_tensor->type());
 }
 
+/// Feature: ToTensor Op
+/// Description: Check type changing with ToTensor C op to float64 (DataType param)
+/// Expectation: Run successfully
+TEST_F(MindDataTestToTensorOp, TestToTensorOpFloat64DataType) {
+  MS_LOG(INFO) << "Doing MindDataTestToTensorOp::TestToTensorOpFloat64DataType.";
+  auto to_tensor_op = std::make_unique<ToTensorOp>(DataType::DE_FLOAT64);
+  std::shared_ptr<Tensor> output_tensor;
+  Status s = to_tensor_op->Compute(input_tensor_, &output_tensor);
+  ASSERT_TRUE(DataType("float64") == output_tensor->type());
+}
+
 /// Feature: ToTensor Op
 /// Description: Check type changing with ToTensor C op to float32 with uint32 type input
 /// Expectation: Catch error for invalid type
@@ -123,7 +200,7 @@ TEST_F(MindDataTestToTensorOp, TestToTensorOpInputUInt32Invalid) {
   Status s_cast = type_cast_op->Compute(input_tensor_, &interim_tensor);
   ASSERT_OK(s_cast);
 
-  auto to_tensor_op = std::make_unique<ToTensorOp>("float32");
+  auto to_tensor_op = std::make_unique<ToTensorOp>(DataType::DE_FLOAT32);
   std::shared_ptr<Tensor> output_tensor;
   Status s_to_tensor = to_tensor_op->Compute(interim_tensor, &output_tensor);
   ASSERT_ERROR(s_to_tensor);
@@ -141,7 +218,7 @@ TEST_F(MindDataTestToTensorOp, TestToTensorOpInputUInt64Invalid) {
   Status s_cast = type_cast_op->Compute(input_tensor_, &interim_tensor);
   ASSERT_OK(s_cast);
 
-  auto to_tensor_op = std::make_unique<ToTensorOp>("float32");
+  auto to_tensor_op = std::make_unique<ToTensorOp>(DataType::DE_FLOAT32);
   std::shared_ptr<Tensor> output_tensor;
   Status s_to_tensor = to_tensor_op->Compute(interim_tensor, &output_tensor);
   ASSERT_ERROR(s_to_tensor);
@@ -159,7 +236,7 @@ TEST_F(MindDataTestToTensorOp, TestToTensorOpInputInt64Invalid) {
   Status s_cast = type_cast_op->Compute(input_tensor_, &interim_tensor);
   ASSERT_OK(s_cast);
 
-  auto to_tensor_op = std::make_unique<ToTensorOp>("float32");
+  auto to_tensor_op = std::make_unique<ToTensorOp>(DataType::DE_FLOAT32);
   std::shared_ptr<Tensor> output_tensor;
   Status s_to_tensor = to_tensor_op->Compute(interim_tensor, &output_tensor);
   ASSERT_ERROR(s_to_tensor);
@@ -176,7 +253,7 @@ TEST_F(MindDataTestToTensorOp, TestToTensorOpInputStringInvalid) {
   std::shared_ptr<Tensor> string_tensor;
   Tensor::CreateFromVector(strings, TensorShape({1, 2, 3}), &string_tensor);
 
-  auto to_tensor_op = std::make_unique<ToTensorOp>("float32");
+  auto to_tensor_op = std::make_unique<ToTensorOp>(DataType::DE_FLOAT32);
   std::shared_ptr<Tensor> output_tensor;
   Status s_to_tensor = to_tensor_op->Compute(string_tensor, &output_tensor);
   ASSERT_ERROR(s_to_tensor);