diff --git a/mindspore/ccsrc/minddata/dataset/api/datasets.cc b/mindspore/ccsrc/minddata/dataset/api/datasets.cc index f70bc49b5e7..f5296723dba 100644 --- a/mindspore/ccsrc/minddata/dataset/api/datasets.cc +++ b/mindspore/ccsrc/minddata/dataset/api/datasets.cc @@ -104,6 +104,7 @@ #endif #include "minddata/dataset/engine/ir/datasetops/source/mnist_node.h" #ifndef ENABLE_ANDROID +#include "minddata/dataset/engine/ir/datasetops/source/multi30k_node.h" #include "minddata/dataset/engine/ir/datasetops/source/penn_treebank_node.h" #include "minddata/dataset/engine/ir/datasetops/source/photo_tour_node.h" #include "minddata/dataset/engine/ir/datasetops/source/places365_node.h" @@ -1532,6 +1533,16 @@ MnistDataset::MnistDataset(const std::vector &dataset_dir, const std::vect } #ifndef ENABLE_ANDROID +Multi30kDataset::Multi30kDataset(const std::vector &dataset_dir, const std::vector &usage, + const std::vector> &language_pair, int64_t num_samples, + ShuffleMode shuffle, int32_t num_shards, int32_t shard_id, + const std::shared_ptr &cache) { + auto ds = + std::make_shared(CharToString(dataset_dir), CharToString(usage), VectorCharToString(language_pair), + num_samples, shuffle, num_shards, shard_id, cache); + ir_node_ = std::static_pointer_cast(ds); +} + PennTreebankDataset::PennTreebankDataset(const std::vector &dataset_dir, const std::vector &usage, int64_t num_samples, ShuffleMode shuffle, int32_t num_shards, int32_t shard_id, const std::shared_ptr &cache) { diff --git a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc index 645b7374423..29014ef3b8d 100644 --- a/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc +++ b/mindspore/ccsrc/minddata/dataset/api/python/bindings/dataset/engine/ir/datasetops/source/bindings.cc @@ -68,6 +68,7 @@ #include "minddata/dataset/engine/ir/datasetops/source/lj_speech_node.h" #include "minddata/dataset/engine/ir/datasetops/source/manifest_node.h" #include "minddata/dataset/engine/ir/datasetops/source/minddata_node.h" +#include "minddata/dataset/engine/ir/datasetops/source/multi30k_node.h" #include "minddata/dataset/engine/ir/datasetops/source/photo_tour_node.h" #include "minddata/dataset/engine/ir/datasetops/source/places365_node.h" #include "minddata/dataset/engine/ir/datasetops/source/qmnist_node.h" @@ -464,6 +465,20 @@ PYBIND_REGISTER(MnistNode, 2, ([](const py::module *m) { })); })); +PYBIND_REGISTER(Multi30kNode, 2, ([](const py::module *m) { + (void)py::class_>(*m, "Multi30kNode", + "to create a Multi30kNode") + .def(py::init([](const std::string &dataset_dir, const std::string &usage, + const std::vector &language_pair, int64_t num_samples, + int32_t shuffle, int32_t num_shards, int32_t shard_id) { + std::shared_ptr multi30k = + std::make_shared(dataset_dir, usage, language_pair, num_samples, + toShuffleMode(shuffle), num_shards, shard_id, nullptr); + THROW_IF_ERROR(multi30k->ValidateParams()); + return multi30k; + })); + })); + PYBIND_REGISTER(PennTreebankNode, 2, ([](const py::module *m) { (void)py::class_>( *m, "PennTreebankNode", "to create a PennTreebankNode") diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/CMakeLists.txt index 4926b65a490..88cf7432e9a 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/CMakeLists.txt @@ -30,6 +30,7 @@ set(DATASET_ENGINE_DATASETOPS_SOURCE_SRC_FILES lj_speech_op.cc mappable_leaf_op.cc mnist_op.cc + multi30k_op.cc nonmappable_leaf_op.cc penn_treebank_op.cc photo_tour_op.cc diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/multi30k_op.cc b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/multi30k_op.cc new file mode 100644 index 00000000000..80b8367d7ab --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/multi30k_op.cc @@ -0,0 +1,152 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "minddata/dataset/engine/datasetops/source/multi30k_op.h" + +#include +#include +#include +#include +#include +#include + +#include "debug/common.h" +#include "minddata/dataset/engine/datasetops/source/io_block.h" +#include "utils/file_utils.h" + +namespace mindspore { +namespace dataset { +// constructor of Multi30k. +Multi30kOp::Multi30kOp(int32_t num_workers, int64_t num_samples, const std::vector &language_pair, + int32_t worker_connector_size, std::unique_ptr schema, + const std::vector &text_files_list, int32_t op_connector_size, bool shuffle_files, + int32_t num_devices, int32_t device_id) + : TextFileOp(num_workers, num_samples, worker_connector_size, std::move(schema), std::move(text_files_list), + op_connector_size, shuffle_files, num_devices, device_id), + language_pair_(language_pair) {} + +// Print info of operator. +void Multi30kOp::Print(std::ostream &out, bool show_all) { + // Print parameter to debug function. + std::vector multi30k_files_list = TextFileOp::FileNames(); + if (!show_all) { + // Call the super class for displaying any common 1-liner info. + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal 1-liner info for this op. + out << "\n"; + } else { + // Call the super class for displaying any common detailed info. + ParallelOp::Print(out, show_all); + // Then show any custom derived-internal stuff. + out << "\nSample count: " << total_rows_ << "\nDevice id: " << device_id_ << "\nNumber of devices: " << num_devices_ + << "\nShuffle files: " << ((shuffle_files_) ? "yes" : "no") << "\nMulti30k files list:\n"; + for (int i = 0; i < multi30k_files_list.size(); ++i) { + out << " " << multi30k_files_list[i]; + } + out << "\n\n"; + } +} + +Status Multi30kOp::LoadTensor(const std::string &line, TensorRow *out_row, size_t index) { + RETURN_UNEXPECTED_IF_NULL(out_row); + std::shared_ptr tensor; + RETURN_IF_NOT_OK(Tensor::CreateScalar(line, &tensor)); + (*out_row)[index] = std::move(tensor); + return Status::OK(); +} + +Status Multi30kOp::LoadFile(const std::string &file_en, int64_t start_offset, int64_t end_offset, int32_t worker_id) { + auto realpath_en = FileUtils::GetRealPath(file_en.data()); + if (!realpath_en.has_value()) { + MS_LOG(ERROR) << "Invalid file path, " << DatasetName() + " Dataset file: " << file_en << " does not exist."; + RETURN_STATUS_UNEXPECTED("Invalid file path, " + DatasetName() + " Dataset file: " + file_en + " does not exist."); + } + + // We use English files to find Germany files, to make sure that data are ordered. + Path path_en(file_en); + Path parent_path(path_en.ParentPath()); + std::string basename = path_en.Basename(); + int suffix_len = 3; + std::string suffix_de = ".de"; + basename = basename.replace(basename.find("."), suffix_len, suffix_de); + Path BaseName(basename); + Path path_de = parent_path / BaseName; + std::string file_de = path_de.ToString(); + auto realpath_de = FileUtils::GetRealPath(file_de.data()); + if (!realpath_de.has_value()) { + MS_LOG(ERROR) << "Invalid file path, " << DatasetName() + " Dataset file: " << file_de << " does not exist."; + RETURN_STATUS_UNEXPECTED("Invalid file path, " + DatasetName() + " Dataset file: " + file_de + " does not exist."); + } + + std::ifstream handle_en(realpath_en.value()); + CHECK_FAIL_RETURN_UNEXPECTED(handle_en.is_open(), "Invalid file, failed to open en file: " + file_en); + std::ifstream handle_de(realpath_de.value()); + CHECK_FAIL_RETURN_UNEXPECTED(handle_de.is_open(), "Invalid file, failed to open de file: " + file_de); + + // Set path for path in class TensorRow. + std::string line_en; + std::string line_de; + std::vector path = {file_en, file_de}; + + int row_total = 0; + while (getline(handle_en, line_en) && getline(handle_de, line_de)) { + if (line_en.empty() && line_de.empty()) { + continue; + } + // If read to the end offset of this file, break. + if (row_total >= end_offset) { + break; + } + // Skip line before start offset. + if (row_total < start_offset) { + ++row_total; + continue; + } + + int tensor_size = 2; + TensorRow tRow(tensor_size, nullptr); + + Status rc_en; + Status rc_de; + if (language_pair_[0] == "en") { + rc_en = LoadTensor(line_en, &tRow, 0); + rc_de = LoadTensor(line_de, &tRow, 1); + } else if (language_pair_[0] == "de") { + rc_en = LoadTensor(line_en, &tRow, 1); + rc_de = LoadTensor(line_de, &tRow, 0); + } + if (rc_en.IsError() || rc_de.IsError()) { + handle_en.close(); + handle_de.close(); + RETURN_IF_NOT_OK(rc_en); + RETURN_IF_NOT_OK(rc_de); + } + (&tRow)->setPath(path); + + Status rc = jagged_rows_connector_->Add(worker_id, std::move(tRow)); + if (rc.IsError()) { + handle_en.close(); + handle_de.close(); + return rc; + } + ++row_total; + } + + handle_en.close(); + handle_de.close(); + return Status::OK(); +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/multi30k_op.h b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/multi30k_op.h new file mode 100644 index 00000000000..c22abbfbf4f --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/engine/datasetops/source/multi30k_op.h @@ -0,0 +1,85 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_MULTI30K_OP_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_MULTI30K_OP_H_ + +#include +#include +#include + +#include "minddata/dataset/engine/datasetops/source/text_file_op.h" + +namespace mindspore { +namespace dataset { +class JaggedConnector; +using StringIndex = AutoIndexObj; + +class Multi30kOp : public TextFileOp { + public: + /// \brief Constructor of Multi30kOp + /// \note The builder class should be used to call this constructor. + /// \param[in] num_workers Number of worker threads reading data from multi30k_file files. + /// \param[in] num_samples Number of rows to read. + /// \param[in] language_pair List containing text and translation language. + /// \param[in] worker_connector_size List of filepaths for the dataset files. + /// \param[in] schema The data schema object. + /// \param[in] text_files_list File path of multi30k files. + /// \param[in] op_connector_size Size of each queue in the connector that the child operator pulls from. + /// \param[in] shuffle_files Whether or not to shuffle the files before reading data. + /// \param[in] num_devices Shards of data. + /// \param[in] device_id The device ID within num_devices. + Multi30kOp(int32_t num_workers, int64_t num_samples, const std::vector &language_pair, + int32_t worker_connector_size, std::unique_ptr schema, + const std::vector &text_files_list, int32_t op_connector_size, bool shuffle_files, + int32_t num_devices, int32_t device_id); + + /// \Default destructor. + ~Multi30kOp() = default; + + /// \brief A print method typically used for debugging. + /// \param[in] out The output stream to write output to. + /// \param[in] show_all A bool to control if you want to show all info or just a summary. + void Print(std::ostream &out, bool show_all); + + /// \brief Return the name of Operator. + /// \return Status - return the name of Operator. + std::string Name() const override { return "Multi30kOp"; } + + /// \brief DatasetName name getter. + /// \param[in] upper If true, the return value is uppercase, otherwise, it is lowercase. + /// \return std::string DatasetName of the current Op. + std::string DatasetName(bool upper = false) const { return upper ? "Multi30k" : "multi30k"; } + + private: + /// \brief Load data into Tensor. + /// \param[in] line Data read from files. + /// \param[in] out_row Output tensor. + /// \param[in] index The index of Tensor. + Status LoadTensor(const std::string &line, TensorRow *out_row, size_t index); + + /// \brief Read data from files. + /// \param[in] file_en The paths of multi30k dataset files. + /// \param[in] start_offset The location of reading start. + /// \param[in] end_offset The location of reading finished. + /// \param[in] worker_id The id of the worker that is executing this function. + Status LoadFile(const std::string &file_en, int64_t start_offset, int64_t end_offset, int32_t worker_id); + + std::vector language_pair_; +}; +} // namespace dataset +} // namespace mindspore +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_DATASETOPS_SOURCE_MULTI30K_OP_H_ diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.h b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.h index 72a2846f350..fba0dc28812 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.h +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/dataset_node.h @@ -106,6 +106,7 @@ constexpr char kLJSpeechNode[] = "LJSpeechDataset"; constexpr char kManifestNode[] = "ManifestDataset"; constexpr char kMindDataNode[] = "MindDataDataset"; constexpr char kMnistNode[] = "MnistDataset"; +constexpr char kMulti30kNode[] = "Multi30kDataset"; constexpr char kPennTreebankNode[] = "PennTreebankDataset"; constexpr char kPhotoTourNode[] = "PhotoTourDataset"; constexpr char kPlaces365Node[] = "Places365Dataset"; diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/CMakeLists.txt b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/CMakeLists.txt index 816c450dc59..c811e4ad681 100644 --- a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/CMakeLists.txt +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/CMakeLists.txt @@ -32,6 +32,7 @@ set(DATASET_ENGINE_IR_DATASETOPS_SOURCE_SRC_FILES manifest_node.cc minddata_node.cc mnist_node.cc + multi30k_node.cc penn_treebank_node.cc photo_tour_node.cc places365_node.cc diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/multi30k_node.cc b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/multi30k_node.cc new file mode 100644 index 00000000000..bfe43688b0f --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/multi30k_node.cc @@ -0,0 +1,198 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "minddata/dataset/engine/ir/datasetops/source/multi30k_node.h" + +#include +#include +#include +#include +#include +#include + +#include "minddata/dataset/engine/datasetops/source/multi30k_op.h" + +namespace mindspore { +namespace dataset { +Multi30kNode::Multi30kNode(const std::string &dataset_dir, const std::string &usage, + const std::vector &language_pair, int32_t num_samples, ShuffleMode shuffle, + int32_t num_shards, int32_t shard_id, std::shared_ptr cache) + : NonMappableSourceNode(std::move(cache)), + dataset_dir_(dataset_dir), + usage_(usage), + language_pair_(language_pair), + num_samples_(num_samples), + shuffle_(shuffle), + num_shards_(num_shards), + shard_id_(shard_id), + multi30k_files_list_(WalkAllFiles(usage, dataset_dir)) { + GlobalContext::config_manager()->set_num_shards_for_auto_num_workers(num_shards_); +} + +void Multi30kNode::Print(std::ostream &out) const { + out << (Name() + "(cache: " + ((cache_ != nullptr) ? "true" : "false") + + ", num_shards: " + std::to_string(num_shards_) + ", shard_id: " + std::to_string(shard_id_) + ")"); +} + +std::shared_ptr Multi30kNode::Copy() { + auto node = std::make_shared(dataset_dir_, usage_, language_pair_, num_samples_, shuffle_, num_shards_, + shard_id_, cache_); + return node; +} + +// Function to build Multi30kNode +Status Multi30kNode::Build(std::vector> *const node_ops) { + bool shuffle_files = (shuffle_ == ShuffleMode::kGlobal || shuffle_ == ShuffleMode::kFiles); + + std::vector sorted_dataset_files = multi30k_files_list_; + std::sort(sorted_dataset_files.begin(), sorted_dataset_files.end()); + + auto schema = std::make_unique(); + RETURN_IF_NOT_OK(schema->AddColumn(ColDescriptor("text", DataType(DataType::DE_STRING), TensorImpl::kFlexible, 1))); + RETURN_IF_NOT_OK( + schema->AddColumn(ColDescriptor("translation", DataType(DataType::DE_STRING), TensorImpl::kFlexible, 1))); + + std::shared_ptr multi30k_op = + std::make_shared(num_workers_, num_samples_, language_pair_, worker_connector_size_, std::move(schema), + sorted_dataset_files, connector_que_size_, shuffle_files, num_shards_, shard_id_); + RETURN_IF_NOT_OK(multi30k_op->Init()); + + if (shuffle_ == ShuffleMode::kGlobal) { + // Inject ShuffleOp + std::shared_ptr shuffle_op = nullptr; + int64_t num_rows = 0; + + // First, get the number of rows in the dataset + RETURN_IF_NOT_OK(Multi30kOp::CountAllFileRows(sorted_dataset_files, &num_rows)); + + // Add the shuffle op after this op + RETURN_IF_NOT_OK( + AddShuffleOp(sorted_dataset_files.size(), num_shards_, num_rows, 0, connector_que_size_, &shuffle_op)); + shuffle_op->SetTotalRepeats(GetTotalRepeats()); + shuffle_op->SetNumRepeatsPerEpoch(GetNumRepeatsPerEpoch()); + node_ops->push_back(shuffle_op); + } + multi30k_op->SetTotalRepeats(GetTotalRepeats()); + multi30k_op->SetNumRepeatsPerEpoch(GetNumRepeatsPerEpoch()); + // Add Multi30kOp + node_ops->push_back(multi30k_op); + + return Status::OK(); +} + +Status Multi30kNode::ValidateParams() { + RETURN_IF_NOT_OK(DatasetNode::ValidateParams()); + RETURN_IF_NOT_OK(ValidateDatasetDirParam("Multi30kDataset", dataset_dir_)); + RETURN_IF_NOT_OK(ValidateDatasetFilesParam("Multi30kDataset", multi30k_files_list_)); + RETURN_IF_NOT_OK(ValidateStringValue("Multi30kDataset", usage_, {"train", "valid", "test", "all"})); + + const int kLanguagePairSize = 2; + if (language_pair_.size() != kLanguagePairSize) { + std::string err_msg = + "Multi30kDataset: language_pair expecting size 2, but got: " + std::to_string(language_pair_.size()); + LOG_AND_RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + + const std::vector> support_language_pair = {{"en", "de"}, {"de", "en"}}; + if (language_pair_ != support_language_pair[0] && language_pair_ != support_language_pair[1]) { + std::string err_msg = R"(Multi30kDataset: language_pair must be {"en", "de"} or {"de", "en"}.)"; + LOG_AND_RETURN_STATUS_SYNTAX_ERROR(err_msg); + } + + RETURN_IF_NOT_OK(ValidateScalar("Multi30kDataset", "num_samples", num_samples_, {0}, false)); + RETURN_IF_NOT_OK(ValidateDatasetShardParams("Multi30kDataset", num_shards_, shard_id_)); + return Status::OK(); +} + +Status Multi30kNode::GetShardId(int32_t *shard_id) { + *shard_id = shard_id_; + return Status::OK(); +} + +Status Multi30kNode::GetDatasetSize(const std::shared_ptr &size_getter, bool estimate, + int64_t *dataset_size) { + if (dataset_size_ > 0) { + *dataset_size = dataset_size_; + return Status::OK(); + } + int64_t num_rows, sample_size = num_samples_; + RETURN_IF_NOT_OK(Multi30kOp::CountAllFileRows(multi30k_files_list_, &num_rows)); + num_rows = static_cast(ceil(num_rows / (1.0 * num_shards_))); + *dataset_size = sample_size > 0 ? std::min(num_rows, sample_size) : num_rows; + dataset_size_ = *dataset_size; + return Status::OK(); +} + +Status Multi30kNode::to_json(nlohmann::json *out_json) { + nlohmann::json args; + args["num_parallel_workers"] = num_workers_; + args["dataset_dir"] = dataset_dir_; + args["num_samples"] = num_samples_; + args["shuffle"] = shuffle_; + args["num_shards"] = num_shards_; + args["shard_id"] = shard_id_; + args["language_pair"] = language_pair_; + if (cache_ != nullptr) { + nlohmann::json cache_args; + RETURN_IF_NOT_OK(cache_->to_json(&cache_args)); + args["cache"] = cache_args; + } + *out_json = args; + return Status::OK(); +} + +Status Multi30kNode::SetupSamplerForCache(std::shared_ptr *sampler) { + bool shuffle_files = (shuffle_ == ShuffleMode::kGlobal || shuffle_ == ShuffleMode::kFiles); + *sampler = SelectSampler(num_samples_, shuffle_files, num_shards_, shard_id_); + return Status::OK(); +} + +Status Multi30kNode::MakeSimpleProducer() { + shard_id_ = 0; + num_shards_ = 1; + shuffle_ = ShuffleMode::kFalse; + num_samples_ = 0; + return Status::OK(); +} + +std::vector Multi30kNode::WalkAllFiles(const std::string &usage, const std::string &dataset_dir) { + std::vector multi30k_files_list; + Path train_en("training/train.en"); + Path test_en("mmt16_task1_test/test.en"); + Path valid_en("validation/val.en"); + Path dir(dataset_dir); + + if (usage == "train") { + Path temp_path = dir / train_en; + multi30k_files_list.push_back(temp_path.ToString()); + } else if (usage == "test") { + Path temp_path = dir / test_en; + multi30k_files_list.push_back(temp_path.ToString()); + } else if (usage == "valid") { + Path temp_path = dir / valid_en; + multi30k_files_list.push_back(temp_path.ToString()); + } else { + Path temp_path = dir / train_en; + multi30k_files_list.push_back(temp_path.ToString()); + Path temp_path1 = dir / test_en; + multi30k_files_list.push_back(temp_path1.ToString()); + Path temp_path2 = dir / valid_en; + multi30k_files_list.push_back(temp_path2.ToString()); + } + return multi30k_files_list; +} +} // namespace dataset +} // namespace mindspore diff --git a/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/multi30k_node.h b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/multi30k_node.h new file mode 100644 index 00000000000..f293d01d987 --- /dev/null +++ b/mindspore/ccsrc/minddata/dataset/engine/ir/datasetops/source/multi30k_node.h @@ -0,0 +1,134 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_IR_DATASETOPS_SOURCE_MULTI30K_NODE_H_ +#define MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_IR_DATASETOPS_SOURCE_MULTI30K_NODE_H_ + +#include +#include +#include + +#include "minddata/dataset/engine/ir/datasetops/dataset_node.h" + +namespace mindspore { +namespace dataset { +class Multi30kNode : public NonMappableSourceNode { + public: + /// \brief Constructor of Multi30kNode. + /// \param[in] dataset_dir Path to the root directory that contains the dataset. + /// \param[in] usage Part of dataset of MULTI30K, can be "train", "test", "valid" or "all". + /// \param[in] language_pair List containing text and translation language. + /// \param[in] num_samples The number of samples to be included in the dataset + /// \param[in] shuffle The mode for shuffling data every epoch. + /// Can be any of: + /// ShuffleMode::kFalse - No shuffling is performed. + /// ShuffleMode::kFiles - Shuffle files only. + /// ShuffleMode::kGlobal - Shuffle both the files and samples. + /// \param[in] num_shards Number of shards that the dataset should be divided into. + /// \param[in] shared_id The shard ID within num_shards. This argument should be + /// specified only when num_shards is also specified. + /// \param[in] cache Tensor cache to use. + Multi30kNode(const std::string &dataset_dir, const std::string &usage, const std::vector &language_pair, + int32_t num_samples, ShuffleMode shuffle, int32_t num_shards, int32_t shared_id, + std::shared_ptr cache); + + /// \brief Destructor of Multi30kNode. + ~Multi30kNode() = default; + + /// \brief Node name getter. + /// \return Name of the current node. + std::string Name() const override { return kMulti30kNode; } + + /// \brief Print the description. + /// \param[in] out The output stream to write output to. + void Print(std::ostream &out) const override; + + /// \brief Copy the node to a new object. + /// \return A shared pointer to the new copy. + std::shared_ptr Copy() override; + + /// \brief a base class override function to create the required runtime dataset op objects for this class. + /// \param[in] node_ops A vector containing shared pointer to the Dataset Ops that this object will create. + /// \return Status Status::OK() if build successfully. + Status Build(std::vector> *const node_ops); + + /// \brief Parameters validation. + /// \return Status Status::OK() if all the parameters are valid. + Status ValidateParams() override; + + /// \brief Get the shard id of node. + /// \param[in] shard_id The shard id. + /// \return Status Status::OK() if get shard id successfully. + Status GetShardId(int32_t *shard_id) override; + + /// \brief Base-class override for GetDatasetSize. + /// \param[in] size_getter Shared pointer to DatasetSizeGetter. + /// \param[in] estimate This is only supported by some of the ops and it's used to speed up the process of getting. + /// dataset size at the expense of accuracy. + /// \param[out] dataset_size the size of the dataset. + /// \return Status of the function. + Status GetDatasetSize(const std::shared_ptr &size_getter, bool estimate, + int64_t *dataset_size) override; + + /// \brief Get the arguments of node. + /// \param[out] out_json JSON string of all attributes. + /// \return Status of the function. + Status to_json(nlohmann::json *out_json) override; + + /// \brief Multi30k by itself is a non-mappable dataset that does not support sampling. + /// However, if a cache operator is injected at some other place higher in the tree, that cache can + /// inherit this sampler from the leaf, providing sampling support from the caching layer. + /// That is why we setup the sampler for a leaf node that does not use sampling. + /// Note: This function is common among NonMappableSourceNode and should be promoted to its parent class. + /// \param[in] sampler The sampler to setup. + /// \return Status of the function. + Status SetupSamplerForCache(std::shared_ptr *sampler) override; + + /// \brief If a cache has been added into the ascendant tree over this Multi30k node, then the cache will be executing + /// a sampler for fetching the data. As such, any options in the Multi30k node need to be reset to its defaults + /// so that this Multi30k node will produce the full set of data into the cache. + /// Note: This function is common among NonMappableSourceNode and should be promoted to its parent class. + /// \return Status of the function + Status MakeSimpleProducer() override; + + /// \brief Getter functions + int32_t NumSamples() const { return num_samples_; } + int32_t NumShards() const { return num_shards_; } + int32_t ShardId() const { return shard_id_; } + ShuffleMode Shuffle() const { return shuffle_; } + const std::string &DatasetDir() const { return dataset_dir_; } + const std::string &Usage() const { return usage_; } + const std::vector &LanguagePair() const { return language_pair_; } + + /// \brief Generate a list of read file names according to usage. + /// \param[in] usage Part of dataset of Multi30k. + /// \param[in] dataset_dir Path to the root directory that contains the dataset. + /// \return std::vector A list of read file names. + std::vector WalkAllFiles(const std::string &usage, const std::string &dataset_dir); + + private: + std::string dataset_dir_; + std::string usage_; + std::vector language_pair_; + int32_t num_samples_; + ShuffleMode shuffle_; + int32_t num_shards_; + int32_t shard_id_; + std::vector multi30k_files_list_; +}; +} // namespace dataset +} // namespace mindspore +#endif // MINDSPORE_CCSRC_MINDDATA_DATASET_ENGINE_IR_DATASETOPS_SOURCE_MULTI30K_NODE_H_ diff --git a/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h b/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h index b7af3bc43eb..b7c8252804e 100644 --- a/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h +++ b/mindspore/ccsrc/minddata/dataset/include/dataset/datasets.h @@ -3723,6 +3723,75 @@ inline std::shared_ptr MS_API Mnist(const std::string &dataset_dir return std::make_shared(StringToChar(dataset_dir), StringToChar(usage), sampler, cache); } +/// \class Multi30kDataset +/// \brief A source dataset that reads and parses Multi30k dataset. +class MS_API Multi30kDataset : public Dataset { + public: + /// \brief Constructor of Multi30kDataset. + /// \param[in] dataset_dir Path to the root directory that contains the dataset. + /// \param[in] usage Part of dataset of MULTI30K, can be "train", "test", "valid" or "all". + /// \param[in] language_pair List containing text and translation language. + /// \param[in] num_samples The number of samples to be included in the dataset. + /// \param[in] shuffle The mode for shuffling data every epoch. + /// Can be any of: + /// ShuffleMode::kFalse - No shuffling is performed. + /// ShuffleMode::kFiles - Shuffle files only. + /// ShuffleMode::kGlobal - Shuffle both the files and samples. + /// \param[in] num_shards Number of shards that the dataset should be divided into. + /// \param[in] shard_id The shard ID within num_shards. This argument should be + /// specified only when num_shards is also specified. + /// \param[in] cache Tensor cache to use. + Multi30kDataset(const std::vector &dataset_dir, const std::vector &usage, + const std::vector> &language_pair, int64_t num_samples, ShuffleMode shuffle, + int32_t num_shards, int32_t shard_id, const std::shared_ptr &cache); + + /// \brief Destructor of Multi30kDataset. + ~Multi30kDataset() = default; +}; + +/// \brief Function to create a Multi30kDataset. +/// \note The generated dataset has two columns ["text", "translation"]. +/// \param[in] dataset_dir Path to the root directory that contains the dataset. +/// \param[in] usage Part of dataset of MULTI30K, can be "train", "test", "valid" or "all" (default = "all"). +/// \param[in] language_pair List containing text and translation language (default = {"en", "de"}). +/// \param[in] num_samples The number of samples to be included in the dataset +/// (Default = 0, means all samples). +/// \param[in] shuffle The mode for shuffling data every epoch (Default=ShuffleMode.kGlobal). +/// Can be any of: +/// ShuffleMode::kFalse - No shuffling is performed. +/// ShuffleMode::kFiles - Shuffle files only. +/// ShuffleMode::kGlobal - Shuffle both the files and samples. +/// \param[in] num_shards Number of shards that the dataset should be divided into (Default = 1). +/// \param[in] shard_id The shard ID within num_shards. This argument should be +/// specified only when num_shards is also specified (Default = 0). +/// \param[in] cache Tensor cache to use (default=nullptr, which means no cache is used). +/// \return Shared pointer to the Multi30kDataset. +/// \par Example +/// \code +/// /* Define dataset path and MindData object */ +/// std::string dataset_dir = "/path/to/multi30k_dataset_directory"; +/// std::shared_ptr ds = Multi30k(dataset_dir, "all"); +/// +/// /* Create iterator to read dataset */ +/// std::shared_ptr iter = ds->CreateIterator(); +/// std::unordered_map row; +/// iter->GetNextRow(&row); +/// +/// /* Note: In Multi30kdataset, each dictionary has keys "text" and "translation" */ +/// auto text = row["text"]; +/// \endcode +inline std::shared_ptr MS_API Multi30k(const std::string &dataset_dir, + const std::string &usage = "all", + const std::vector &language_pair = {"en", "de"}, + int64_t num_samples = 0, + ShuffleMode shuffle = ShuffleMode::kGlobal, + int32_t num_shards = 1, int32_t shard_id = 0, + const std::shared_ptr &cache = nullptr) { + return std::make_shared(StringToChar(dataset_dir), StringToChar(usage), + VectorStringToChar(language_pair), num_samples, shuffle, num_shards, + shard_id, cache); +} + /// \class PennTreebankDataset /// \brief A source dataset for reading and parsing PennTreebank dataset. class MS_API PennTreebankDataset : public Dataset { diff --git a/mindspore/python/mindspore/dataset/engine/__init__.py b/mindspore/python/mindspore/dataset/engine/__init__.py index 55563e7b29c..8d56fe99c62 100644 --- a/mindspore/python/mindspore/dataset/engine/__init__.py +++ b/mindspore/python/mindspore/dataset/engine/__init__.py @@ -74,6 +74,7 @@ __all__ = ["Caltech101Dataset", # Vision "IMDBDataset", # Text "IWSLT2016Dataset", # Text "IWSLT2017Dataset", # Text + "Multi30kDataset", # Text "PennTreebankDataset", # Text "SogouNewsDataset", # Text "TextFileDataset", # Text diff --git a/mindspore/python/mindspore/dataset/engine/datasets_text.py b/mindspore/python/mindspore/dataset/engine/datasets_text.py index 04ebd71fae0..97185c5dde8 100644 --- a/mindspore/python/mindspore/dataset/engine/datasets_text.py +++ b/mindspore/python/mindspore/dataset/engine/datasets_text.py @@ -30,7 +30,7 @@ from .validators import check_imdb_dataset, check_iwslt2016_dataset, check_iwslt check_penn_treebank_dataset, check_ag_news_dataset, check_amazon_review_dataset, check_udpos_dataset, \ check_wiki_text_dataset, check_conll2000_dataset, check_cluedataset, \ check_sogou_news_dataset, check_textfiledataset, check_dbpedia_dataset, check_yelp_review_dataset, \ - check_en_wik9_dataset, check_yahoo_answers_dataset + check_en_wik9_dataset, check_yahoo_answers_dataset, check_multi30k_dataset from ..core.validator_helpers import replace_none @@ -961,6 +961,106 @@ class IWSLT2017Dataset(SourceDataset, TextBaseDataset): self.shuffle_flag, self.num_shards, self.shard_id) +class Multi30kDataset(SourceDataset, TextBaseDataset): + """ + A source dataset that reads and parses Multi30k dataset. + + The generated dataset has two columns :py:obj:`[text, translation]`. + The tensor of column :py:obj:'text' is of the string type. + The tensor of column :py:obj:'translation' is of the string type. + + Args: + dataset_dir (str): Path to the root directory that contains the dataset. + usage (str, optional): Acceptable usages include `train`, `test, `valid` or `all` (default=`all`). + language_pair (str, optional): Acceptable language_pair include ['en', 'de'], ['de', 'en'] + (default=['en', 'de']). + num_samples (int, optional): The number of images to be included in the dataset + (default=None, all samples). + num_parallel_workers (int, optional): Number of workers to read the data + (default=None, number set in the config). + shuffle (Union[bool, Shuffle level], optional): Perform reshuffling of the data every epoch + (default=Shuffle.GLOBAL). + If shuffle is False, no shuffling will be performed; + If shuffle is True, the behavior is the same as setting shuffle to be Shuffle.GLOBAL + Otherwise, there are two levels of shuffling: + + - Shuffle.GLOBAL: Shuffle both the files and samples. + + - Shuffle.FILES: Shuffle files only. + + num_shards (int, optional): Number of shards that the dataset will be divided + into (default=None). When this argument is specified, `num_samples` reflects + the max sample number of per shard. + shard_id (int, optional): The shard ID within num_shards (default=None). This + argument can only be specified when num_shards is also specified. + cache (DatasetCache, optional): Use tensor caching service to speed up dataset processing + (default=None, which means no cache is used). + + Raises: + RuntimeError: If dataset_dir does not contain data files. + RuntimeError: If usage is not "train", "test", "valid" or "all". + RuntimeError: If the length of language_pair is not equal to 2. + RuntimeError: If num_parallel_workers exceeds the max thread numbers. + RuntimeError: If num_shards is specified but shard_id is None. + RuntimeError: If shard_id is specified but num_shards is None. + RuntimeError: If num_samples is less than 0. + + Examples: + >>> multi30k_dataset_dir = "/path/to/multi30k_dataset_directory" + >>> data = ds.Multi30kDataset(dataset_dir=multi30k_dataset_dir, usage='all', language_pair=['de', 'en']) + + About Multi30k dataset: + + Multi30K is a dataset to stimulate multilingual multimodal research for English-German. + It is based on the Flickr30k dataset, which contains images sourced from online + photo-sharing websites. Each image is paired with five English descriptions, which were + collected from Amazon Mechanical Turk. The Multi30K dataset extends the Flickr30K + dataset with translated and independent German sentences. + + You can unzip the dataset files into the following directory structure and read by MindSpore's API. + + .. code-block:: + + └── multi30k_dataset_directory + ├── training + │ ├── train.de + │ └── train.en + ├── validation + │ ├── val.de + │ └── val.en + └── mmt16_task1_test + ├── val.de + └── val.en + + Citation: + + .. code-block:: + + @article{elliott-EtAl:2016:VL16, + author = {{Elliott}, D. and {Frank}, S. and {Sima'an}, K. and {Specia}, L.}, + title = {Multi30K: Multilingual English-German Image Descriptions}, + booktitle = {Proceedings of the 5th Workshop on Vision and Language}, + year = {2016}, + pages = {70--74}, + year = 2016 + } + """ + + @check_multi30k_dataset + def __init__(self, dataset_dir, usage=None, language_pair=None, num_samples=None, + num_parallel_workers=None, shuffle=None, num_shards=None, shard_id=None, cache=None): + super().__init__(num_parallel_workers=num_parallel_workers, num_samples=num_samples, shuffle=shuffle, + num_shards=num_shards, shard_id=shard_id, cache=cache) + self.dataset_dir = dataset_dir + self.usage = replace_none(usage, 'all') + self.language_pair = replace_none(language_pair, ["en", "de"]) + self.shuffle = replace_none(shuffle, Shuffle.GLOBAL) + + def parse(self, children=None): + return cde.Multi30kNode(self.dataset_dir, self.usage, self.language_pair, self.num_samples, + self.shuffle_flag, self.num_shards, self.shard_id) + + class PennTreebankDataset(SourceDataset, TextBaseDataset): """ A source dataset that reads and parses PennTreebank datasets. diff --git a/mindspore/python/mindspore/dataset/engine/validators.py b/mindspore/python/mindspore/dataset/engine/validators.py index b653387b0fc..1f87b1b3951 100644 --- a/mindspore/python/mindspore/dataset/engine/validators.py +++ b/mindspore/python/mindspore/dataset/engine/validators.py @@ -2557,4 +2557,41 @@ def check_en_wik9_dataset(method): return method(self, *args, **kwargs) return new_method - \ No newline at end of file + + +def check_multi30k_dataset(method): + """A wrapper that wraps a parameter checker around the original Dataset (Multi30kDataset).""" + + @wraps(method) + def new_method(self, *args, **kwargs): + _, param_dict = parse_user_args(method, *args, **kwargs) + + nreq_param_int = ['num_samples', 'num_parallel_workers', 'num_shards', 'shard_id'] + nreq_param_bool = ['shuffle', 'decode'] + + dataset_dir = param_dict.get('dataset_dir') + check_dir(dataset_dir) + + usage = param_dict.get('usage') + if usage is not None: + check_valid_str(usage, ["train", "test", "valid", "all"], "usage") + + language_pair = param_dict.get('language_pair') + support_language_pair = [['en', 'de'], ['de', 'en'], ('en', 'de'), ('de', 'en')] + if language_pair is not None: + type_check(language_pair, (list, tuple), "language_pair") + if len(language_pair) != 2: + raise ValueError( + "language_pair should be a list or tuple of length 2, but got {0}".format(len(language_pair))) + if language_pair not in support_language_pair: + raise ValueError( + "language_pair can only be ['en', 'de'] or ['en', 'de'], but got {0}".format(language_pair)) + + validate_dataset_param_value(nreq_param_int, param_dict, int) + validate_dataset_param_value(nreq_param_bool, param_dict, bool) + + check_sampler_shuffle_shard_options(param_dict) + + return method(self, *args, **kwargs) + + return new_method diff --git a/tests/ut/cpp/dataset/CMakeLists.txt b/tests/ut/cpp/dataset/CMakeLists.txt index a945ed29717..8fccd875fb4 100644 --- a/tests/ut/cpp/dataset/CMakeLists.txt +++ b/tests/ut/cpp/dataset/CMakeLists.txt @@ -40,6 +40,7 @@ SET(DE_UT_SRCS c_api_dataset_lj_speech_test.cc c_api_dataset_manifest_test.cc c_api_dataset_minddata_test.cc + c_api_dataset_multi30k_test.cc c_api_dataset_ops_test.cc c_api_dataset_penn_treebank_test.cc c_api_dataset_photo_tour_test.cc diff --git a/tests/ut/cpp/dataset/c_api_dataset_multi30k_test.cc b/tests/ut/cpp/dataset/c_api_dataset_multi30k_test.cc new file mode 100644 index 00000000000..104482fb5e3 --- /dev/null +++ b/tests/ut/cpp/dataset/c_api_dataset_multi30k_test.cc @@ -0,0 +1,652 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "common/common.h" +#include "minddata/dataset/include/dataset/datasets.h" + +using namespace mindspore::dataset; +using mindspore::Status; +using mindspore::dataset::ShuffleMode; + +class MindDataTestPipeline : public UT::DatasetOpTesting { + protected: +}; + +/// Feature: Test Multi30k Dataset(English). +/// Description: read Multi30kDataset data and get data. +/// Expectation: the data is processed successfully. +TEST_F(MindDataTestPipeline, TestMulti30kSuccessEn) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kSuccessEn."; + // Test Multi30k English files with default parameters + + // Create a Multi30k dataset + std::string en_file = datasets_root_path_ + "/testMulti30kDataset"; + + // test train + std::string usage = "train"; + std::shared_ptr ds = Multi30k(en_file, usage, {"en", "de"}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("text"), row.end()); + std::vector expected = {"This is the first English sentence in train.", + "This is the second English sentence in train.", + "This is the third English sentence in train."}; + + uint64_t i = 0; + while (row.size() != 0) { + auto text = row["text"]; + std::shared_ptr de_text; + ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); + std::string_view sv; + ASSERT_OK(de_text->GetItemAt(&sv, {})); + std::string ss(sv); + EXPECT_STREQ(ss.c_str(), expected[i].c_str()); + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + + EXPECT_EQ(i, 3); + + // Manually terminate the pipeline + iter->Stop(); + + // test valid + usage = "valid"; + expected = {"This is the first English sentence in valid.", + "This is the second English sentence in valid."}; + + ds = Multi30k(en_file, usage, {"en", "de"}, 0, ShuffleMode::kFalse); + + EXPECT_NE(ds, nullptr); + iter = ds->CreateIterator(); + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("text"), row.end()); + + i = 0; + while (row.size() != 0) { + auto text = row["text"]; + std::shared_ptr de_text; + ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); + std::string_view sv; + ASSERT_OK(de_text->GetItemAt(&sv, {})); + std::string ss(sv); + EXPECT_STREQ(ss.c_str(), expected[i].c_str()); + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + + EXPECT_EQ(i, 2); + + iter->Stop(); + + // test test + usage = "test"; + expected = {"This is the first English sentence in test.", + "This is the second English sentence in test.", + "This is the third English sentence in test."}; + + ds = Multi30k(en_file, usage, {"en", "de"}, 0, ShuffleMode::kFalse); + + EXPECT_NE(ds, nullptr); + iter = ds->CreateIterator(); + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("text"), row.end()); + + i = 0; + while (row.size() != 0) { + auto text = row["text"]; + std::shared_ptr de_text; + ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); + std::string_view sv; + ASSERT_OK(de_text->GetItemAt(&sv, {})); + std::string ss(sv); + EXPECT_STREQ(ss.c_str(), expected[i].c_str()); + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + + EXPECT_EQ(i, 3); + + iter->Stop(); +} + +/// Feature: Test Multi30k Dataset(Germany). +/// Description: read Multi30kDataset data and get data. +/// Expectation: the data is processed successfully. +TEST_F(MindDataTestPipeline, TestMulti30kSuccessDe) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kSuccessDe."; + // Test Multi30k Germany files with default parameters + + // Create a Multi30k dataset + std::string en_file = datasets_root_path_ + "/testMulti30kDataset"; + + // test train + std::string usage = "train"; + std::shared_ptr ds = Multi30k(en_file, usage, {"en", "de"}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("translation"), row.end()); + std::vector expected = {"This is the first Germany sentence in train.", + "This is the second Germany sentence in train.", + "This is the third Germany sentence in train."}; + + uint64_t i = 0; + while (row.size() != 0) { + auto text = row["translation"]; + std::shared_ptr de_text; + ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); + std::string_view sv; + ASSERT_OK(de_text->GetItemAt(&sv, {})); + std::string ss(sv); + EXPECT_STREQ(ss.c_str(), expected[i].c_str()); + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + + EXPECT_EQ(i, 3); + + // Manually terminate the pipeline + iter->Stop(); + + // test valid + usage = "valid"; + expected = {"This is the first Germany sentence in valid.", + "This is the second Germany sentence in valid."}; + + ds = Multi30k(en_file, usage, {"en", "de"}, 0, ShuffleMode::kFalse); + + EXPECT_NE(ds, nullptr); + iter = ds->CreateIterator(); + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("translation"), row.end()); + + i = 0; + while (row.size() != 0) { + auto text = row["translation"]; + std::shared_ptr de_text; + ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); + std::string_view sv; + ASSERT_OK(de_text->GetItemAt(&sv, {})); + std::string ss(sv); + EXPECT_STREQ(ss.c_str(), expected[i].c_str()); + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + + EXPECT_EQ(i, 2); + + iter->Stop(); + + // test test + usage = "test"; + expected = {"This is the first Germany sentence in test.", + "This is the second Germany sentence in test.", + "This is the third Germany sentence in test."}; + + ds = Multi30k(en_file, usage, {"en", "de"}, 0, ShuffleMode::kFalse); + + EXPECT_NE(ds, nullptr); + iter = ds->CreateIterator(); + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("translation"), row.end()); + + i = 0; + while (row.size() != 0) { + auto text = row["translation"]; + std::shared_ptr de_text; + ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); + std::string_view sv; + ASSERT_OK(de_text->GetItemAt(&sv, {})); + std::string ss(sv); + EXPECT_STREQ(ss.c_str(), expected[i].c_str()); + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + + EXPECT_EQ(i, 3); + + iter->Stop(); +} + +/// Feature: Test Multi30k Dataset(Germany). +/// Description: read Multi30kDataset data and get data. +/// Expectation: the data is processed successfully. +TEST_F(MindDataTestPipeline, TestMulti30kDatasetBasicWithPipeline) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kDatasetBasicWithPipeline."; + + // Create two Multi30kFile Dataset, with single Multi30k file + std::string train_en_file = datasets_root_path_ + "/testMulti30kDataset"; + std::string usage = "train"; + std::shared_ptr ds1 = Multi30k(train_en_file, usage, {"en", "de"}, 2, ShuffleMode::kFalse); + std::shared_ptr ds2 = Multi30k(train_en_file, usage, {"en", "de"}, 2, ShuffleMode::kFalse); + EXPECT_NE(ds1, nullptr); + EXPECT_NE(ds2, nullptr); + + // Create two Repeat operation on ds + int32_t repeat_num = 2; + ds1 = ds1->Repeat(repeat_num); + EXPECT_NE(ds1, nullptr); + repeat_num = 3; + ds2 = ds2->Repeat(repeat_num); + EXPECT_NE(ds2, nullptr); + + // Create two Project operation on ds + std::vector column_project = {"text"}; + ds1 = ds1->Project(column_project); + EXPECT_NE(ds1, nullptr); + ds2 = ds2->Project(column_project); + EXPECT_NE(ds2, nullptr); + + // Create a Concat operation on the ds + ds1 = ds1->Concat({ds2}); + EXPECT_NE(ds1, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds1->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("text"), row.end()); + uint64_t i = 0; + while (row.size() != 0) { + auto text = row["text"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); + i++; + ASSERT_OK(iter->GetNextRow(&row)); + } + + // Expect 10 samples + EXPECT_EQ(i, 10); + + // Manually terminate the pipeline + iter->Stop(); +} + +/// Feature: Test Getters. +/// Description: includes tests for shape, type, size. +/// Expectation: the data is processed successfully. +TEST_F(MindDataTestPipeline, TestMulti30kGetters) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kGetters."; + + std::string train_en_file = datasets_root_path_ + "/testMulti30kDataset"; + std::string usage = "train"; + std::shared_ptr ds = Multi30k(train_en_file, usage, {"en", "de"}, 2, ShuffleMode::kFalse); + std::vector column_names = {"text","translation"}; + EXPECT_NE(ds, nullptr); + + EXPECT_EQ(ds->GetDatasetSize(), 2); + EXPECT_EQ(ds->GetColumnNames(), column_names); +} + +/// Feature: Test Multi30kDataset in distribution. +/// Description: test interface in a distributed state. +/// Expectation: the data is processed successfully. +TEST_F(MindDataTestPipeline, TestMulti30kDatasetDistribution) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kDatasetDistribution."; + + // Create a Multi30kFile Dataset, with single Multi30k file + std::string train_en_file = datasets_root_path_ + "/testMulti30kDataset"; + std::string usage = "train"; + std::shared_ptr ds = Multi30k(train_en_file, usage, {"en", "de"}, 0, ShuffleMode::kGlobal, 3, 2); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("text"), row.end()); + uint64_t i = 0; + while (row.size() != 0) { + auto text = row["text"]; + MS_LOG(INFO) << "Tensor text shape: " << text.Shape(); + i++; + ASSERT_OK(iter->GetNextRow(&row)); + } + + // Expect 1 samples + EXPECT_EQ(i, 1); + + // Manually terminate the pipeline + iter->Stop(); +} + +/// Feature: Error Test. +/// Description: test the wrong input. +/// Expectation: unable to read in data. +TEST_F(MindDataTestPipeline, TestMulti30kDatasetFailInvalidFilePath) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kDatasetFailInvalidFilePath."; + + // Create a Multi30k Dataset + // with invalid file path + std::string train_en_file = datasets_root_path_ + "/invalid/file.path"; + std::string usage = "train"; + std::shared_ptr ds = Multi30k(train_en_file, usage, {"en", "de"}); + EXPECT_NE(ds, nullptr); +} + +/// Feature: Error Test. +/// Description: test the wrong input. +/// Expectation: unable to read in data. +TEST_F(MindDataTestPipeline, TestMulti30kDatasetFailInvalidUsage) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kDatasetFailInvaildUsage."; + + // Create a Multi30k Dataset + // with invalid usage + std::string train_en_file = datasets_root_path_ + "/testMulti30kDataset"; + std::string usage = "invalid_usage"; + std::shared_ptr ds = Multi30k(train_en_file, usage, {"en", "de"}); + EXPECT_NE(ds, nullptr); +} + +/// Feature: Error Test. +/// Description: test the wrong input. +/// Expectation: unable to read in data. +TEST_F(MindDataTestPipeline, TestMulti30kDatasetFailInvalidLanguagePair) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kDatasetFailLanguagePair."; + + // Create a Multi30k Dataset + // with invalid usage + std::string train_en_file = datasets_root_path_ + "/testMulti30kDataset"; + std::string usage = "train"; + std::vector language_pair0 = {"ch", "ja"}; + std::shared_ptr ds0 = Multi30k(train_en_file, usage, language_pair0); + EXPECT_NE(ds0, nullptr); + std::vector language_pair1 = {"en", "de", "aa"}; + std::shared_ptr ds1 = Multi30k(train_en_file, usage, language_pair1); + EXPECT_NE(ds1, nullptr); +} + +/// Feature: Error Test. +/// Description: test the wrong input. +/// Expectation: unable to read in data. +TEST_F(MindDataTestPipeline, TestMulti30kDatasetFailInvalidNumSamples) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kDatasetFailInvalidNumSamples."; + + // Create a Multi30k Dataset + // with invalid samplers=-1 + std::string train_en_file = datasets_root_path_ + "/testMulti30kDataset"; + std::string usage = "train"; + std::shared_ptr ds = Multi30k(train_en_file, usage, {"en", "de"}, -1); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset. + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: TextFile number of samples cannot be negative + EXPECT_EQ(iter, nullptr); +} + +/// Feature: Error Test. +/// Description: test the wrong input. +/// Expectation: unable to read in data. +TEST_F(MindDataTestPipeline, TestMulti30kDatasetFailInvalidShards) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kDatasetFailInvalidShards."; + + // Create a Multi30k Dataset + // with invalid shards. + std::string train_en_file = datasets_root_path_ + "/testMulti30kDataset"; + std::string usage = "train"; + std::shared_ptr ds = Multi30k(train_en_file, usage, {"en", "de"}, 0, ShuffleMode::kFalse, 2, 3); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset. + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: TextFile number of samples cannot be negative + EXPECT_EQ(iter, nullptr); +} + +/// Feature: Error Test. +/// Description: test the wrong input. +/// Expectation: unable to read in data. +TEST_F(MindDataTestPipeline, TestMulti30kDatasetFailInvalidShardID) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kDatasetFailInvalidShardID."; + + // Create a Multi30k Dataset + // with invalid shard ID. + std::string train_en_file = datasets_root_path_ + "/testMulti30kDataset"; + std::string usage = "train"; + std::shared_ptr ds = Multi30k(train_en_file, usage, {"en", "de"}, 0, ShuffleMode::kFalse, 0, -1); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset. + std::shared_ptr iter = ds->CreateIterator(); + // Expect failure: TextFile number of samples cannot be negative + EXPECT_EQ(iter, nullptr); +} + +/// Feature: Error Test. +/// Description: test the wrong input. +/// Expectation: unable to read in data. +TEST_F(MindDataTestPipeline, TestMulti30kDatasetLanguagePair) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kDatasetLanguagePair."; + + std::string train_en_file = datasets_root_path_ + "/testMulti30kDataset"; + std::string usage = "train"; + std::shared_ptr ds = Multi30k(train_en_file, usage, {"de", "en"}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("translation"), row.end()); + std::vector expected = {"This is the first English sentence in train.", + "This is the second English sentence in train.", + "This is the third English sentence in train."}; + + uint64_t i = 0; + while (row.size() != 0) { + auto text = row["translation"]; + std::shared_ptr de_text; + ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); + std::string_view sv; + ASSERT_OK(de_text->GetItemAt(&sv, {})); + std::string ss(sv); + EXPECT_STREQ(ss.c_str(), expected[i].c_str()); + ASSERT_OK(iter->GetNextRow(&row)); + i++; + } + + EXPECT_EQ(i, 3); + + iter->Stop(); +} + +/// Feature: Test Multi30k Dataset(shufflemode=kFalse). +/// Description: test Multi30k Dataset interface with different ShuffleMode. +/// Expectation: the data is processed successfully. +TEST_F(MindDataTestPipeline, TestMulti30kDatasetShuffleFilesFalse) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kDatasetShuffleFilesFalse."; + + // Set configuration + uint32_t original_seed = GlobalContext::config_manager()->seed(); + uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers(); + MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers; + GlobalContext::config_manager()->set_seed(135); + GlobalContext::config_manager()->set_num_parallel_workers(1); + + std::string train_en_file = datasets_root_path_ + "/testMulti30kDataset"; + std::string usage = "train"; + std::shared_ptr ds = Multi30k(train_en_file, usage, {"en", "de"}, 0, ShuffleMode::kFalse); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset. + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("text"), row.end()); + std::vector expected = {"This is the first English sentence in train.", + "This is the second English sentence in train.", + "This is the third English sentence in train."}; + + uint64_t i = 0; + while (row.size() != 0) { + auto text = row["text"]; + std::shared_ptr de_text; + ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); + std::string_view sv; + ASSERT_OK(de_text->GetItemAt(&sv, {})); + std::string ss(sv); + MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); + // Compare against expected result + EXPECT_STREQ(ss.c_str(), expected[i].c_str()); + i++; + ASSERT_OK(iter->GetNextRow(&row)); + } + + EXPECT_EQ(i, 3); + + // Manually terminate the pipeline + iter->Stop(); + + // Restore configuration + GlobalContext::config_manager()->set_seed(original_seed); + GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers); +} + +/// Feature: Test Multi30k Dataset(shufflemode=kFiles). +/// Description: test Multi30k Dataset interface with different ShuffleMode. +/// Expectation: the data is processed successfully. +TEST_F(MindDataTestPipeline, TestMulti30kDatasetShuffleFilesFiles) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kDatasetShuffleFilesFiles."; + + // Set configuration + uint32_t original_seed = GlobalContext::config_manager()->seed(); + uint32_t original_num_parallel_workers = GlobalContext::config_manager()->num_parallel_workers(); + MS_LOG(DEBUG) << "ORIGINAL seed: " << original_seed << ", num_parallel_workers: " << original_num_parallel_workers; + GlobalContext::config_manager()->set_seed(135); + GlobalContext::config_manager()->set_num_parallel_workers(1); + + std::string train_en_file = datasets_root_path_ + "/testMulti30kDataset"; + std::string usage = "train"; + std::shared_ptr ds = Multi30k(train_en_file, usage, {"en", "de"}, 0, ShuffleMode::kFiles); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset. + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("text"), row.end()); + std::vector expected = {"This is the first English sentence in train.", + "This is the second English sentence in train.", + "This is the third English sentence in train."}; + + uint64_t i = 0; + while (row.size() != 0) { + auto text = row["text"]; + std::shared_ptr de_text; + ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); + std::string_view sv; + ASSERT_OK(de_text->GetItemAt(&sv, {})); + std::string ss(sv); + MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); + // Compare against expected result + EXPECT_STREQ(ss.c_str(), expected[i].c_str()); + i++; + ASSERT_OK(iter->GetNextRow(&row)); + } + + EXPECT_EQ(i, 3); + + // Manually terminate the pipeline + iter->Stop(); + + // Restore configuration + GlobalContext::config_manager()->set_seed(original_seed); + GlobalContext::config_manager()->set_num_parallel_workers(original_num_parallel_workers); +} + +/// Feature: Test Multi30k Dataset(shufflemode=kGlobal). +/// Description: test Multi30k Dataset interface with different ShuffleMode. +/// Expectation: the data is processed successfully. +TEST_F(MindDataTestPipeline, TestMulti30kDatasetShuffleFilesGlobal) { + MS_LOG(INFO) << "Doing MindDataTestPipeline-TestMulti30kDatasetShuffleFilesGlobal."; + + std::string train_en_file = datasets_root_path_ + "/testMulti30kDataset"; + std::string usage = "train"; + std::shared_ptr ds = Multi30k(train_en_file, usage, {"en", "de"}, 0, ShuffleMode::kGlobal); + EXPECT_NE(ds, nullptr); + + // Create an iterator over the result of the above dataset. + // This will trigger the creation of the Execution Tree and launch it. + std::shared_ptr iter = ds->CreateIterator(); + EXPECT_NE(iter, nullptr); + + // Iterate the dataset and get each row + std::unordered_map row; + ASSERT_OK(iter->GetNextRow(&row)); + + EXPECT_NE(row.find("text"), row.end()); + + uint64_t i = 0; + while (row.size() != 0) { + auto text = row["text"]; + std::shared_ptr de_text; + ASSERT_OK(Tensor::CreateFromMSTensor(text, &de_text)); + std::string_view sv; + ASSERT_OK(de_text->GetItemAt(&sv, {})); + std::string ss(sv); + MS_LOG(INFO) << "Text length: " << ss.length() << ", Text: " << ss.substr(0, 50); + i++; + ASSERT_OK(iter->GetNextRow(&row)); + } + + EXPECT_EQ(i, 3); + + // Manually terminate the pipeline + iter->Stop(); +} \ No newline at end of file diff --git a/tests/ut/data/dataset/testMulti30kDataset/mmt16_task1_test/test.de b/tests/ut/data/dataset/testMulti30kDataset/mmt16_task1_test/test.de new file mode 100644 index 00000000000..86ed8bd2bfb --- /dev/null +++ b/tests/ut/data/dataset/testMulti30kDataset/mmt16_task1_test/test.de @@ -0,0 +1,3 @@ +This is the first Germany sentence in test. +This is the second Germany sentence in test. +This is the third Germany sentence in test. diff --git a/tests/ut/data/dataset/testMulti30kDataset/mmt16_task1_test/test.en b/tests/ut/data/dataset/testMulti30kDataset/mmt16_task1_test/test.en new file mode 100644 index 00000000000..9fde08d12b7 --- /dev/null +++ b/tests/ut/data/dataset/testMulti30kDataset/mmt16_task1_test/test.en @@ -0,0 +1,3 @@ +This is the first English sentence in test. +This is the second English sentence in test. +This is the third English sentence in test. diff --git a/tests/ut/data/dataset/testMulti30kDataset/training/train.de b/tests/ut/data/dataset/testMulti30kDataset/training/train.de new file mode 100644 index 00000000000..4cf1209e769 --- /dev/null +++ b/tests/ut/data/dataset/testMulti30kDataset/training/train.de @@ -0,0 +1,3 @@ +This is the first Germany sentence in train. +This is the second Germany sentence in train. +This is the third Germany sentence in train. diff --git a/tests/ut/data/dataset/testMulti30kDataset/training/train.en b/tests/ut/data/dataset/testMulti30kDataset/training/train.en new file mode 100644 index 00000000000..ce5dfb2cf15 --- /dev/null +++ b/tests/ut/data/dataset/testMulti30kDataset/training/train.en @@ -0,0 +1,3 @@ +This is the first English sentence in train. +This is the second English sentence in train. +This is the third English sentence in train. diff --git a/tests/ut/data/dataset/testMulti30kDataset/validation/val.de b/tests/ut/data/dataset/testMulti30kDataset/validation/val.de new file mode 100644 index 00000000000..1cbfbcf40d2 --- /dev/null +++ b/tests/ut/data/dataset/testMulti30kDataset/validation/val.de @@ -0,0 +1,2 @@ +This is the first Germany sentence in valid. +This is the second Germany sentence in valid. diff --git a/tests/ut/data/dataset/testMulti30kDataset/validation/val.en b/tests/ut/data/dataset/testMulti30kDataset/validation/val.en new file mode 100644 index 00000000000..9138391d71a --- /dev/null +++ b/tests/ut/data/dataset/testMulti30kDataset/validation/val.en @@ -0,0 +1,2 @@ +This is the first English sentence in valid. +This is the second English sentence in valid. diff --git a/tests/ut/python/dataset/test_datasets_multi30k.py b/tests/ut/python/dataset/test_datasets_multi30k.py new file mode 100644 index 00000000000..ac25108ae80 --- /dev/null +++ b/tests/ut/python/dataset/test_datasets_multi30k.py @@ -0,0 +1,267 @@ +# Copyright 2022 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +import pytest + +import mindspore.dataset as ds +import mindspore.dataset.text.transforms as a_c_trans +from mindspore import log as logger +from util import config_get_set_num_parallel_workers, config_get_set_seed + +INVALID_FILE = '../data/dataset/testMulti30kDataset/invalid_dir' +DATA_ALL_FILE = '../data/dataset/testMulti30kDataset' + + +def test_data_file_multi30k_text(): + """ + Feature: Test Multi30k Dataset. + Description: read data from a single file. + Expectation: the data is processed successfully. + """ + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + original_seed = config_get_set_seed(987) + dataset = ds.Multi30kDataset(DATA_ALL_FILE, usage="train", shuffle=False) + count = 0 + line = ["This is the first English sentence in train.", + "This is the second English sentence in train.", + "This is the third English sentence in train." + ] + for i in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): + strs = i["text"].item().decode("utf8") + assert strs == line[count] + count += 1 + assert count == 3 + ds.config.set_num_parallel_workers(original_num_parallel_workers) + ds.config.set_seed(original_seed) + + +def test_data_file_multi30k_translation(): + """ + Feature: Test Multi30k Dataset. + Description: read data from a single file. + Expectation: the data is processed successfully. + """ + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + original_seed = config_get_set_seed(987) + dataset = ds.Multi30kDataset(DATA_ALL_FILE, usage="train", shuffle=False) + count = 0 + line = ["This is the first Germany sentence in train.", + "This is the second Germany sentence in train.", + "This is the third Germany sentence in train." + ] + for i in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): + strs = i["translation"].item().decode("utf8") + assert strs == line[count] + count += 1 + assert count == 3 + ds.config.set_num_parallel_workers(original_num_parallel_workers) + ds.config.set_seed(original_seed) + + +def test_all_file_multi30k(): + """ + Feature: Test Multi30k Dataset. + Description: read data from all file. + Expectation: the data is processed successfully. + """ + dataset = ds.Multi30kDataset(DATA_ALL_FILE) + count = 0 + for i in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): + logger.info("{}".format(i["text"])) + count += 1 + assert count == 8 + + +def test_dataset_num_samples_none(): + """ + Feature: Test Multi30k Dataset(num_samples = default). + Description: test get num_samples. + Expectation: the data is processed successfully. + """ + original_num_parallel_workers = config_get_set_num_parallel_workers(1) + original_seed = config_get_set_seed(987) + dataset = ds.Multi30kDataset(DATA_ALL_FILE, shuffle=False) + count = 0 + line = ["This is the first English sentence in test.", + "This is the second English sentence in test.", + "This is the third English sentence in test.", + "This is the first English sentence in train.", + "This is the second English sentence in train.", + "This is the third English sentence in train.", + "This is the first English sentence in valid.", + "This is the second English sentence in valid." + ] + for i in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): + strs = i["text"].item().decode("utf8") + assert strs == line[count] + count += 1 + assert count == 8 + ds.config.set_num_parallel_workers(original_num_parallel_workers) + ds.config.set_seed(original_seed) + + +def test_num_shards_multi30k(): + """ + Feature: Test Multi30k Dataset(num_shards = 3). + Description: test get num_samples. + Expectation: the data is processed successfully. + """ + dataset = ds.Multi30kDataset(DATA_ALL_FILE, usage='train', num_shards=3, shard_id=1) + count = 0 + for i in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): + logger.info("{}".format(i["text"])) + count += 1 + assert count == 1 + + +def test_multi30k_dataset_num_samples(): + """ + Feature: Test Multi30k Dataset(num_samples = 2). + Description: test get num_samples. + Expectation: the data is processed successfully. + """ + dataset = ds.Multi30kDataset(DATA_ALL_FILE, usage="test", num_samples=2) + count = 0 + for _ in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): + count += 1 + assert count == 2 + + +def test_multi30k_dataset_shuffle_files(): + """ + Feature: Test Multi30k Dataset. + Description: test get all files. + Expectation: the data is processed successfully. + """ + dataset = ds.Multi30kDataset(DATA_ALL_FILE, shuffle=True) + count = 0 + for _ in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): + count += 1 + assert count == 8 + + +def test_multi30k_dataset_shuffle_false(): + """ + Feature: Test Multi30k Dataset (shuffle = false). + Description: test get all files. + Expectation: the data is processed successfully. + """ + dataset = ds.Multi30kDataset(DATA_ALL_FILE, shuffle=False) + count = 0 + for _ in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): + count += 1 + assert count == 8 + + +def test_multi30k_dataset_repeat(): + """ + Feature: Test Multi30k in distribution (repeat 3 times). + Description: test in a distributed state. + Expectation: the data is processed successfully. + """ + dataset = ds.Multi30kDataset(DATA_ALL_FILE, usage='train') + dataset = dataset.repeat(3) + count = 0 + for _ in dataset.create_dict_iterator(num_epochs=1, output_numpy=True): + count += 1 + assert count == 9 + + +def test_multi30k_dataset_get_datasetsize(): + """ + Feature: Test Getters. + Description: test get_dataset_size of Multi30k dataset. + Expectation: the data is processed successfully. + """ + dataset = ds.Multi30kDataset(DATA_ALL_FILE) + size = dataset.get_dataset_size() + assert size == 8 + + +def test_multi30k_dataset_exceptions(): + """ + Feature: Test Multi30k Dataset. + Description: Test exceptions. + Expectation: Exception thrown to be caught + """ + with pytest.raises(ValueError) as error_info: + _ = ds.Multi30kDataset(INVALID_FILE) + assert "The folder ../data/dataset/testMulti30kDataset/invalid_dir does not exist or is not a directory or" \ + " permission denied" in str(error_info.value) + with pytest.raises(ValueError) as error_info: + _ = ds.Multi30kDataset(DATA_ALL_FILE, usage="INVALID") + assert "Input usage is not within the valid set of ['train', 'test', 'valid', 'all']." in str(error_info.value) + with pytest.raises(ValueError) as error_info: + _ = ds.Multi30kDataset(DATA_ALL_FILE, usage="test", language_pair=["ch", "ja"]) + assert "language_pair can only be ['en', 'de'] or ['en', 'de'], but got ['ch', 'ja']" in str(error_info.value) + with pytest.raises(ValueError) as error_info: + _ = ds.Multi30kDataset(DATA_ALL_FILE, usage="test", language_pair=["en", "en", "de"]) + assert "language_pair should be a list or tuple of length 2, but got 3" in str(error_info.value) + with pytest.raises(ValueError) as error_info: + _ = ds.Multi30kDataset(DATA_ALL_FILE, usage='test', num_samples=-1) + assert "num_samples exceeds the boundary between 0 and 9223372036854775807(INT64_MAX)!" in str(error_info.value) + + +def test_multi30k_dataset_en_pipeline(): + """ + Feature: Multi30kDataset + Description: test Multi30kDataset in pipeline mode + Expectation: the data is processed successfully + """ + expected = ["this is the first english sentence in train.", + "this is the second english sentence in train.", + "this is the third english sentence in train."] + dataset = ds.Multi30kDataset(DATA_ALL_FILE, 'train', shuffle=False) + filter_wikipedia_xml_op = a_c_trans.CaseFold() + dataset = dataset.map(input_columns=["text"], operations=filter_wikipedia_xml_op, num_parallel_workers=1) + count = 0 + for i in dataset.create_dict_iterator(output_numpy=True): + strs = i["text"].item().decode("utf8") + assert strs == expected[count] + count += 1 + + +def test_multi30k_dataset_de_pipeline(): + """ + Feature: Multi30kDataset + Description: test Multi30kDataset in pipeline mode + Expectation: the data is processed successfully + """ + expected = ["this is the first germany sentence in train.", + "this is the second germany sentence in train.", + "this is the third germany sentence in train."] + dataset = ds.Multi30kDataset(DATA_ALL_FILE, 'train', shuffle=False) + filter_wikipedia_xml_op = a_c_trans.CaseFold() + dataset = dataset.map(input_columns=["translation"], operations=filter_wikipedia_xml_op, num_parallel_workers=1) + count = 0 + for i in dataset.create_dict_iterator(output_numpy=True): + strs = i["translation"].item().decode("utf8") + assert strs == expected[count] + count += 1 + + +if __name__ == "__main__": + test_data_file_multi30k_text() + test_data_file_multi30k_translation() + test_all_file_multi30k() + test_dataset_num_samples_none() + test_num_shards_multi30k() + test_multi30k_dataset_num_samples() + test_multi30k_dataset_shuffle_files() + test_multi30k_dataset_shuffle_false() + test_multi30k_dataset_repeat() + test_multi30k_dataset_get_datasetsize() + test_multi30k_dataset_exceptions() + test_multi30k_dataset_en_pipeline() + test_multi30k_dataset_de_pipeline()