From 8ab2af36a7ac64df5d26b3305f919b7a095ebab2 Mon Sep 17 00:00:00 2001 From: Parallels Date: Mon, 9 May 2022 20:13:20 +0800 Subject: [PATCH] Add file configuration util --- .../distributed/recovery/configuration.h | 53 +++++++++++++ .../recovery/file_configuration.cc | 71 +++++++++++++++++ .../distributed/recovery/file_configuration.h | 53 +++++++++++++ tests/ut/cpp/CMakeLists.txt | 14 ++-- .../recovery/file_configuration_test.cc | 78 +++++++++++++++++++ 5 files changed, 264 insertions(+), 5 deletions(-) create mode 100644 mindspore/ccsrc/distributed/recovery/configuration.h create mode 100644 mindspore/ccsrc/distributed/recovery/file_configuration.cc create mode 100644 mindspore/ccsrc/distributed/recovery/file_configuration.h create mode 100644 tests/ut/cpp/distributed/recovery/file_configuration_test.cc diff --git a/mindspore/ccsrc/distributed/recovery/configuration.h b/mindspore/ccsrc/distributed/recovery/configuration.h new file mode 100644 index 00000000000..17c1e242c2b --- /dev/null +++ b/mindspore/ccsrc/distributed/recovery/configuration.h @@ -0,0 +1,53 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DISTRIBUTED_RECOVERY_CONFIGURATION_H_ +#define MINDSPORE_CCSRC_DISTRIBUTED_RECOVERY_CONFIGURATION_H_ + +#include + +namespace mindspore { +namespace distributed { +namespace recovery { +// An abstract configuration class to store and recover key-value style metadata, which could be stored in a local file +// or other storages. +class Configuration { + public: + Configuration() = default; + virtual ~Configuration() = default; + + // These two methods should be implemented in sub-class to allocate and release resources owned by this configuration + // instance. + virtual bool Initialize() = 0; + virtual bool Finalize() { return true; } + + // Get the configuration item value by the specified key, returns the default value if the key does not + // exist. + virtual std::string Get(const std::string &key, const std::string &defaultvalue) const = 0; + + // Persist the key-value pair metadata. + virtual void Put(const std::string &key, const std::string &value) = 0; + + // Check whether the specified configuration key exists. + virtual bool Exists(const std::string &key) const = 0; + + // Flush all the key-value pairs in memory into the specific sub-class's storage. + virtual bool Flush() = 0; +}; +} // namespace recovery +} // namespace distributed +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DISTRIBUTED_RECOVERY_CONFIGURATION_H_ diff --git a/mindspore/ccsrc/distributed/recovery/file_configuration.cc b/mindspore/ccsrc/distributed/recovery/file_configuration.cc new file mode 100644 index 00000000000..43c2b74e124 --- /dev/null +++ b/mindspore/ccsrc/distributed/recovery/file_configuration.cc @@ -0,0 +1,71 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include "utils/log_adapter.h" +#include "distributed/persistent/storage/file_io_utils.h" +#include "distributed/recovery/file_configuration.h" + +namespace mindspore { +namespace distributed { +namespace recovery { +bool FileConfiguration::Initialize() { + // If there is no local file, create an empty one. + if (!storage::FileIOUtils::IsFileOrDirExist(file_)) { + storage::FileIOUtils::CreateFile(file_); + RETURN_IF_FALSE_WITH_LOG(storage::FileIOUtils::IsFileOrDirExist(file_), + "Failed to create the local configuration file " + file_); + // There is already an existing local file, load and parse the values. + } else { + std::ifstream in_stream(file_); + try { + in_stream >> values_; + in_stream.close(); + } catch (nlohmann::json::exception &e) { + in_stream.close(); + std::string exception = e.what(); + MS_LOG(ERROR) << "Failed to parse the existing local file: " << file_ << ", the exception: " << exception; + return false; + } + } + return true; +} + +std::string FileConfiguration::Get(const std::string &key, const std::string &defaultvalue) const { + if (!values_.contains(key)) { + return defaultvalue; + } + return values_.at(key); +} + +void FileConfiguration::Put(const std::string &key, const std::string &value) { values_[key] = value; } + +bool FileConfiguration::Exists(const std::string &key) const { return values_.contains(key); } + +bool FileConfiguration::Flush() { + if (!storage::FileIOUtils::IsFileOrDirExist(file_)) { + MS_LOG(EXCEPTION) << "The local configuration file : " << file_ << " does not exist."; + } + // Write all the configuration items into local file. + std::ofstream output_file(file_); + output_file << values_.dump(); + output_file.close(); + + return true; +} +} // namespace recovery +} // namespace distributed +} // namespace mindspore diff --git a/mindspore/ccsrc/distributed/recovery/file_configuration.h b/mindspore/ccsrc/distributed/recovery/file_configuration.h new file mode 100644 index 00000000000..6de7f74b61e --- /dev/null +++ b/mindspore/ccsrc/distributed/recovery/file_configuration.h @@ -0,0 +1,53 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_CCSRC_DISTRIBUTED_RECOVERY_FILE_CONFIGURATION_H_ +#define MINDSPORE_CCSRC_DISTRIBUTED_RECOVERY_FILE_CONFIGURATION_H_ + +#include +#include +#include "distributed/recovery/configuration.h" + +namespace mindspore { +namespace distributed { +namespace recovery { +// Local file for saving and restore metadata. +class FileConfiguration : public Configuration { + public: + explicit FileConfiguration(const std::string &path) : file_(path) {} + ~FileConfiguration() = default; + + bool Initialize() override; + + std::string Get(const std::string &key, const std::string &defaultvalue) const override; + + void Put(const std::string &key, const std::string &value) override; + + bool Exists(const std::string &key) const override; + + bool Flush() override; + + private: + // The full path of the local configuration file. + std::string file_; + + // All the key-value pairs managed by this configuration are stored in json format. + nlohmann::json values_; +}; +} // namespace recovery +} // namespace distributed +} // namespace mindspore +#endif // MINDSPORE_CCSRC_DISTRIBUTED_RECOVERY_FILE_CONFIGURATION_H_ diff --git a/tests/ut/cpp/CMakeLists.txt b/tests/ut/cpp/CMakeLists.txt index ba00296d17a..835f9e37165 100644 --- a/tests/ut/cpp/CMakeLists.txt +++ b/tests/ut/cpp/CMakeLists.txt @@ -69,11 +69,6 @@ if(ENABLE_MINDDATA) ./vm/*.cc ./ps/*.cc ./fl/*.cc - ./distributed/persistent/*.cc - ./distributed/rpc/tcp/*.cc - ./distributed/cluster/*.cc - ./distributed/cluster/topology/*.cc - ./distributed/recovery/*.cc ./cxx_api/*.cc ./tbe/*.cc ./mindapi/*.cc @@ -85,6 +80,15 @@ if(ENABLE_MINDDATA) ./debug/*.cc) list(APPEND UT_SRCS ${UT_SRCS_DEBUG}) endif() + if(NOT ENABLE_CPU OR WIN32 OR APPLE) + file(GLOB_RECURSE UT_DISTRIBUTED_SRCS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} + ./distributed/persistent/*.cc + ./distributed/rpc/tcp/*.cc + ./distributed/cluster/*.cc + ./distributed/cluster/topology/*.cc + ./distributed/recovery/*.cc) + list(APPEND UT_SRCS ${UT_DISTRIBUTED_SRCS}) + endif() if(NOT ENABLE_PYTHON) set(PYTHON_RELATED_SRCS dataset/filter_op_test.cc diff --git a/tests/ut/cpp/distributed/recovery/file_configuration_test.cc b/tests/ut/cpp/distributed/recovery/file_configuration_test.cc new file mode 100644 index 00000000000..fd98727e364 --- /dev/null +++ b/tests/ut/cpp/distributed/recovery/file_configuration_test.cc @@ -0,0 +1,78 @@ +/** + * Copyright 2022 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +#include +#include "common/common_test.h" +#include "distributed/persistent/storage/file_io_utils.h" +#include "distributed/recovery/file_configuration.h" + +namespace mindspore { +namespace distributed { +namespace recovery { +class TestFileConfiguration : public UT::Common { + public: + TestFileConfiguration() = default; + virtual ~TestFileConfiguration() = default; + + void SetUp() override {} + void TearDown() override {} +}; + +/// Feature: test save and load function of file configuration. +/// Description: Create a new local file configuration, put some key-value pairs. +/// Expectation: All the key-value pairs could be saved to the local file and the local file could be loaded +/// successfully. +TEST_F(TestFileConfiguration, SaveAndLoadLocalFile) { + std::string local_file = "metadata.json"; + char *dir = getcwd(nullptr, 0); + EXPECT_NE(nullptr, dir); + + std::string path = dir; + free(dir); + dir = nullptr; + + std::string full_file_path = path + "/" + local_file; + if (storage::FileIOUtils::IsFileOrDirExist(full_file_path)) { + remove(full_file_path.c_str()); + } + EXPECT_TRUE(!storage::FileIOUtils::IsFileOrDirExist(full_file_path)); + + std::unique_ptr config = std::make_unique(full_file_path); + EXPECT_NE(nullptr, config); + EXPECT_TRUE(config->Initialize()); + + for (int i = 0; i < 10; ++i) { + config->Put("key_" + std::to_string(i), "value_" + std::to_string(i)); + } + config->Flush(); + config.reset(); + + std::unique_ptr recovery_config = std::make_unique(full_file_path); + EXPECT_NE(nullptr, recovery_config); + EXPECT_TRUE(recovery_config->Initialize()); + + for (int i = 0; i < 10; ++i) { + EXPECT_EQ("value_" + std::to_string(i), recovery_config->Get("key_" + std::to_string(i), "")); + } + EXPECT_FALSE(recovery_config->Exists("key_11")); + recovery_config.reset(); + + remove(full_file_path.c_str()); +} +} // namespace recovery +} // namespace distributed +} // namespace mindspore