Add file configuration util

mindspore/ccsrc/distributed/recovery/configuration.h

@@ -0,0 +1,53 @@
+/**
+ * Copyright 2022 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_CCSRC_DISTRIBUTED_RECOVERY_CONFIGURATION_H_
+#define MINDSPORE_CCSRC_DISTRIBUTED_RECOVERY_CONFIGURATION_H_
+
+#include <string>
+
+namespace mindspore {
+namespace distributed {
+namespace recovery {
+// An abstract configuration class to store and recover key-value style metadata, which could be stored in a local file
+// or other storages.
+class Configuration {
+ public:
+  Configuration() = default;
+  virtual ~Configuration() = default;
+
+  // These two methods should be implemented in sub-class to allocate and release resources owned by this configuration
+  // instance.
+  virtual bool Initialize() = 0;
+  virtual bool Finalize() { return true; }
+
+  // Get the configuration item value by the specified key, returns the default value if the key does not
+  // exist.
+  virtual std::string Get(const std::string &key, const std::string &defaultvalue) const = 0;
+
+  // Persist the key-value pair metadata.
+  virtual void Put(const std::string &key, const std::string &value) = 0;
+
+  // Check whether the specified configuration key exists.
+  virtual bool Exists(const std::string &key) const = 0;
+
+  // Flush all the key-value pairs in memory into the specific sub-class's storage.
+  virtual bool Flush() = 0;
+};
+}  // namespace recovery
+}  // namespace distributed
+}  // namespace mindspore
+#endif  // MINDSPORE_CCSRC_DISTRIBUTED_RECOVERY_CONFIGURATION_H_

mindspore/ccsrc/distributed/recovery/file_configuration.cc

@@ -0,0 +1,71 @@
+/**
+ * Copyright 2022 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <fstream>
+#include "utils/log_adapter.h"
+#include "distributed/persistent/storage/file_io_utils.h"
+#include "distributed/recovery/file_configuration.h"
+
+namespace mindspore {
+namespace distributed {
+namespace recovery {
+bool FileConfiguration::Initialize() {
+  // If there is no local file, create an empty one.
+  if (!storage::FileIOUtils::IsFileOrDirExist(file_)) {
+    storage::FileIOUtils::CreateFile(file_);
+    RETURN_IF_FALSE_WITH_LOG(storage::FileIOUtils::IsFileOrDirExist(file_),
+                             "Failed to create the local configuration file " + file_);
+    // There is already an existing local file, load and parse the values.
+  } else {
+    std::ifstream in_stream(file_);
+    try {
+      in_stream >> values_;
+      in_stream.close();
+    } catch (nlohmann::json::exception &e) {
+      in_stream.close();
+      std::string exception = e.what();
+      MS_LOG(ERROR) << "Failed to parse the existing local file: " << file_ << ", the exception: " << exception;
+      return false;
+    }
+  }
+  return true;
+}
+
+std::string FileConfiguration::Get(const std::string &key, const std::string &defaultvalue) const {
+  if (!values_.contains(key)) {
+    return defaultvalue;
+  }
+  return values_.at(key);
+}
+
+void FileConfiguration::Put(const std::string &key, const std::string &value) { values_[key] = value; }
+
+bool FileConfiguration::Exists(const std::string &key) const { return values_.contains(key); }
+
+bool FileConfiguration::Flush() {
+  if (!storage::FileIOUtils::IsFileOrDirExist(file_)) {
+    MS_LOG(EXCEPTION) << "The local configuration file : " << file_ << " does not exist.";
+  }
+  // Write all the configuration items into local file.
+  std::ofstream output_file(file_);
+  output_file << values_.dump();
+  output_file.close();
+
+  return true;
+}
+}  // namespace recovery
+}  // namespace distributed
+}  // namespace mindspore

mindspore/ccsrc/distributed/recovery/file_configuration.h

@@ -0,0 +1,53 @@
+/**
+ * Copyright 2022 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef MINDSPORE_CCSRC_DISTRIBUTED_RECOVERY_FILE_CONFIGURATION_H_
+#define MINDSPORE_CCSRC_DISTRIBUTED_RECOVERY_FILE_CONFIGURATION_H_
+
+#include <string>
+#include <nlohmann/json.hpp>
+#include "distributed/recovery/configuration.h"
+
+namespace mindspore {
+namespace distributed {
+namespace recovery {
+// Local file for saving and restore metadata.
+class FileConfiguration : public Configuration {
+ public:
+  explicit FileConfiguration(const std::string &path) : file_(path) {}
+  ~FileConfiguration() = default;
+
+  bool Initialize() override;
+
+  std::string Get(const std::string &key, const std::string &defaultvalue) const override;
+
+  void Put(const std::string &key, const std::string &value) override;
+
+  bool Exists(const std::string &key) const override;
+
+  bool Flush() override;
+
+ private:
+  // The full path of the local configuration file.
+  std::string file_;
+
+  // All the key-value pairs managed by this configuration are stored in json format.
+  nlohmann::json values_;
+};
+}  // namespace recovery
+}  // namespace distributed
+}  // namespace mindspore
+#endif  // MINDSPORE_CCSRC_DISTRIBUTED_RECOVERY_FILE_CONFIGURATION_H_

tests/ut/cpp/CMakeLists.txt

@@ -69,11 +69,6 @@ if(ENABLE_MINDDATA)
             ./vm/*.cc
             ./ps/*.cc
             ./fl/*.cc
-            ./distributed/persistent/*.cc
-            ./distributed/rpc/tcp/*.cc
-            ./distributed/cluster/*.cc
-            ./distributed/cluster/topology/*.cc
-            ./distributed/recovery/*.cc
             ./cxx_api/*.cc
             ./tbe/*.cc
             ./mindapi/*.cc
@@ -85,6 +80,15 @@ if(ENABLE_MINDDATA)
                 ./debug/*.cc)
         list(APPEND UT_SRCS ${UT_SRCS_DEBUG})
     endif()
+    if(NOT ENABLE_CPU OR WIN32 OR APPLE)
+        file(GLOB_RECURSE UT_DISTRIBUTED_SRCS RELATIVE ${CMAKE_CURRENT_SOURCE_DIR}
+                ./distributed/persistent/*.cc
+                ./distributed/rpc/tcp/*.cc
+                ./distributed/cluster/*.cc
+                ./distributed/cluster/topology/*.cc
+                ./distributed/recovery/*.cc)
+        list(APPEND UT_SRCS ${UT_DISTRIBUTED_SRCS})
+    endif()
     if(NOT ENABLE_PYTHON)
         set(PYTHON_RELATED_SRCS
                 dataset/filter_op_test.cc

tests/ut/cpp/distributed/recovery/file_configuration_test.cc

@@ -0,0 +1,78 @@
+/**
+ * Copyright 2022 Huawei Technologies Co., Ltd
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <unistd.h>
+#include <cstdio>
+#include "common/common_test.h"
+#include "distributed/persistent/storage/file_io_utils.h"
+#include "distributed/recovery/file_configuration.h"
+
+namespace mindspore {
+namespace distributed {
+namespace recovery {
+class TestFileConfiguration : public UT::Common {
+ public:
+  TestFileConfiguration() = default;
+  virtual ~TestFileConfiguration() = default;
+
+  void SetUp() override {}
+  void TearDown() override {}
+};
+
+/// Feature: test save and load function of file configuration.
+/// Description: Create a new local file configuration, put some key-value pairs.
+/// Expectation: All the key-value pairs could be saved to the local file and the local file could be loaded
+/// successfully.
+TEST_F(TestFileConfiguration, SaveAndLoadLocalFile) {
+  std::string local_file = "metadata.json";
+  char *dir = getcwd(nullptr, 0);
+  EXPECT_NE(nullptr, dir);
+
+  std::string path = dir;
+  free(dir);
+  dir = nullptr;
+
+  std::string full_file_path = path + "/" + local_file;
+  if (storage::FileIOUtils::IsFileOrDirExist(full_file_path)) {
+    remove(full_file_path.c_str());
+  }
+  EXPECT_TRUE(!storage::FileIOUtils::IsFileOrDirExist(full_file_path));
+
+  std::unique_ptr<FileConfiguration> config = std::make_unique<FileConfiguration>(full_file_path);
+  EXPECT_NE(nullptr, config);
+  EXPECT_TRUE(config->Initialize());
+
+  for (int i = 0; i < 10; ++i) {
+    config->Put("key_" + std::to_string(i), "value_" + std::to_string(i));
+  }
+  config->Flush();
+  config.reset();
+
+  std::unique_ptr<FileConfiguration> recovery_config = std::make_unique<FileConfiguration>(full_file_path);
+  EXPECT_NE(nullptr, recovery_config);
+  EXPECT_TRUE(recovery_config->Initialize());
+
+  for (int i = 0; i < 10; ++i) {
+    EXPECT_EQ("value_" + std::to_string(i), recovery_config->Get("key_" + std::to_string(i), ""));
+  }
+  EXPECT_FALSE(recovery_config->Exists("key_11"));
+  recovery_config.reset();
+
+  remove(full_file_path.c_str());
+}
+}  // namespace recovery
+}  // namespace distributed
+}  // namespace mindspore