wbtiger
/
ByConity
mirror of https://github.com/ByConity/ByConity
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add 'params' column to system.user_directories table.

This commit is contained in:
Vitaly Baranov 2020-09-15 01:51:53 +03:00
parent 9e3797b252
commit 0f53b449fd
8 changed files with 70 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/Access/AccessControlManager.cpp
View File

@ -186,7 +186,7 @@ void AccessControlManager::addUsersConfigStorage(
{ {
if (auto users_config_storage = typeid_cast<std::shared_ptr<UsersConfigAccessStorage>>(storage)) if (auto users_config_storage = typeid_cast<std::shared_ptr<UsersConfigAccessStorage>>(storage))
{ {
if (users_config_storage->getStoragePath() == users_config_path_) if (users_config_storage->isPathEqual(users_config_path_))
return; return;
} }
} }
@ -229,7 +229,7 @@ void AccessControlManager::addDiskStorage(const String & storage_name_, const St
{ {
if (auto disk_storage = typeid_cast<std::shared_ptr<DiskAccessStorage>>(storage)) if (auto disk_storage = typeid_cast<std::shared_ptr<DiskAccessStorage>>(storage))
{ {
if (disk_storage->isStoragePathEqual(directory_)) if (disk_storage->isPathEqual(directory_))
{ {
if (readonly_) if (readonly_)
disk_storage->setReadOnly(readonly_); disk_storage->setReadOnly(readonly_);

20
src/Access/DiskAccessStorage.cpp
View File

@ -33,6 +33,9 @@
#include <Interpreters/InterpreterShowGrantsQuery.h> #include <Interpreters/InterpreterShowGrantsQuery.h>
#include <Common/quoteString.h> #include <Common/quoteString.h>
#include <Core/Defines.h> #include <Core/Defines.h>
#include <Poco/JSON/JSON.h>
#include <Poco/JSON/Object.h>
#include <Poco/JSON/Stringifier.h>
#include <boost/range/adaptor/map.hpp> #include <boost/range/adaptor/map.hpp>
#include <boost/range/algorithm/copy.hpp> #include <boost/range/algorithm/copy.hpp>
#include <boost/range/algorithm_ext/push_back.hpp> #include <boost/range/algorithm_ext/push_back.hpp>
@ -342,9 +345,22 @@ DiskAccessStorage::~DiskAccessStorage()
} }
bool DiskAccessStorage::isStoragePathEqual(const String & directory_path_) const String DiskAccessStorage::getStorageParamsJSON() const
{ {
return getStoragePath() == makeDirectoryPathCanonical(directory_path_); std::lock_guard lock{mutex};
Poco::JSON::Object json;
json.set("path", directory_path);
if (readonly)
json.set("readonly", readonly.load());
std::ostringstream oss;
Poco::JSON::Stringifier::stringify(json, oss);
return oss.str();
}
bool DiskAccessStorage::isPathEqual(const String & directory_path_) const
{
return getPath() == makeDirectoryPathCanonical(directory_path_);
} }

7
src/Access/DiskAccessStorage.h
View File

@ -18,12 +18,13 @@ public:
~DiskAccessStorage() override; ~DiskAccessStorage() override;
const char * getStorageType() const override { return STORAGE_TYPE; } const char * getStorageType() const override { return STORAGE_TYPE; }
String getStorageParamsJSON() const override;
String getStoragePath() const override { return directory_path; } String getPath() const { return directory_path; }
bool isStoragePathEqual(const String & directory_path_) const; bool isPathEqual(const String & directory_path_) const;
void setReadOnly(bool readonly_) { readonly = readonly_; } void setReadOnly(bool readonly_) { readonly = readonly_; }
bool isStorageReadOnly() const override { return readonly; } bool isReadOnly() const { return readonly; }
private: private:
std::optional<UUID> findImpl(EntityType type, const String & name) const override; std::optional<UUID> findImpl(EntityType type, const String & name) const override;

5
src/Access/IAccessStorage.h
View File

@ -25,8 +25,9 @@ public:
/// Returns the name of this storage. /// Returns the name of this storage.
const String & getStorageName() const { return storage_name; } const String & getStorageName() const { return storage_name; }
virtual const char * getStorageType() const = 0; virtual const char * getStorageType() const = 0;
virtual String getStoragePath() const { return {}; }
virtual bool isStorageReadOnly() const { return false; } /// Returns a JSON with the parameters of the storage. It's up to the storage type to fill the JSON.
virtual String getStorageParamsJSON() const { return "{}"; }
using EntityType = IAccessEntity::Type; using EntityType = IAccessEntity::Type;
using EntityTypeInfo = IAccessEntity::TypeInfo; using EntityTypeInfo = IAccessEntity::TypeInfo;

22
src/Access/UsersConfigAccessStorage.cpp
View File

@ -10,6 +10,9 @@
#include <Core/Settings.h> #include <Core/Settings.h>
#include <Poco/Util/AbstractConfiguration.h> #include <Poco/Util/AbstractConfiguration.h>
#include <Poco/MD5Engine.h> #include <Poco/MD5Engine.h>
#include <Poco/JSON/JSON.h>
#include <Poco/JSON/Object.h>
#include <Poco/JSON/Stringifier.h>
#include <common/logger_useful.h> #include <common/logger_useful.h>
#include <boost/range/algorithm/copy.hpp> #include <boost/range/algorithm/copy.hpp>
#include <boost/range/adaptor/map.hpp> #include <boost/range/adaptor/map.hpp>
@ -482,12 +485,29 @@ UsersConfigAccessStorage::UsersConfigAccessStorage(const String & storage_name_,
UsersConfigAccessStorage::~UsersConfigAccessStorage() = default; UsersConfigAccessStorage::~UsersConfigAccessStorage() = default;
String UsersConfigAccessStorage::getStoragePath() const String UsersConfigAccessStorage::getStorageParamsJSON() const
{
std::lock_guard lock{load_mutex};
Poco::JSON::Object json;
if (!path.empty())
json.set("path", path);
std::ostringstream oss;
Poco::JSON::Stringifier::stringify(json, oss);
return oss.str();
}
String UsersConfigAccessStorage::getPath() const
{ {
std::lock_guard lock{load_mutex}; std::lock_guard lock{load_mutex};
return path; return path;
} }
bool UsersConfigAccessStorage::isPathEqual(const String & path_) const
{
return getPath() == path_;
}
void UsersConfigAccessStorage::setConfig(const Poco::Util::AbstractConfiguration & config) void UsersConfigAccessStorage::setConfig(const Poco::Util::AbstractConfiguration & config)
{ {

6
src/Access/UsersConfigAccessStorage.h
View File

@ -26,8 +26,10 @@ public:
~UsersConfigAccessStorage() override; ~UsersConfigAccessStorage() override;
const char * getStorageType() const override { return STORAGE_TYPE; } const char * getStorageType() const override { return STORAGE_TYPE; }
String getStoragePath() const override; String getStorageParamsJSON() const override;
bool isStorageReadOnly() const override { return true; }
String getPath() const;
bool isPathEqual(const String & path_) const;
void setConfig(const Poco::Util::AbstractConfiguration & config); void setConfig(const Poco::Util::AbstractConfiguration & config);

12
src/Storages/System/StorageSystemUserDirectories.cpp
View File

@ -15,8 +15,7 @@ NamesAndTypesList StorageSystemUserDirectories::getNamesAndTypes()
NamesAndTypesList names_and_types{ NamesAndTypesList names_and_types{
{"name", std::make_shared<DataTypeString>()}, {"name", std::make_shared<DataTypeString>()},
{"type", std::make_shared<DataTypeString>()}, {"type", std::make_shared<DataTypeString>()},
{"path", std::make_shared<DataTypeString>()}, {"params", std::make_shared<DataTypeString>()},
{"readonly", std::make_shared<DataTypeUInt8>()},
{"precedence", std::make_shared<DataTypeUInt64>()}, {"precedence", std::make_shared<DataTypeUInt64>()},
}; };
return names_and_types; return names_and_types;
@ -31,21 +30,18 @@ void StorageSystemUserDirectories::fillData(MutableColumns & res_columns, const
size_t column_index = 0; size_t column_index = 0;
auto & column_name = assert_cast<ColumnString &>(*res_columns[column_index++]); auto & column_name = assert_cast<ColumnString &>(*res_columns[column_index++]);
auto & column_type = assert_cast<ColumnString &>(*res_columns[column_index++]); auto & column_type = assert_cast<ColumnString &>(*res_columns[column_index++]);
auto & column_path = assert_cast<ColumnString &>(*res_columns[column_index++]); auto & column_params = assert_cast<ColumnString &>(*res_columns[column_index++]);
auto & column_readonly = assert_cast<ColumnUInt8 &>(*res_columns[column_index++]);
auto & column_precedence = assert_cast<ColumnUInt64 &>(*res_columns[column_index++]); auto & column_precedence = assert_cast<ColumnUInt64 &>(*res_columns[column_index++]);
auto add_row = [&](const IAccessStorage & storage, size_t precedence) auto add_row = [&](const IAccessStorage & storage, size_t precedence)
{ {
const String & name = storage.getStorageName(); const String & name = storage.getStorageName();
std::string_view type = storage.getStorageType(); std::string_view type = storage.getStorageType();
const String & path = storage.getStoragePath(); String params = storage.getStorageParamsJSON();
bool readonly = storage.isStorageReadOnly();
column_name.insertData(name.data(), name.length()); column_name.insertData(name.data(), name.length());
column_type.insertData(type.data(), type.length()); column_type.insertData(type.data(), type.length());
column_path.insertData(path.data(), path.length()); column_params.insertData(params.data(), params.length());
column_readonly.insert(readonly);
column_precedence.insert(precedence); column_precedence.insert(precedence);
}; };

28
tests/integration/test_user_directories/test.py
View File

@ -24,40 +24,40 @@ def started_cluster():
def test_old_style(): def test_old_style():
node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/old_style.xml"), '/etc/clickhouse-server/config.d/z.xml') node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/old_style.xml"), '/etc/clickhouse-server/config.d/z.xml')
node.restart_clickhouse() node.restart_clickhouse()
assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", "/etc/clickhouse-server/users2.xml", 1, 1], assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users2.xml"}', 1],
["local directory", "local directory", "/var/lib/clickhouse/access2/", 0, 2]]) ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access2\\\\/"}', 2]])
def test_local_directories(): def test_local_directories():
node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/local_directories.xml"), '/etc/clickhouse-server/config.d/z.xml') node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/local_directories.xml"), '/etc/clickhouse-server/config.d/z.xml')
node.restart_clickhouse() node.restart_clickhouse()
assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", "/etc/clickhouse-server/users3.xml", 1, 1], assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users3.xml"}', 1],
["local directory", "local directory", "/var/lib/clickhouse/access3/", 0, 2], ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access3\\\\/"}', 2],
["local directory (ro)", "local directory", "/var/lib/clickhouse/access3-ro/", 1, 3]]) ["local directory (ro)", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access3-ro\\\\/","readonly":true}', 3]])
def test_relative_path(): def test_relative_path():
node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/relative_path.xml"), '/etc/clickhouse-server/config.d/z.xml') node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/relative_path.xml"), '/etc/clickhouse-server/config.d/z.xml')
node.restart_clickhouse() node.restart_clickhouse()
assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", "/etc/clickhouse-server/users4.xml", 1, 1]]) assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users4.xml"}', 1]])
def test_memory(): def test_memory():
node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/memory.xml"), '/etc/clickhouse-server/config.d/z.xml') node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/memory.xml"), '/etc/clickhouse-server/config.d/z.xml')
node.restart_clickhouse() node.restart_clickhouse()
assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", "/etc/clickhouse-server/users5.xml", 1, 1], assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users5.xml"}', 1],
["memory", "memory", "", 0, 2]]) ["memory", "memory", '{}', 2]])
def test_mixed_style(): def test_mixed_style():
node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/mixed_style.xml"), '/etc/clickhouse-server/config.d/z.xml') node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/mixed_style.xml"), '/etc/clickhouse-server/config.d/z.xml')
node.restart_clickhouse() node.restart_clickhouse()
assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", "/etc/clickhouse-server/users6.xml", 1, 1], assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users6.xml"}', 1],
["local directory", "local directory", "/var/lib/clickhouse/access6/", 0, 2], ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access6\\\\/"}', 2],
["local directory", "local directory", "/var/lib/clickhouse/access6a/", 0, 3], ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access6a\\\\/"}', 3],
["memory", "memory", "", 0, 4]]) ["memory", "memory", '{}', 4]])
def test_duplicates(): def test_duplicates():
node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/duplicates.xml"), '/etc/clickhouse-server/config.d/z.xml') node.copy_file_to_container(os.path.join(SCRIPT_DIR, "configs/duplicates.xml"), '/etc/clickhouse-server/config.d/z.xml')
node.restart_clickhouse() node.restart_clickhouse()
assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", "/etc/clickhouse-server/users7.xml", 1, 1], assert node.query("SELECT * FROM system.user_directories") == TSV([["users.xml", "users.xml", '{"path":"\\\\/etc\\\\/clickhouse-server\\\\/users7.xml"}', 1],
["local directory", "local directory", "/var/lib/clickhouse/access7/", 0, 2]]) ["local directory", "local directory", '{"path":"\\\\/var\\\\/lib\\\\/clickhouse\\\\/access7\\\\/"}', 2]])