wbtiger
/
ByConity
mirror of https://github.com/ByConity/ByConity
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

attach all system tables

This commit is contained in:
Alexander Tokmakov 2020-08-13 20:09:40 +03:00
parent a6ff049eec
commit 0f1c4bc9c4
4 changed files with 94 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Databases/DatabaseAtomic.cpp
View File

@ -366,6 +366,8 @@ void DatabaseAtomic::loadStoredObjects(Context & context, bool has_force_restore
std::lock_guard lock{mutex};
table_names = table_name_to_path;
}
Poco::File(path_to_table_symlinks).createDirectories();
for (const auto & table : table_names)
tryCreateSymlink(table.first, table.second);
}

122
src/Storages/System/StorageSystemDetachedParts.cpp
View File

@ -12,85 +12,63 @@
namespace DB
{
/**
* Implements system table 'detached_parts' which allows to get information
* about detached data parts for tables of MergeTree family.
* We don't use StorageSystemPartsBase, because it introduces virtual _state
* column and column aliases which we don't need.
*/
class StorageSystemDetachedParts final :
public ext::shared_ptr_helper<StorageSystemDetachedParts>,
public IStorage
StorageSystemDetachedParts::StorageSystemDetachedParts(const StorageID & table_id_)
: IStorage(table_id_)
{
friend struct ext::shared_ptr_helper<StorageSystemDetachedParts>;
public:
std::string getName() const override { return "SystemDetachedParts"; }
StorageInMemoryMetadata storage_metadata;
storage_metadata.setColumns(ColumnsDescription{{
{"database", std::make_shared<DataTypeString>()},
{"table", std::make_shared<DataTypeString>()},
{"partition_id", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>())},
{"name", std::make_shared<DataTypeString>()},
{"disk", std::make_shared<DataTypeString>()},
{"reason", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>())},
{"min_block_number", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt64>())},
{"max_block_number", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt64>())},
{"level", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt32>())}
}});
setInMemoryMetadata(storage_metadata);
}
protected:
explicit StorageSystemDetachedParts(const StorageID & table_id_)
: IStorage(table_id_)
Pipes StorageSystemDetachedParts::read(
const Names & /* column_names */,
const StorageMetadataPtr & metadata_snapshot,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum /*processed_stage*/,
const size_t /*max_block_size*/,
const unsigned /*num_streams*/)
{
StoragesInfoStream stream(query_info, context);
/// Create the result.
Block block = metadata_snapshot->getSampleBlock();
MutableColumns new_columns = block.cloneEmptyColumns();
while (StoragesInfo info = stream.next())
{
StorageInMemoryMetadata storage_metadata;
storage_metadata.setColumns(ColumnsDescription{{
{"database", std::make_shared<DataTypeString>()},
{"table", std::make_shared<DataTypeString>()},
{"partition_id", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>())},
{"name", std::make_shared<DataTypeString>()},
{"disk", std::make_shared<DataTypeString>()},
{"reason", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeString>())},
{"min_block_number", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt64>())},
{"max_block_number", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt64>())},
{"level", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt32>())}
}});
setInMemoryMetadata(storage_metadata);
}
Pipes read(
const Names & /* column_names */,
const StorageMetadataPtr & metadata_snapshot,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum /*processed_stage*/,
const size_t /*max_block_size*/,
const unsigned /*num_streams*/) override
{
StoragesInfoStream stream(query_info, context);
/// Create the result.
Block block = metadata_snapshot->getSampleBlock();
MutableColumns new_columns = block.cloneEmptyColumns();
while (StoragesInfo info = stream.next())
const auto parts = info.data->getDetachedParts();
for (const auto & p : parts)
{
const auto parts = info.data->getDetachedParts();
for (const auto & p : parts)
{
size_t i = 0;
new_columns[i++]->insert(info.database);
new_columns[i++]->insert(info.table);
new_columns[i++]->insert(p.valid_name ? p.partition_id : Field());
new_columns[i++]->insert(p.dir_name);
new_columns[i++]->insert(p.disk);
new_columns[i++]->insert(p.valid_name ? p.prefix : Field());
new_columns[i++]->insert(p.valid_name ? p.min_block : Field());
new_columns[i++]->insert(p.valid_name ? p.max_block : Field());
new_columns[i++]->insert(p.valid_name ? p.level : Field());
}
size_t i = 0;
new_columns[i++]->insert(info.database);
new_columns[i++]->insert(info.table);
new_columns[i++]->insert(p.valid_name ? p.partition_id : Field());
new_columns[i++]->insert(p.dir_name);
new_columns[i++]->insert(p.disk);
new_columns[i++]->insert(p.valid_name ? p.prefix : Field());
new_columns[i++]->insert(p.valid_name ? p.min_block : Field());
new_columns[i++]->insert(p.valid_name ? p.max_block : Field());
new_columns[i++]->insert(p.valid_name ? p.level : Field());
}
UInt64 num_rows = new_columns.at(0)->size();
Chunk chunk(std::move(new_columns), num_rows);
Pipes pipes;
pipes.emplace_back(std::make_shared<SourceFromSingleChunk>(std::move(block), std::move(chunk)));
return pipes;
}
};
StoragePtr
createDetachedPartsTable()
{
return StorageSystemDetachedParts::create(StorageID{"system", "detached_parts"});
UInt64 num_rows = new_columns.at(0)->size();
Chunk chunk(std::move(new_columns), num_rows);
Pipes pipes;
pipes.emplace_back(std::make_shared<SourceFromSingleChunk>(std::move(block), std::move(chunk)));
return pipes;
}
}

29
src/Storages/System/StorageSystemDetachedParts.h
View File

@ -1,10 +1,35 @@
#pragma once
#include <Storages/IStorage_fwd.h>
#include <Storages/IStorage.h>
namespace DB
{
StoragePtr createDetachedPartsTable();
/**
* Implements system table 'detached_parts' which allows to get information
* about detached data parts for tables of MergeTree family.
* We don't use StorageSystemPartsBase, because it introduces virtual _state
* column and column aliases which we don't need.
*/
class StorageSystemDetachedParts final :
public ext::shared_ptr_helper<StorageSystemDetachedParts>,
public IStorage
{
friend struct ext::shared_ptr_helper<StorageSystemDetachedParts>;
public:
std::string getName() const override { return "SystemDetachedParts"; }
protected:
explicit StorageSystemDetachedParts(const StorageID & table_id_);
Pipes read(
const Names & /* column_names */,
const StorageMetadataPtr & metadata_snapshot,
const SelectQueryInfo & query_info,
const Context & context,
QueryProcessingStage::Enum /*processed_stage*/,
const size_t /*max_block_size*/,
const unsigned /*num_streams*/) override;
};
}

28
src/Storages/System/attachSystemTables.cpp
View File

@ -69,19 +69,23 @@ namespace DB
namespace
{
template<typename StorageT>
void attach(IDatabase & system_database, const String & table_name)
template<typename StorageT, typename... StorageArgs>
void attach(IDatabase & system_database, const String & table_name, StorageArgs && ... args)
{
if (system_database.getUUID() == UUIDHelpers::Nil)
{
/// Attach to Ordinary database
auto table_id = StorageID(DatabaseCatalog::SYSTEM_DATABASE, table_name);
system_database.attachTable(table_name, StorageT::create(table_id));
system_database.attachTable(table_name, StorageT::create(table_id, std::forward<StorageArgs>(args)...));
}
else
{
/// Attach to Atomic database
/// NOTE: UUIDs are not persistent, but it's ok since no data are stored on disk for these storages
/// and path is actually not used
auto table_id = StorageID(DatabaseCatalog::SYSTEM_DATABASE, table_name, UUIDHelpers::generateV4());
String path = "store/" + DatabaseCatalog::getPathForUUID(table_id.uuid);
system_database.attachTable(table_name, StorageT::create(table_id), path);
system_database.attachTable(table_name, StorageT::create(table_id, std::forward<StorageArgs>(args)...), path);
}
}
@ -92,12 +96,10 @@ void attach(IDatabase & system_database, const String & table_name)
void attachSystemTablesLocal(IDatabase & system_database)
{
attach<StorageSystemOne>(system_database, "one");
//system_database.attachTable("numbers", StorageSystemNumbers::create(StorageID("system", "numbers"), false));
//system_database.attachTable("numbers_mt", StorageSystemNumbers::create(StorageID("system", "numbers_mt"), true));
//system_database.attachTable("zeros", StorageSystemZeros::create(StorageID("system", "zeros"), false));
//system_database.attachTable("zeros_mt", StorageSystemZeros::create(StorageID("system", "zeros_mt"), true));
attach<StorageSystemNumbers>(system_database, "numbers", false);
attach<StorageSystemNumbers>(system_database, "numbers_mt", true);
attach<StorageSystemZeros>(system_database, "zeros", false);
attach<StorageSystemZeros>(system_database, "zeros_mt", true);
attach<StorageSystemDatabases>(system_database, "databases");
attach<StorageSystemTables>(system_database, "tables");
attach<StorageSystemColumns>(system_database, "columns");
@ -141,7 +143,7 @@ void attachSystemTablesServer(IDatabase & system_database, bool has_zookeeper)
attachSystemTablesLocal(system_database);
attach<StorageSystemParts>(system_database, "parts");
//attach<>(system_database, "detached_parts", createDetachedPartsTable());
attach<StorageSystemDetachedParts>(system_database, "detached_parts");
attach<StorageSystemPartsColumns>(system_database, "parts_columns");
attach<StorageSystemDisks>(system_database, "disks");
attach<StorageSystemStoragePolicies>(system_database, "storage_policies");
@ -162,9 +164,9 @@ void attachSystemTablesServer(IDatabase & system_database, bool has_zookeeper)
attach<StorageSystemZooKeeper>(system_database, "zookeeper");
}
void attachSystemTablesAsync(IDatabase & /*system_database*/, AsynchronousMetrics & /*async_metrics*/)
void attachSystemTablesAsync(IDatabase & system_database, AsynchronousMetrics & async_metrics)
{
//system_database.attachTable("asynchronous_metrics", StorageSystemAsynchronousMetrics::create("asynchronous_metrics", async_metrics));
attach<StorageSystemAsynchronousMetrics>(system_database, "asynchronous_metrics", async_metrics);
}
}