wbtiger
/
ByConity
mirror of https://github.com/ByConity/ByConity
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into base64-update

This commit is contained in:
Alexey Milovidov 2019-12-28 02:27:32 +03:00
parent 063682ef9c 9e8c3acffc
commit b04586ef55
320 changed files with 7158 additions and 2819 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitmodules vendored
View File

@ -128,6 +128,9 @@
[submodule "contrib/icu"]
path = contrib/icu
url = https://github.com/unicode-org/icu.git
[submodule "contrib/flatbuffers"]
path = contrib/flatbuffers
url = https://github.com/google/flatbuffers.git
[submodule "contrib/libc-headers"]
path = contrib/libc-headers
url = https://github.com/ClickHouse-Extras/libc-headers.git

4
cmake/find/parquet.cmake
View File

@ -54,10 +54,12 @@ elseif(NOT MISSING_INTERNAL_PARQUET_LIBRARY AND NOT OS_FREEBSD)
endif()
if(${USE_STATIC_LIBRARIES})
set(FLATBUFFERS_LIBRARY flatbuffers)
set(ARROW_LIBRARY arrow_static)
set(PARQUET_LIBRARY parquet_static)
set(THRIFT_LIBRARY thrift_static)
else()
set(FLATBUFFERS_LIBRARY flatbuffers_shared)
set(ARROW_LIBRARY arrow_shared)
set(PARQUET_LIBRARY parquet_shared)
if(USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE)
@ -74,7 +76,7 @@ endif()
endif()
if(USE_PARQUET)
message(STATUS "Using Parquet: ${ARROW_LIBRARY}:${ARROW_INCLUDE_DIR} ; ${PARQUET_LIBRARY}:${PARQUET_INCLUDE_DIR} ; ${THRIFT_LIBRARY}")
message(STATUS "Using Parquet: ${ARROW_LIBRARY}:${ARROW_INCLUDE_DIR} ; ${PARQUET_LIBRARY}:${PARQUET_INCLUDE_DIR} ; ${THRIFT_LIBRARY} ; ${FLATBUFFERS_LIBRARY}")
else()
message(STATUS "Building without Parquet support")
endif()

2
contrib/CMakeLists.txt vendored
View File

@ -159,6 +159,8 @@ if (USE_INTERNAL_PARQUET_LIBRARY_NATIVE_CMAKE)
set (ARROW_PARQUET ON CACHE INTERNAL "")
set (ARROW_VERBOSE_THIRDPARTY_BUILD ON CACHE INTERNAL "")
set (ARROW_BUILD_SHARED 1 CACHE INTERNAL "")
set (ARROW_BUILD_UTILITIES OFF CACHE INTERNAL "")
set (ARROW_BUILD_INTEGRATION OFF CACHE INTERNAL "")
set (ARROW_BOOST_HEADER_ONLY ON CACHE INTERNAL "")
set (Boost_FOUND 1 CACHE INTERNAL "")
if (MAKE_STATIC_LIBRARIES)

2
contrib/arrow vendored

@ -1 +1 @@
Subproject commit 87ac6fddaf21d0b4ee8b8090533ff293db0da1b4
Subproject commit b789226ccb2124285792107c758bb3b40b3d082a

166
contrib/arrow-cmake/CMakeLists.txt
View File

@ -1,3 +1,5 @@
include(ExternalProject)
# === thrift
set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/thrift/lib/cpp)
@ -70,6 +72,77 @@ add_custom_command(OUTPUT orc_proto.pb.h orc_proto.pb.cc
--cpp_out="${CMAKE_CURRENT_BINARY_DIR}"
"${PROTO_DIR}/orc_proto.proto")
# === flatbuffers
##############################################################
# fbs - Step 1: build flatbuffers lib and flatc compiler
##############################################################
set(FLATBUFFERS_SRC_DIR ${ClickHouse_SOURCE_DIR}/contrib/flatbuffers)
set(FLATBUFFERS_BINARY_DIR ${ClickHouse_BINARY_DIR}/contrib/flatbuffers)
set(FLATBUFFERS_INCLUDE_DIR ${FLATBUFFERS_SRC_DIR}/include)
set(FLATBUFFERS_COMPILER "${FLATBUFFERS_BINARY_DIR}/flatc")
# set flatbuffers CMake options
if (${USE_STATIC_LIBRARIES})
set(FLATBUFFERS_BUILD_FLATLIB ON CACHE BOOL "Enable the build of the flatbuffers library")
set(FLATBUFFERS_BUILD_SHAREDLIB OFF CACHE BOOL "Disable the build of the flatbuffers shared library")
else ()
set(FLATBUFFERS_BUILD_SHAREDLIB ON CACHE BOOL "Enable the build of the flatbuffers shared library")
set(FLATBUFFERS_BUILD_FLATLIB OFF CACHE BOOL "Disable the build of the flatbuffers library")
endif ()
set(FLATBUFFERS_BUILD_FLATC ON CACHE BOOL "Build flatbuffers compiler")
set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "Skip flatbuffers tests")
add_subdirectory(${FLATBUFFERS_SRC_DIR} "${FLATBUFFERS_BINARY_DIR}")
###################################
# fbs - Step 2: compile *.fbs files
###################################
set(ARROW_IPC_SRC_DIR ${ARROW_SRC_DIR}/arrow/ipc)
set(ARROW_FORMAT_SRC_DIR ${ARROW_SRC_DIR}/../../format)
set(ARROW_GENERATED_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/arrow_gen_headers)
set(FLATBUFFERS_COMPILED_OUT_DIR ${ARROW_GENERATED_INCLUDE_DIR}/arrow/ipc)
set(FBS_OUTPUT_FILES
"${FLATBUFFERS_COMPILED_OUT_DIR}/File_generated.h"
"${FLATBUFFERS_COMPILED_OUT_DIR}/Message_generated.h"
"${FLATBUFFERS_COMPILED_OUT_DIR}/feather_generated.h"
"${FLATBUFFERS_COMPILED_OUT_DIR}/Schema_generated.h"
"${FLATBUFFERS_COMPILED_OUT_DIR}/SparseTensor_generated.h"
"${FLATBUFFERS_COMPILED_OUT_DIR}/Tensor_generated.h")
set(FBS_SRC
${ARROW_FORMAT_SRC_DIR}/Message.fbs
${ARROW_FORMAT_SRC_DIR}/File.fbs
${ARROW_FORMAT_SRC_DIR}/Schema.fbs
${ARROW_FORMAT_SRC_DIR}/Tensor.fbs
${ARROW_FORMAT_SRC_DIR}/SparseTensor.fbs
${ARROW_IPC_SRC_DIR}/feather.fbs)
foreach (FIL ${FBS_SRC})
get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
list(APPEND ABS_FBS_SRC ${ABS_FIL})
endforeach ()
message(STATUS "FLATBUFFERS_LIBRARY: ${FLATBUFFERS_LIBRARY}, FLATBUFFERS_COMPILER: ${FLATBUFFERS_COMPILER}")
message(STATUS "FLATBUFFERS_COMPILED_OUT_DIR: ${FLATBUFFERS_COMPILED_OUT_DIR}")
message(STATUS "flatc: ${FLATBUFFERS_COMPILER} -c -o ${FLATBUFFERS_COMPILED_OUT_DIR}/ ${ABS_FBS_SRC}")
add_custom_command(OUTPUT ${FBS_OUTPUT_FILES}
COMMAND ${FLATBUFFERS_COMPILER}
-c
-o
${FLATBUFFERS_COMPILED_OUT_DIR}/
${ABS_FBS_SRC}
DEPENDS flatc ${ABS_FBS_SRC}
COMMENT "Running flatc compiler on ${ABS_FBS_SRC}"
VERBATIM)
add_custom_target(metadata_fbs DEPENDS ${FBS_OUTPUT_FILES})
add_dependencies(metadata_fbs flatc)
# arrow-cmake cmake file calling orc cmake subroutine which detects certain compiler features.
# Apple Clang compiler failed to compile this code without specifying c++11 standard.
# As result these compiler features detected as absent. In result it failed to compile orc itself.
@ -86,6 +159,7 @@ configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/A
set(ORC_SRCS
${ARROW_SRC_DIR}/arrow/adapters/orc/adapter.cc
${ARROW_SRC_DIR}/arrow/adapters/orc/adapter_util.cc
${ORC_SOURCE_SRC_DIR}/Exceptions.cc
${ORC_SOURCE_SRC_DIR}/OrcFile.cc
${ORC_SOURCE_SRC_DIR}/Reader.cc
@ -119,11 +193,29 @@ set(ORC_SRCS
# === arrow
set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src/arrow)
configure_file("${LIBRARY_DIR}/util/config.h.cmake" "${CMAKE_CURRENT_SOURCE_DIR}/cpp/src/arrow/util/config.h")
# arrow/cpp/src/arrow/CMakeLists.txt
set(ARROW_SRCS
${LIBRARY_DIR}/array.cc
${LIBRARY_DIR}/buffer.cc
${LIBRARY_DIR}/builder.cc
${LIBRARY_DIR}/compare.cc
${LIBRARY_DIR}/extension_type.cc
${LIBRARY_DIR}/memory_pool.cc
${LIBRARY_DIR}/pretty_print.cc
${LIBRARY_DIR}/record_batch.cc
${LIBRARY_DIR}/result.cc
${LIBRARY_DIR}/scalar.cc
${LIBRARY_DIR}/sparse_tensor.cc
${LIBRARY_DIR}/status.cc
${LIBRARY_DIR}/table_builder.cc
${LIBRARY_DIR}/table.cc
${LIBRARY_DIR}/tensor.cc
${LIBRARY_DIR}/type.cc
${LIBRARY_DIR}/visitor.cc
${LIBRARY_DIR}/array/builder_adaptive.cc
${LIBRARY_DIR}/array/builder_base.cc
${LIBRARY_DIR}/array/builder_binary.cc
@ -131,46 +223,56 @@ set(ARROW_SRCS
${LIBRARY_DIR}/array/builder_dict.cc
${LIBRARY_DIR}/array/builder_nested.cc
${LIBRARY_DIR}/array/builder_primitive.cc
${LIBRARY_DIR}/buffer.cc
${LIBRARY_DIR}/compare.cc
${LIBRARY_DIR}/memory_pool.cc
${LIBRARY_DIR}/pretty_print.cc
${LIBRARY_DIR}/record_batch.cc
${LIBRARY_DIR}/status.cc
${LIBRARY_DIR}/table.cc
${LIBRARY_DIR}/table_builder.cc
${LIBRARY_DIR}/tensor.cc
${LIBRARY_DIR}/sparse_tensor.cc
${LIBRARY_DIR}/type.cc
${LIBRARY_DIR}/visitor.cc
${LIBRARY_DIR}/array/builder_union.cc
${LIBRARY_DIR}/array/concatenate.cc
${LIBRARY_DIR}/array/dict_internal.cc
${LIBRARY_DIR}/array/diff.cc
${LIBRARY_DIR}/csv/converter.cc
${LIBRARY_DIR}/csv/chunker.cc
${LIBRARY_DIR}/csv/column-builder.cc
${LIBRARY_DIR}/csv/column_builder.cc
${LIBRARY_DIR}/csv/options.cc
${LIBRARY_DIR}/csv/parser.cc
${LIBRARY_DIR}/csv/reader.cc
${LIBRARY_DIR}/ipc/dictionary.cc
${LIBRARY_DIR}/ipc/feather.cc
${LIBRARY_DIR}/ipc/message.cc
${LIBRARY_DIR}/ipc/metadata_internal.cc
${LIBRARY_DIR}/ipc/options.cc
${LIBRARY_DIR}/ipc/reader.cc
${LIBRARY_DIR}/ipc/writer.cc
${LIBRARY_DIR}/io/buffered.cc
${LIBRARY_DIR}/io/compressed.cc
${LIBRARY_DIR}/io/file.cc
${LIBRARY_DIR}/io/interfaces.cc
${LIBRARY_DIR}/io/memory.cc
${LIBRARY_DIR}/io/readahead.cc
${LIBRARY_DIR}/io/slow.cc
${LIBRARY_DIR}/util/bit-util.cc
${LIBRARY_DIR}/util/basic_decimal.cc
${LIBRARY_DIR}/util/bit_util.cc
${LIBRARY_DIR}/util/compression.cc
${LIBRARY_DIR}/util/cpu-info.cc
${LIBRARY_DIR}/util/compression_lz4.cc
${LIBRARY_DIR}/util/compression_snappy.cc
${LIBRARY_DIR}/util/compression_zlib.cc
${LIBRARY_DIR}/util/compression_zstd.cc
${LIBRARY_DIR}/util/cpu_info.cc
${LIBRARY_DIR}/util/decimal.cc
${LIBRARY_DIR}/util/int-util.cc
${LIBRARY_DIR}/util/io-util.cc
${LIBRARY_DIR}/util/logging.cc
${LIBRARY_DIR}/util/int_util.cc
${LIBRARY_DIR}/util/io_util.cc
${LIBRARY_DIR}/util/key_value_metadata.cc
${LIBRARY_DIR}/util/task-group.cc
${LIBRARY_DIR}/util/thread-pool.cc
${LIBRARY_DIR}/util/logging.cc
${LIBRARY_DIR}/util/memory.cc
${LIBRARY_DIR}/util/string_builder.cc
${LIBRARY_DIR}/util/string.cc
${LIBRARY_DIR}/util/task_group.cc
${LIBRARY_DIR}/util/thread_pool.cc
${LIBRARY_DIR}/util/trie.cc
${LIBRARY_DIR}/util/utf8.cc
${LIBRARY_DIR}/vendored/base64.cpp
${ORC_SRCS}
)
@ -179,7 +281,7 @@ set(ARROW_SRCS ${ARROW_SRCS}
${LIBRARY_DIR}/compute/kernels/boolean.cc
${LIBRARY_DIR}/compute/kernels/cast.cc
${LIBRARY_DIR}/compute/kernels/hash.cc
${LIBRARY_DIR}/compute/kernels/util-internal.cc
${LIBRARY_DIR}/compute/kernels/util_internal.cc
)
if (LZ4_INCLUDE_DIR AND LZ4_LIBRARY)
@ -221,6 +323,12 @@ endif()
add_library(${ARROW_LIBRARY} ${ARROW_SRCS})
# Arrow dependencies
add_dependencies(${ARROW_LIBRARY} ${FLATBUFFERS_LIBRARY} metadata_fbs)
target_link_libraries(${ARROW_LIBRARY} PRIVATE boost_system_internal boost_filesystem_internal boost_regex_internal)
target_link_libraries(${ARROW_LIBRARY} PRIVATE ${FLATBUFFERS_LIBRARY})
if (USE_INTERNAL_PROTOBUF_LIBRARY)
add_dependencies(${ARROW_LIBRARY} protoc)
endif ()
@ -248,6 +356,8 @@ target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_BUILD_SRC_DIR})
target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_BUILD_INCLUDE_DIR})
target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_ADDITION_SOURCE_DIR})
target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ARROW_SRC_DIR})
target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ARROW_GENERATED_INCLUDE_DIR})
# === parquet
@ -255,23 +365,25 @@ set(LIBRARY_DIR ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src/parquet)
# arrow/cpp/src/parquet/CMakeLists.txt
set(PARQUET_SRCS
${LIBRARY_DIR}/arrow/reader.cc
${LIBRARY_DIR}/arrow/record_reader.cc
${LIBRARY_DIR}/arrow/reader_internal.cc
${LIBRARY_DIR}/arrow/schema.cc
${LIBRARY_DIR}/arrow/writer.cc
${LIBRARY_DIR}/bloom_filter.cc
${LIBRARY_DIR}/column_reader.cc
${LIBRARY_DIR}/column_scanner.cc
${LIBRARY_DIR}/column_writer.cc
${LIBRARY_DIR}/deprecated_io.cc
${LIBRARY_DIR}/encoding.cc
${LIBRARY_DIR}/file_reader.cc
${LIBRARY_DIR}/file_writer.cc
${LIBRARY_DIR}/metadata.cc
${LIBRARY_DIR}/murmur3.cc
${LIBRARY_DIR}/platform.cc
${LIBRARY_DIR}/printer.cc
${LIBRARY_DIR}/properties.cc
${LIBRARY_DIR}/schema.cc
${LIBRARY_DIR}/statistics.cc
${LIBRARY_DIR}/types.cc
${LIBRARY_DIR}/util/comparison.cc
${LIBRARY_DIR}/util/memory.cc
)
#list(TRANSFORM PARQUET_SRCS PREPEND ${LIBRARY_DIR}/) # cmake 3.12
list(APPEND PARQUET_SRCS
@ -292,7 +404,7 @@ endif()
# === tools
set(TOOLS_DIR ${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/tools/parquet)
set(PARQUET_TOOLS parquet-dump-schema parquet-reader parquet-scan)
set(PARQUET_TOOLS parquet_dump_schema parquet_reader parquet_scan)
foreach (TOOL ${PARQUET_TOOLS})
add_executable(${TOOL} ${TOOLS_DIR}/${TOOL}.cc)
target_link_libraries(${TOOL} PRIVATE ${PARQUET_LIBRARY})

24
contrib/arrow-cmake/cpp/src/arrow/util/config.h Normal file
View File

@ -0,0 +1,24 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#define ARROW_VERSION_MAJOR
#define ARROW_VERSION_MINOR
#define ARROW_VERSION_PATCH
#define ARROW_VERSION ((ARROW_VERSION_MAJOR * 1000) + ARROW_VERSION_MINOR) * 1000 + ARROW_VERSION_PATCH
/* #undef DOUBLE_CONVERSION_HAS_CASE_INSENSIBILITY */
/* #undef GRPCPP_PP_INCLUDE */

2
contrib/arrow-cmake/cpp/src/parquet/parquet_constants.cpp
View File

@ -1,5 +1,5 @@
/**
* Autogenerated by Thrift Compiler (0.11.0)
* Autogenerated by Thrift Compiler (0.12.0)
*
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
* @generated

2
contrib/arrow-cmake/cpp/src/parquet/parquet_constants.h
View File

@ -1,5 +1,5 @@
/**
* Autogenerated by Thrift Compiler (0.11.0)
* Autogenerated by Thrift Compiler (0.12.0)
*
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
* @generated

2048
contrib/arrow-cmake/cpp/src/parquet/parquet_types.cpp
View File

File diff suppressed because it is too large Load Diff

375
contrib/arrow-cmake/cpp/src/parquet/parquet_types.h
View File

@ -1,5 +1,5 @@
/**
* Autogenerated by Thrift Compiler (0.11.0)
* Autogenerated by Thrift Compiler (0.12.0)
*
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
* @generated
@ -17,7 +17,7 @@
#include <thrift/stdcxx.h>
#include "parquet/util/windows_compatibility.h"
#include "parquet/windows_compatibility.h"
namespace parquet { namespace format {
@ -161,6 +161,8 @@ class MilliSeconds;
class MicroSeconds;
class NanoSeconds;
class TimeUnit;
class TimestampType;
@ -215,14 +217,14 @@ class OffsetIndex;
class ColumnIndex;
class FileMetaData;
class AesGcmV1;
class AesGcmCtrV1;
class EncryptionAlgorithm;
class FileMetaData;
class FileCryptoMetaData;
typedef struct _Statistics__isset {
@ -629,10 +631,42 @@ void swap(MicroSeconds &a, MicroSeconds &b);
std::ostream& operator<<(std::ostream& out, const MicroSeconds& obj);
class NanoSeconds : public virtual ::apache::thrift::TBase {
public:
NanoSeconds(const NanoSeconds&);
NanoSeconds& operator=(const NanoSeconds&);
NanoSeconds() {
}
virtual ~NanoSeconds() throw();
bool operator == (const NanoSeconds & /* rhs */) const
{
return true;
}
bool operator != (const NanoSeconds &rhs) const {
return !(*this == rhs);
}
bool operator < (const NanoSeconds & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
virtual void printTo(std::ostream& out) const;
};
void swap(NanoSeconds &a, NanoSeconds &b);
std::ostream& operator<<(std::ostream& out, const NanoSeconds& obj);
typedef struct _TimeUnit__isset {
_TimeUnit__isset() : MILLIS(false), MICROS(false) {}
_TimeUnit__isset() : MILLIS(false), MICROS(false), NANOS(false) {}
bool MILLIS :1;
bool MICROS :1;
bool NANOS :1;
} _TimeUnit__isset;
class TimeUnit : public virtual ::apache::thrift::TBase {
@ -646,6 +680,7 @@ class TimeUnit : public virtual ::apache::thrift::TBase {
virtual ~TimeUnit() throw();
MilliSeconds MILLIS;
MicroSeconds MICROS;
NanoSeconds NANOS;
_TimeUnit__isset __isset;
@ -653,6 +688,8 @@ class TimeUnit : public virtual ::apache::thrift::TBase {
void __set_MICROS(const MicroSeconds& val);
void __set_NANOS(const NanoSeconds& val);
bool operator == (const TimeUnit & rhs) const
{
if (__isset.MILLIS != rhs.__isset.MILLIS)
@ -663,6 +700,10 @@ class TimeUnit : public virtual ::apache::thrift::TBase {
return false;
else if (__isset.MICROS && !(MICROS == rhs.MICROS))
return false;
if (__isset.NANOS != rhs.__isset.NANOS)
return false;
else if (__isset.NANOS && !(NANOS == rhs.NANOS))
return false;
return true;
}
bool operator != (const TimeUnit &rhs) const {
@ -867,7 +908,7 @@ void swap(BsonType &a, BsonType &b);
std::ostream& operator<<(std::ostream& out, const BsonType& obj);
typedef struct _LogicalType__isset {
_LogicalType__isset() : STRING(false), MAP(false), LIST(false), ENUM(false), DECIMAL(false), DATE(false), TIME(false), TIMESTAMP(false), INTEGER(false), UNKNOWN(false), JSON(false), BSON(false) {}
_LogicalType__isset() : STRING(false), MAP(false), LIST(false), ENUM(false), DECIMAL(false), DATE(false), TIME(false), TIMESTAMP(false), INTEGER(false), UNKNOWN(false), JSON(false), BSON(false), UUID(false) {}
bool STRING :1;
bool MAP :1;
bool LIST :1;
@ -880,6 +921,7 @@ typedef struct _LogicalType__isset {
bool UNKNOWN :1;
bool JSON :1;
bool BSON :1;
bool UUID :1;
} _LogicalType__isset;
class LogicalType : public virtual ::apache::thrift::TBase {
@ -903,6 +945,7 @@ class LogicalType : public virtual ::apache::thrift::TBase {
NullType UNKNOWN;
JsonType JSON;
BsonType BSON;
UUIDType UUID;
_LogicalType__isset __isset;
@ -930,6 +973,8 @@ class LogicalType : public virtual ::apache::thrift::TBase {
void __set_BSON(const BsonType& val);
void __set_UUID(const UUIDType& val);
bool operator == (const LogicalType & rhs) const
{
if (__isset.STRING != rhs.__isset.STRING)
@ -980,6 +1025,10 @@ class LogicalType : public virtual ::apache::thrift::TBase {
return false;
else if (__isset.BSON && !(BSON == rhs.BSON))
return false;
if (__isset.UUID != rhs.__isset.UUID)
return false;
else if (__isset.UUID && !(UUID == rhs.UUID))
return false;
return true;
}
bool operator != (const LogicalType &rhs) const {
@ -1722,8 +1771,8 @@ void swap(EncryptionWithFooterKey &a, EncryptionWithFooterKey &b);
std::ostream& operator<<(std::ostream& out, const EncryptionWithFooterKey& obj);
typedef struct _EncryptionWithColumnKey__isset {
_EncryptionWithColumnKey__isset() : column_key_metadata(false) {}
bool column_key_metadata :1;
_EncryptionWithColumnKey__isset() : key_metadata(false) {}
bool key_metadata :1;
} _EncryptionWithColumnKey__isset;
class EncryptionWithColumnKey : public virtual ::apache::thrift::TBase {
@ -1731,26 +1780,26 @@ class EncryptionWithColumnKey : public virtual ::apache::thrift::TBase {
EncryptionWithColumnKey(const EncryptionWithColumnKey&);
EncryptionWithColumnKey& operator=(const EncryptionWithColumnKey&);
EncryptionWithColumnKey() : column_key_metadata() {
EncryptionWithColumnKey() : key_metadata() {
}
virtual ~EncryptionWithColumnKey() throw();
std::vector<std::string> path_in_schema;
std::string column_key_metadata;
std::string key_metadata;
_EncryptionWithColumnKey__isset __isset;
void __set_path_in_schema(const std::vector<std::string> & val);
void __set_column_key_metadata(const std::string& val);
void __set_key_metadata(const std::string& val);
bool operator == (const EncryptionWithColumnKey & rhs) const
{
if (!(path_in_schema == rhs.path_in_schema))
return false;
if (__isset.column_key_metadata != rhs.__isset.column_key_metadata)
if (__isset.key_metadata != rhs.__isset.key_metadata)
return false;
else if (__isset.column_key_metadata && !(column_key_metadata == rhs.column_key_metadata))
else if (__isset.key_metadata && !(key_metadata == rhs.key_metadata))
return false;
return true;
}
@ -1823,14 +1872,15 @@ void swap(ColumnCryptoMetaData &a, ColumnCryptoMetaData &b);
std::ostream& operator<<(std::ostream& out, const ColumnCryptoMetaData& obj);
typedef struct _ColumnChunk__isset {
_ColumnChunk__isset() : file_path(false), meta_data(false), offset_index_offset(false), offset_index_length(false), column_index_offset(false), column_index_length(false), crypto_meta_data(false) {}
_ColumnChunk__isset() : file_path(false), meta_data(false), offset_index_offset(false), offset_index_length(false), column_index_offset(false), column_index_length(false), crypto_metadata(false), encrypted_column_metadata(false) {}
bool file_path :1;
bool meta_data :1;
bool offset_index_offset :1;
bool offset_index_length :1;
bool column_index_offset :1;
bool column_index_length :1;
bool crypto_meta_data :1;
bool crypto_metadata :1;
bool encrypted_column_metadata :1;
} _ColumnChunk__isset;
class ColumnChunk : public virtual ::apache::thrift::TBase {
@ -1838,7 +1888,7 @@ class ColumnChunk : public virtual ::apache::thrift::TBase {
ColumnChunk(const ColumnChunk&);
ColumnChunk& operator=(const ColumnChunk&);
ColumnChunk() : file_path(), file_offset(0), offset_index_offset(0), offset_index_length(0), column_index_offset(0), column_index_length(0) {
ColumnChunk() : file_path(), file_offset(0), offset_index_offset(0), offset_index_length(0), column_index_offset(0), column_index_length(0), encrypted_column_metadata() {
}
virtual ~ColumnChunk() throw();
@ -1849,7 +1899,8 @@ class ColumnChunk : public virtual ::apache::thrift::TBase {
int32_t offset_index_length;
int64_t column_index_offset;
int32_t column_index_length;
ColumnCryptoMetaData crypto_meta_data;
ColumnCryptoMetaData crypto_metadata;
std::string encrypted_column_metadata;
_ColumnChunk__isset __isset;
@ -1867,7 +1918,9 @@ class ColumnChunk : public virtual ::apache::thrift::TBase {
void __set_column_index_length(const int32_t val);
void __set_crypto_meta_data(const ColumnCryptoMetaData& val);
void __set_crypto_metadata(const ColumnCryptoMetaData& val);
void __set_encrypted_column_metadata(const std::string& val);
bool operator == (const ColumnChunk & rhs) const
{
@ -1897,9 +1950,13 @@ class ColumnChunk : public virtual ::apache::thrift::TBase {
return false;
else if (__isset.column_index_length && !(column_index_length == rhs.column_index_length))
return false;
if (__isset.crypto_meta_data != rhs.__isset.crypto_meta_data)
if (__isset.crypto_metadata != rhs.__isset.crypto_metadata)
return false;
else if (__isset.crypto_meta_data && !(crypto_meta_data == rhs.crypto_meta_data))
else if (__isset.crypto_metadata && !(crypto_metadata == rhs.crypto_metadata))
return false;
if (__isset.encrypted_column_metadata != rhs.__isset.encrypted_column_metadata)
return false;
else if (__isset.encrypted_column_metadata && !(encrypted_column_metadata == rhs.encrypted_column_metadata))
return false;
return true;
}
@ -1920,10 +1977,11 @@ void swap(ColumnChunk &a, ColumnChunk &b);
std::ostream& operator<<(std::ostream& out, const ColumnChunk& obj);
typedef struct _RowGroup__isset {
_RowGroup__isset() : sorting_columns(false), file_offset(false), total_compressed_size(false) {}
_RowGroup__isset() : sorting_columns(false), file_offset(false), total_compressed_size(false), ordinal(false) {}
bool sorting_columns :1;
bool file_offset :1;
bool total_compressed_size :1;
bool ordinal :1;
} _RowGroup__isset;
class RowGroup : public virtual ::apache::thrift::TBase {
@ -1931,7 +1989,7 @@ class RowGroup : public virtual ::apache::thrift::TBase {
RowGroup(const RowGroup&);
RowGroup& operator=(const RowGroup&);
RowGroup() : total_byte_size(0), num_rows(0), file_offset(0), total_compressed_size(0) {
RowGroup() : total_byte_size(0), num_rows(0), file_offset(0), total_compressed_size(0), ordinal(0) {
}
virtual ~RowGroup() throw();
@ -1941,6 +1999,7 @@ class RowGroup : public virtual ::apache::thrift::TBase {
std::vector<SortingColumn> sorting_columns;
int64_t file_offset;
int64_t total_compressed_size;
int16_t ordinal;
_RowGroup__isset __isset;
@ -1956,6 +2015,8 @@ class RowGroup : public virtual ::apache::thrift::TBase {
void __set_total_compressed_size(const int64_t val);
void __set_ordinal(const int16_t val);
bool operator == (const RowGroup & rhs) const
{
if (!(columns == rhs.columns))
@ -1976,6 +2037,10 @@ class RowGroup : public virtual ::apache::thrift::TBase {
return false;
else if (__isset.total_compressed_size && !(total_compressed_size == rhs.total_compressed_size))
return false;
if (__isset.ordinal != rhs.__isset.ordinal)
return false;
else if (__isset.ordinal && !(ordinal == rhs.ordinal))
return false;
return true;
}
bool operator != (const RowGroup &rhs) const {
@ -2215,90 +2280,11 @@ void swap(ColumnIndex &a, ColumnIndex &b);
std::ostream& operator<<(std::ostream& out, const ColumnIndex& obj);
typedef struct _FileMetaData__isset {
_FileMetaData__isset() : key_value_metadata(false), created_by(false), column_orders(false) {}
bool key_value_metadata :1;
bool created_by :1;
bool column_orders :1;
} _FileMetaData__isset;
class FileMetaData : public virtual ::apache::thrift::TBase {
public:
FileMetaData(const FileMetaData&);
FileMetaData& operator=(const FileMetaData&);
FileMetaData() : version(0), num_rows(0), created_by() {
}
virtual ~FileMetaData() throw();
int32_t version;
std::vector<SchemaElement> schema;
int64_t num_rows;
std::vector<RowGroup> row_groups;
std::vector<KeyValue> key_value_metadata;
std::string created_by;
std::vector<ColumnOrder> column_orders;
_FileMetaData__isset __isset;
void __set_version(const int32_t val);
void __set_schema(const std::vector<SchemaElement> & val);
void __set_num_rows(const int64_t val);
void __set_row_groups(const std::vector<RowGroup> & val);
void __set_key_value_metadata(const std::vector<KeyValue> & val);
void __set_created_by(const std::string& val);
void __set_column_orders(const std::vector<ColumnOrder> & val);
bool operator == (const FileMetaData & rhs) const
{
if (!(version == rhs.version))
return false;
if (!(schema == rhs.schema))
return false;
if (!(num_rows == rhs.num_rows))
return false;
if (!(row_groups == rhs.row_groups))
return false;
if (__isset.key_value_metadata != rhs.__isset.key_value_metadata)
return false;
else if (__isset.key_value_metadata && !(key_value_metadata == rhs.key_value_metadata))
return false;
if (__isset.created_by != rhs.__isset.created_by)
return false;
else if (__isset.created_by && !(created_by == rhs.created_by))
return false;
if (__isset.column_orders != rhs.__isset.column_orders)
return false;
else if (__isset.column_orders && !(column_orders == rhs.column_orders))
return false;
return true;
}
bool operator != (const FileMetaData &rhs) const {
return !(*this == rhs);
}
bool operator < (const FileMetaData & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
virtual void printTo(std::ostream& out) const;
};
void swap(FileMetaData &a, FileMetaData &b);
std::ostream& operator<<(std::ostream& out, const FileMetaData& obj);
typedef struct _AesGcmV1__isset {
_AesGcmV1__isset() : aad_metadata(false), iv_prefix(false) {}
bool aad_metadata :1;
bool iv_prefix :1;
_AesGcmV1__isset() : aad_prefix(false), aad_file_unique(false), supply_aad_prefix(false) {}
bool aad_prefix :1;
bool aad_file_unique :1;
bool supply_aad_prefix :1;
} _AesGcmV1__isset;
class AesGcmV1 : public virtual ::apache::thrift::TBase {
@ -2306,28 +2292,35 @@ class AesGcmV1 : public virtual ::apache::thrift::TBase {
AesGcmV1(const AesGcmV1&);
AesGcmV1& operator=(const AesGcmV1&);
AesGcmV1() : aad_metadata(), iv_prefix() {
AesGcmV1() : aad_prefix(), aad_file_unique(), supply_aad_prefix(0) {
}
virtual ~AesGcmV1() throw();
std::string aad_metadata;
std::string iv_prefix;
std::string aad_prefix;
std::string aad_file_unique;
bool supply_aad_prefix;
_AesGcmV1__isset __isset;
void __set_aad_metadata(const std::string& val);
void __set_aad_prefix(const std::string& val);
void __set_iv_prefix(const std::string& val);
void __set_aad_file_unique(const std::string& val);
void __set_supply_aad_prefix(const bool val);
bool operator == (const AesGcmV1 & rhs) const
{
if (__isset.aad_metadata != rhs.__isset.aad_metadata)
if (__isset.aad_prefix != rhs.__isset.aad_prefix)
return false;
else if (__isset.aad_metadata && !(aad_metadata == rhs.aad_metadata))
else if (__isset.aad_prefix && !(aad_prefix == rhs.aad_prefix))
return false;
if (__isset.iv_prefix != rhs.__isset.iv_prefix)
if (__isset.aad_file_unique != rhs.__isset.aad_file_unique)
return false;
else if (__isset.iv_prefix && !(iv_prefix == rhs.iv_prefix))
else if (__isset.aad_file_unique && !(aad_file_unique == rhs.aad_file_unique))
return false;
if (__isset.supply_aad_prefix != rhs.__isset.supply_aad_prefix)
return false;
else if (__isset.supply_aad_prefix && !(supply_aad_prefix == rhs.supply_aad_prefix))
return false;
return true;
}
@ -2348,10 +2341,10 @@ void swap(AesGcmV1 &a, AesGcmV1 &b);
std::ostream& operator<<(std::ostream& out, const AesGcmV1& obj);
typedef struct _AesGcmCtrV1__isset {
_AesGcmCtrV1__isset() : aad_metadata(false), gcm_iv_prefix(false), ctr_iv_prefix(false) {}
bool aad_metadata :1;
bool gcm_iv_prefix :1;
bool ctr_iv_prefix :1;
_AesGcmCtrV1__isset() : aad_prefix(false), aad_file_unique(false), supply_aad_prefix(false) {}
bool aad_prefix :1;
bool aad_file_unique :1;
bool supply_aad_prefix :1;
} _AesGcmCtrV1__isset;
class AesGcmCtrV1 : public virtual ::apache::thrift::TBase {
@ -2359,35 +2352,35 @@ class AesGcmCtrV1 : public virtual ::apache::thrift::TBase {
AesGcmCtrV1(const AesGcmCtrV1&);
AesGcmCtrV1& operator=(const AesGcmCtrV1&);
AesGcmCtrV1() : aad_metadata(), gcm_iv_prefix(), ctr_iv_prefix() {
AesGcmCtrV1() : aad_prefix(), aad_file_unique(), supply_aad_prefix(0) {
}
virtual ~AesGcmCtrV1() throw();
std::string aad_metadata;
std::string gcm_iv_prefix;
std::string ctr_iv_prefix;
std::string aad_prefix;
std::string aad_file_unique;
bool supply_aad_prefix;
_AesGcmCtrV1__isset __isset;
void __set_aad_metadata(const std::string& val);
void __set_aad_prefix(const std::string& val);
void __set_gcm_iv_prefix(const std::string& val);
void __set_aad_file_unique(const std::string& val);
void __set_ctr_iv_prefix(const std::string& val);
void __set_supply_aad_prefix(const bool val);
bool operator == (const AesGcmCtrV1 & rhs) const
{
if (__isset.aad_metadata != rhs.__isset.aad_metadata)
if (__isset.aad_prefix != rhs.__isset.aad_prefix)
return false;
else if (__isset.aad_metadata && !(aad_metadata == rhs.aad_metadata))
else if (__isset.aad_prefix && !(aad_prefix == rhs.aad_prefix))
return false;
if (__isset.gcm_iv_prefix != rhs.__isset.gcm_iv_prefix)
if (__isset.aad_file_unique != rhs.__isset.aad_file_unique)
return false;
else if (__isset.gcm_iv_prefix && !(gcm_iv_prefix == rhs.gcm_iv_prefix))
else if (__isset.aad_file_unique && !(aad_file_unique == rhs.aad_file_unique))
return false;
if (__isset.ctr_iv_prefix != rhs.__isset.ctr_iv_prefix)
if (__isset.supply_aad_prefix != rhs.__isset.supply_aad_prefix)
return false;
else if (__isset.ctr_iv_prefix && !(ctr_iv_prefix == rhs.ctr_iv_prefix))
else if (__isset.supply_aad_prefix && !(supply_aad_prefix == rhs.supply_aad_prefix))
return false;
return true;
}
@ -2459,9 +2452,105 @@ void swap(EncryptionAlgorithm &a, EncryptionAlgorithm &b);
std::ostream& operator<<(std::ostream& out, const EncryptionAlgorithm& obj);
typedef struct _FileMetaData__isset {
_FileMetaData__isset() : key_value_metadata(false), created_by(false), column_orders(false), encryption_algorithm(false), footer_signing_key_metadata(false) {}
bool key_value_metadata :1;
bool created_by :1;
bool column_orders :1;
bool encryption_algorithm :1;
bool footer_signing_key_metadata :1;
} _FileMetaData__isset;
class FileMetaData : public virtual ::apache::thrift::TBase {
public:
FileMetaData(const FileMetaData&);
FileMetaData& operator=(const FileMetaData&);
FileMetaData() : version(0), num_rows(0), created_by(), footer_signing_key_metadata() {
}
virtual ~FileMetaData() throw();
int32_t version;
std::vector<SchemaElement> schema;
int64_t num_rows;
std::vector<RowGroup> row_groups;
std::vector<KeyValue> key_value_metadata;
std::string created_by;
std::vector<ColumnOrder> column_orders;
EncryptionAlgorithm encryption_algorithm;
std::string footer_signing_key_metadata;
_FileMetaData__isset __isset;
void __set_version(const int32_t val);
void __set_schema(const std::vector<SchemaElement> & val);
void __set_num_rows(const int64_t val);
void __set_row_groups(const std::vector<RowGroup> & val);
void __set_key_value_metadata(const std::vector<KeyValue> & val);
void __set_created_by(const std::string& val);
void __set_column_orders(const std::vector<ColumnOrder> & val);
void __set_encryption_algorithm(const EncryptionAlgorithm& val);
void __set_footer_signing_key_metadata(const std::string& val);
bool operator == (const FileMetaData & rhs) const
{
if (!(version == rhs.version))
return false;
if (!(schema == rhs.schema))
return false;
if (!(num_rows == rhs.num_rows))
return false;
if (!(row_groups == rhs.row_groups))
return false;
if (__isset.key_value_metadata != rhs.__isset.key_value_metadata)
return false;
else if (__isset.key_value_metadata && !(key_value_metadata == rhs.key_value_metadata))
return false;
if (__isset.created_by != rhs.__isset.created_by)
return false;
else if (__isset.created_by && !(created_by == rhs.created_by))
return false;
if (__isset.column_orders != rhs.__isset.column_orders)
return false;
else if (__isset.column_orders && !(column_orders == rhs.column_orders))
return false;
if (__isset.encryption_algorithm != rhs.__isset.encryption_algorithm)
return false;
else if (__isset.encryption_algorithm && !(encryption_algorithm == rhs.encryption_algorithm))
return false;
if (__isset.footer_signing_key_metadata != rhs.__isset.footer_signing_key_metadata)
return false;
else if (__isset.footer_signing_key_metadata && !(footer_signing_key_metadata == rhs.footer_signing_key_metadata))
return false;
return true;
}
bool operator != (const FileMetaData &rhs) const {
return !(*this == rhs);
}
bool operator < (const FileMetaData & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
virtual void printTo(std::ostream& out) const;
};
void swap(FileMetaData &a, FileMetaData &b);
std::ostream& operator<<(std::ostream& out, const FileMetaData& obj);
typedef struct _FileCryptoMetaData__isset {
_FileCryptoMetaData__isset() : footer_key_metadata(false) {}
bool footer_key_metadata :1;
_FileCryptoMetaData__isset() : key_metadata(false) {}
bool key_metadata :1;
} _FileCryptoMetaData__isset;
class FileCryptoMetaData : public virtual ::apache::thrift::TBase {
@ -2469,36 +2558,26 @@ class FileCryptoMetaData : public virtual ::apache::thrift::TBase {
FileCryptoMetaData(const FileCryptoMetaData&);
FileCryptoMetaData& operator=(const FileCryptoMetaData&);
FileCryptoMetaData() : encrypted_footer(0), footer_key_metadata(), footer_offset(0) {
FileCryptoMetaData() : key_metadata() {
}
virtual ~FileCryptoMetaData() throw();
EncryptionAlgorithm encryption_algorithm;
bool encrypted_footer;
std::string footer_key_metadata;
int64_t footer_offset;
std::string key_metadata;
_FileCryptoMetaData__isset __isset;
void __set_encryption_algorithm(const EncryptionAlgorithm& val);
void __set_encrypted_footer(const bool val);
void __set_footer_key_metadata(const std::string& val);
void __set_footer_offset(const int64_t val);
void __set_key_metadata(const std::string& val);
bool operator == (const FileCryptoMetaData & rhs) const
{
if (!(encryption_algorithm == rhs.encryption_algorithm))
return false;
if (!(encrypted_footer == rhs.encrypted_footer))
if (__isset.key_metadata != rhs.__isset.key_metadata)
return false;
if (__isset.footer_key_metadata != rhs.__isset.footer_key_metadata)
return false;
else if (__isset.footer_key_metadata && !(footer_key_metadata == rhs.footer_key_metadata))
return false;
if (!(footer_offset == rhs.footer_offset))
else if (__isset.key_metadata && !(key_metadata == rhs.key_metadata))
return false;
return true;
}

2
contrib/base64 vendored

@ -1 +1 @@
Subproject commit 23213812749b42529ba73e077e923bf80e96701f
Subproject commit a27c565d1b6c676beaf297fe503c4518185666f7

5
contrib/base64-cmake/README.md Normal file
View File

@ -0,0 +1,5 @@
`tables.h` file was generated by the base64 library with it's makefile:
```
cc -std=c99 -O3 -Wall -Wextra -pedantic -o lib/table_generator lib/table_generator.c
./lib/table_generator > lib/tables.h
```

393
contrib/base64-cmake/tables.h Normal file
View File

@ -0,0 +1,393 @@
#include <stdint.h>
#define CHAR62 '+'
#define CHAR63 '/'
#define CHARPAD '='
#if BASE64_LITTLE_ENDIAN
/* SPECIAL DECODE TABLES FOR LITTLE ENDIAN (INTEL) CPUS */
const uint32_t base64_table_dec_d0[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x000000f8, 0xffffffff, 0xffffffff, 0xffffffff, 0x000000fc,
0x000000d0, 0x000000d4, 0x000000d8, 0x000000dc, 0x000000e0, 0x000000e4,
0x000000e8, 0x000000ec, 0x000000f0, 0x000000f4, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00000004, 0x00000008, 0x0000000c, 0x00000010, 0x00000014, 0x00000018,
0x0000001c, 0x00000020, 0x00000024, 0x00000028, 0x0000002c, 0x00000030,
0x00000034, 0x00000038, 0x0000003c, 0x00000040, 0x00000044, 0x00000048,
0x0000004c, 0x00000050, 0x00000054, 0x00000058, 0x0000005c, 0x00000060,
0x00000064, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00000068, 0x0000006c, 0x00000070, 0x00000074, 0x00000078,
0x0000007c, 0x00000080, 0x00000084, 0x00000088, 0x0000008c, 0x00000090,
0x00000094, 0x00000098, 0x0000009c, 0x000000a0, 0x000000a4, 0x000000a8,
0x000000ac, 0x000000b0, 0x000000b4, 0x000000b8, 0x000000bc, 0x000000c0,
0x000000c4, 0x000000c8, 0x000000cc, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d1[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x0000e003, 0xffffffff, 0xffffffff, 0xffffffff, 0x0000f003,
0x00004003, 0x00005003, 0x00006003, 0x00007003, 0x00008003, 0x00009003,
0x0000a003, 0x0000b003, 0x0000c003, 0x0000d003, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00001000, 0x00002000, 0x00003000, 0x00004000, 0x00005000, 0x00006000,
0x00007000, 0x00008000, 0x00009000, 0x0000a000, 0x0000b000, 0x0000c000,
0x0000d000, 0x0000e000, 0x0000f000, 0x00000001, 0x00001001, 0x00002001,
0x00003001, 0x00004001, 0x00005001, 0x00006001, 0x00007001, 0x00008001,
0x00009001, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x0000a001, 0x0000b001, 0x0000c001, 0x0000d001, 0x0000e001,
0x0000f001, 0x00000002, 0x00001002, 0x00002002, 0x00003002, 0x00004002,
0x00005002, 0x00006002, 0x00007002, 0x00008002, 0x00009002, 0x0000a002,
0x0000b002, 0x0000c002, 0x0000d002, 0x0000e002, 0x0000f002, 0x00000003,
0x00001003, 0x00002003, 0x00003003, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d2[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00800f00, 0xffffffff, 0xffffffff, 0xffffffff, 0x00c00f00,
0x00000d00, 0x00400d00, 0x00800d00, 0x00c00d00, 0x00000e00, 0x00400e00,
0x00800e00, 0x00c00e00, 0x00000f00, 0x00400f00, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00400000, 0x00800000, 0x00c00000, 0x00000100, 0x00400100, 0x00800100,
0x00c00100, 0x00000200, 0x00400200, 0x00800200, 0x00c00200, 0x00000300,
0x00400300, 0x00800300, 0x00c00300, 0x00000400, 0x00400400, 0x00800400,
0x00c00400, 0x00000500, 0x00400500, 0x00800500, 0x00c00500, 0x00000600,
0x00400600, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00800600, 0x00c00600, 0x00000700, 0x00400700, 0x00800700,
0x00c00700, 0x00000800, 0x00400800, 0x00800800, 0x00c00800, 0x00000900,
0x00400900, 0x00800900, 0x00c00900, 0x00000a00, 0x00400a00, 0x00800a00,
0x00c00a00, 0x00000b00, 0x00400b00, 0x00800b00, 0x00c00b00, 0x00000c00,
0x00400c00, 0x00800c00, 0x00c00c00, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d3[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x003e0000, 0xffffffff, 0xffffffff, 0xffffffff, 0x003f0000,
0x00340000, 0x00350000, 0x00360000, 0x00370000, 0x00380000, 0x00390000,
0x003a0000, 0x003b0000, 0x003c0000, 0x003d0000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00010000, 0x00020000, 0x00030000, 0x00040000, 0x00050000, 0x00060000,
0x00070000, 0x00080000, 0x00090000, 0x000a0000, 0x000b0000, 0x000c0000,
0x000d0000, 0x000e0000, 0x000f0000, 0x00100000, 0x00110000, 0x00120000,
0x00130000, 0x00140000, 0x00150000, 0x00160000, 0x00170000, 0x00180000,
0x00190000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x001a0000, 0x001b0000, 0x001c0000, 0x001d0000, 0x001e0000,
0x001f0000, 0x00200000, 0x00210000, 0x00220000, 0x00230000, 0x00240000,
0x00250000, 0x00260000, 0x00270000, 0x00280000, 0x00290000, 0x002a0000,
0x002b0000, 0x002c0000, 0x002d0000, 0x002e0000, 0x002f0000, 0x00300000,
0x00310000, 0x00320000, 0x00330000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
#else
/* SPECIAL DECODE TABLES FOR BIG ENDIAN (IBM/MOTOROLA/SUN) CPUS */
const uint32_t base64_table_dec_d0[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xf8000000, 0xffffffff, 0xffffffff, 0xffffffff, 0xfc000000,
0xd0000000, 0xd4000000, 0xd8000000, 0xdc000000, 0xe0000000, 0xe4000000,
0xe8000000, 0xec000000, 0xf0000000, 0xf4000000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x04000000, 0x08000000, 0x0c000000, 0x10000000, 0x14000000, 0x18000000,
0x1c000000, 0x20000000, 0x24000000, 0x28000000, 0x2c000000, 0x30000000,
0x34000000, 0x38000000, 0x3c000000, 0x40000000, 0x44000000, 0x48000000,
0x4c000000, 0x50000000, 0x54000000, 0x58000000, 0x5c000000, 0x60000000,
0x64000000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x68000000, 0x6c000000, 0x70000000, 0x74000000, 0x78000000,
0x7c000000, 0x80000000, 0x84000000, 0x88000000, 0x8c000000, 0x90000000,
0x94000000, 0x98000000, 0x9c000000, 0xa0000000, 0xa4000000, 0xa8000000,
0xac000000, 0xb0000000, 0xb4000000, 0xb8000000, 0xbc000000, 0xc0000000,
0xc4000000, 0xc8000000, 0xcc000000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d1[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x03e00000, 0xffffffff, 0xffffffff, 0xffffffff, 0x03f00000,
0x03400000, 0x03500000, 0x03600000, 0x03700000, 0x03800000, 0x03900000,
0x03a00000, 0x03b00000, 0x03c00000, 0x03d00000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00100000, 0x00200000, 0x00300000, 0x00400000, 0x00500000, 0x00600000,
0x00700000, 0x00800000, 0x00900000, 0x00a00000, 0x00b00000, 0x00c00000,
0x00d00000, 0x00e00000, 0x00f00000, 0x01000000, 0x01100000, 0x01200000,
0x01300000, 0x01400000, 0x01500000, 0x01600000, 0x01700000, 0x01800000,
0x01900000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x01a00000, 0x01b00000, 0x01c00000, 0x01d00000, 0x01e00000,
0x01f00000, 0x02000000, 0x02100000, 0x02200000, 0x02300000, 0x02400000,
0x02500000, 0x02600000, 0x02700000, 0x02800000, 0x02900000, 0x02a00000,
0x02b00000, 0x02c00000, 0x02d00000, 0x02e00000, 0x02f00000, 0x03000000,
0x03100000, 0x03200000, 0x03300000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d2[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x000f8000, 0xffffffff, 0xffffffff, 0xffffffff, 0x000fc000,
0x000d0000, 0x000d4000, 0x000d8000, 0x000dc000, 0x000e0000, 0x000e4000,
0x000e8000, 0x000ec000, 0x000f0000, 0x000f4000, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00004000, 0x00008000, 0x0000c000, 0x00010000, 0x00014000, 0x00018000,
0x0001c000, 0x00020000, 0x00024000, 0x00028000, 0x0002c000, 0x00030000,
0x00034000, 0x00038000, 0x0003c000, 0x00040000, 0x00044000, 0x00048000,
0x0004c000, 0x00050000, 0x00054000, 0x00058000, 0x0005c000, 0x00060000,
0x00064000, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00068000, 0x0006c000, 0x00070000, 0x00074000, 0x00078000,
0x0007c000, 0x00080000, 0x00084000, 0x00088000, 0x0008c000, 0x00090000,
0x00094000, 0x00098000, 0x0009c000, 0x000a0000, 0x000a4000, 0x000a8000,
0x000ac000, 0x000b0000, 0x000b4000, 0x000b8000, 0x000bc000, 0x000c0000,
0x000c4000, 0x000c8000, 0x000cc000, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
const uint32_t base64_table_dec_d3[256] = {
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00003e00, 0xffffffff, 0xffffffff, 0xffffffff, 0x00003f00,
0x00003400, 0x00003500, 0x00003600, 0x00003700, 0x00003800, 0x00003900,
0x00003a00, 0x00003b00, 0x00003c00, 0x00003d00, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0x00000000,
0x00000100, 0x00000200, 0x00000300, 0x00000400, 0x00000500, 0x00000600,
0x00000700, 0x00000800, 0x00000900, 0x00000a00, 0x00000b00, 0x00000c00,
0x00000d00, 0x00000e00, 0x00000f00, 0x00001000, 0x00001100, 0x00001200,
0x00001300, 0x00001400, 0x00001500, 0x00001600, 0x00001700, 0x00001800,
0x00001900, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0x00001a00, 0x00001b00, 0x00001c00, 0x00001d00, 0x00001e00,
0x00001f00, 0x00002000, 0x00002100, 0x00002200, 0x00002300, 0x00002400,
0x00002500, 0x00002600, 0x00002700, 0x00002800, 0x00002900, 0x00002a00,
0x00002b00, 0x00002c00, 0x00002d00, 0x00002e00, 0x00002f00, 0x00003000,
0x00003100, 0x00003200, 0x00003300, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff,
0xffffffff, 0xffffffff, 0xffffffff, 0xffffffff
};
#endif

1
contrib/flatbuffers vendored Submodule

@ -0,0 +1 @@
Subproject commit bf9eb67ab9371755c6bcece13cadc7693bcbf264

4
dbms/CMakeLists.txt
View File

@ -186,8 +186,8 @@ endif()
list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD})
list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON})
list (APPEND dbms_sources src/Functions/IFunction.cpp src/Functions/FunctionFactory.cpp src/Functions/FunctionHelpers.cpp)
list (APPEND dbms_headers src/Functions/IFunctionImpl.h src/Functions/FunctionFactory.h src/Functions/FunctionHelpers.h)
list (APPEND dbms_sources src/Functions/IFunction.cpp src/Functions/FunctionFactory.cpp src/Functions/FunctionHelpers.cpp src/Functions/extractTimeZoneFromFunctionArguments.cpp)
list (APPEND dbms_headers src/Functions/IFunctionImpl.h src/Functions/FunctionFactory.h src/Functions/FunctionHelpers.h src/Functions/extractTimeZoneFromFunctionArguments.h)
list (APPEND dbms_sources
src/AggregateFunctions/AggregateFunctionFactory.cpp

2
dbms/programs/server/Server.cpp
View File

@ -298,7 +298,7 @@ int Server::main(const std::vector<std::string> & /*args*/)
global_context->shutdown();
LOG_DEBUG(log, "Shutted down storages.");
LOG_DEBUG(log, "Shut down storages.");
/** Explicitly destroy Context. It is more convenient than in destructor of Server, because logger is still available.
* At this moment, no one could own shared part of Context.

1
dbms/programs/server/config.d/log_to_console.xml
View File

@ -1,5 +1,6 @@
<yandex>
<logger>
<console>true</console>
<log remove="remove"/>
<errorlog remove="remove"/>
</logger>

14
dbms/programs/server/users.xml
View File

@ -84,25 +84,17 @@
<!-- Quota for user. -->
<quota>default</quota>
<!-- For testing the table filters -->
<databases>
<!-- Example of row level security policy. -->
<!-- <databases>
<test>
<!-- Simple expression filter -->
<filtered_table1>
<filter>a = 1</filter>
</filtered_table1>
<!-- Complex expression filter -->
<filtered_table2>
<filter>a + b &lt; 1 or c - d &gt; 5</filter>
</filtered_table2>
<!-- Filter with ALIAS column -->
<filtered_table3>
<filter>c = 1</filter>
</filtered_table3>
</test>
</databases>
</databases> -->
</default>
<!-- Example of user with readonly access. -->

11
dbms/src/Access/AccessControlManager.cpp
View File

@ -3,6 +3,7 @@
#include <Access/MemoryAccessStorage.h>
#include <Access/UsersConfigAccessStorage.h>
#include <Access/QuotaContextFactory.h>
#include <Access/RowPolicyContextFactory.h>
namespace DB
@ -21,7 +22,8 @@ namespace
AccessControlManager::AccessControlManager()
: MultipleAccessStorage(createStorages()),
quota_context_factory(std::make_unique<QuotaContextFactory>(*this))
quota_context_factory(std::make_unique<QuotaContextFactory>(*this)),
row_policy_context_factory(std::make_unique<RowPolicyContextFactory>(*this))
{
}
@ -49,4 +51,11 @@ std::vector<QuotaUsageInfo> AccessControlManager::getQuotaUsageInfo() const
{
return quota_context_factory->getUsageInfo();
}
std::shared_ptr<RowPolicyContext> AccessControlManager::getRowPolicyContext(const String & user_name) const
{
return row_policy_context_factory->createContext(user_name);
}
}

5
dbms/src/Access/AccessControlManager.h
View File

@ -22,6 +22,8 @@ namespace DB
class QuotaContext;
class QuotaContextFactory;
struct QuotaUsageInfo;
class RowPolicyContext;
class RowPolicyContextFactory;
/// Manages access control entities.
@ -38,8 +40,11 @@ public:
std::vector<QuotaUsageInfo> getQuotaUsageInfo() const;
std::shared_ptr<RowPolicyContext> getRowPolicyContext(const String & user_name) const;
private:
std::unique_ptr<QuotaContextFactory> quota_context_factory;
std::unique_ptr<RowPolicyContextFactory> row_policy_context_factory;
};
}

3
dbms/src/Access/IAccessEntity.cpp
View File

@ -1,5 +1,6 @@
#include <Access/IAccessEntity.h>
#include <Access/Quota.h>
#include <Access/RowPolicy.h>
#include <common/demangle.h>
@ -9,6 +10,8 @@ String IAccessEntity::getTypeName(std::type_index type)
{
if (type == typeid(Quota))
return "Quota";
if (type == typeid(RowPolicy))
return "Row policy";
return demangle(type.name());
}

111
dbms/src/Access/RowPolicy.cpp Normal file
View File

@ -0,0 +1,111 @@
#include <Access/RowPolicy.h>
#include <Interpreters/Context.h>
#include <Common/quoteString.h>
#include <boost/range/algorithm/equal.hpp>
namespace DB
{
namespace
{
void generateFullNameImpl(const String & database_, const String & table_name_, const String & policy_name_, String & full_name_)
{
full_name_.clear();
full_name_.reserve(database_.length() + table_name_.length() + policy_name_.length() + 6);
full_name_ += backQuoteIfNeed(policy_name_);
full_name_ += " ON ";
if (!database_.empty())
{
full_name_ += backQuoteIfNeed(database_);
full_name_ += '.';
}
full_name_ += backQuoteIfNeed(table_name_);
}
}
String RowPolicy::FullNameParts::getFullName() const
{
String full_name;
generateFullNameImpl(database, table_name, policy_name, full_name);
return full_name;
}
String RowPolicy::FullNameParts::getFullName(const Context & context) const
{
String full_name;
generateFullNameImpl(database.empty() ? context.getCurrentDatabase() : database, table_name, policy_name, full_name);
return full_name;
}
void RowPolicy::setDatabase(const String & database_)
{
database = database_;
generateFullNameImpl(database, table_name, policy_name, full_name);
}
void RowPolicy::setTableName(const String & table_name_)
{
table_name = table_name_;
generateFullNameImpl(database, table_name, policy_name, full_name);
}
void RowPolicy::setName(const String & policy_name_)
{
policy_name = policy_name_;
generateFullNameImpl(database, table_name, policy_name, full_name);
}
void RowPolicy::setFullName(const String & database_, const String & table_name_, const String & policy_name_)
{
database = database_;
table_name = table_name_;
policy_name = policy_name_;
generateFullNameImpl(database, table_name, policy_name, full_name);
}
bool RowPolicy::equal(const IAccessEntity & other) const
{
if (!IAccessEntity::equal(other))
return false;
const auto & other_policy = typeid_cast<const RowPolicy &>(other);
return (database == other_policy.database) && (table_name == other_policy.table_name) && (policy_name == other_policy.policy_name)
&& boost::range::equal(conditions, other_policy.conditions) && restrictive == other_policy.restrictive
&& (roles == other_policy.roles) && (all_roles == other_policy.all_roles) && (except_roles == other_policy.except_roles);
}
const char * RowPolicy::conditionIndexToString(ConditionIndex index)
{
switch (index)
{
case SELECT_FILTER: return "SELECT_FILTER";
case INSERT_CHECK: return "INSERT_CHECK";
case UPDATE_FILTER: return "UPDATE_FILTER";
case UPDATE_CHECK: return "UPDATE_CHECK";
case DELETE_FILTER: return "DELETE_FILTER";
}
__builtin_unreachable();
}
const char * RowPolicy::conditionIndexToColumnName(ConditionIndex index)
{
switch (index)
{
case SELECT_FILTER: return "select_filter";
case INSERT_CHECK: return "insert_check";
case UPDATE_FILTER: return "update_filter";
case UPDATE_CHECK: return "update_check";
case DELETE_FILTER: return "delete_filter";
}
__builtin_unreachable();
}
}

81
dbms/src/Access/RowPolicy.h Normal file
View File

@ -0,0 +1,81 @@
#pragma once
#include <Access/IAccessEntity.h>
namespace DB
{
class Context;
/** Represents a row level security policy for a table.
*/
struct RowPolicy : public IAccessEntity
{
void setDatabase(const String & database_);
void setTableName(const String & table_name_);
void setName(const String & policy_name_) override;
void setFullName(const String & database_, const String & table_name_, const String & policy_name_);
String getDatabase() const { return database; }
String getTableName() const { return table_name; }
String getName() const override { return policy_name; }
struct FullNameParts
{
String database;
String table_name;
String policy_name;
String getFullName() const;
String getFullName(const Context & context) const;
};
/// Filter is a SQL conditional expression used to figure out which rows should be visible
/// for user or available for modification. If the expression returns NULL or false for some rows
/// those rows are silently suppressed.
/// Check is a SQL condition expression used to check whether a row can be written into
/// the table. If the expression returns NULL or false an exception is thrown.
/// If a conditional expression here is empty it means no filtering is applied.
enum ConditionIndex
{
SELECT_FILTER,
INSERT_CHECK,
UPDATE_FILTER,
UPDATE_CHECK,
DELETE_FILTER,
};
static constexpr size_t MAX_CONDITION_INDEX = 5;
static const char * conditionIndexToString(ConditionIndex index);
static const char * conditionIndexToColumnName(ConditionIndex index);
String conditions[MAX_CONDITION_INDEX];
/// Sets that the policy is permissive.
/// A row is only accessible if at least one of the permissive policies passes,
/// in addition to all the restrictive policies.
void setPermissive(bool permissive_ = true) { setRestrictive(!permissive_); }
bool isPermissive() const { return !isRestrictive(); }
/// Sets that the policy is restrictive.
/// A row is only accessible if at least one of the permissive policies passes,
/// in addition to all the restrictive policies.
void setRestrictive(bool restrictive_ = true) { restrictive = restrictive_; }
bool isRestrictive() const { return restrictive; }
bool equal(const IAccessEntity & other) const override;
std::shared_ptr<IAccessEntity> clone() const override { return cloneImpl<RowPolicy>(); }
/// Which roles or users should use this quota.
Strings roles;
bool all_roles = false;
Strings except_roles;
private:
String database;
String table_name;
String policy_name;
bool restrictive = false;
};
using RowPolicyPtr = std::shared_ptr<const RowPolicy>;
}

59
dbms/src/Access/RowPolicyContext.cpp Normal file
View File

@ -0,0 +1,59 @@
#include <Access/RowPolicyContext.h>
#include <boost/range/adaptor/map.hpp>
#include <boost/range/algorithm/copy.hpp>
namespace DB
{
size_t RowPolicyContext::Hash::operator()(const DatabaseAndTableNameRef & database_and_table_name) const
{
return std::hash<StringRef>{}(database_and_table_name.first) - std::hash<StringRef>{}(database_and_table_name.second);
}
RowPolicyContext::RowPolicyContext()
: atomic_map_of_mixed_conditions(std::make_shared<MapOfMixedConditions>())
{
}
RowPolicyContext::~RowPolicyContext() = default;
RowPolicyContext::RowPolicyContext(const String & user_name_)
: user_name(user_name_)
{}
ASTPtr RowPolicyContext::getCondition(const String & database, const String & table_name, ConditionIndex index) const
{
/// We don't lock `mutex` here.
auto map_of_mixed_conditions = std::atomic_load(&atomic_map_of_mixed_conditions);
auto it = map_of_mixed_conditions->find({database, table_name});
if (it == map_of_mixed_conditions->end())
return {};
return it->second.mixed_conditions[index];
}
std::vector<UUID> RowPolicyContext::getCurrentPolicyIDs() const
{
/// We don't lock `mutex` here.
auto map_of_mixed_conditions = std::atomic_load(&atomic_map_of_mixed_conditions);
std::vector<UUID> policy_ids;
for (const auto & mixed_conditions : *map_of_mixed_conditions | boost::adaptors::map_values)
boost::range::copy(mixed_conditions.policy_ids, std::back_inserter(policy_ids));
return policy_ids;
}
std::vector<UUID> RowPolicyContext::getCurrentPolicyIDs(const String & database, const String & table_name) const
{
/// We don't lock `mutex` here.
auto map_of_mixed_conditions = std::atomic_load(&atomic_map_of_mixed_conditions);
auto it = map_of_mixed_conditions->find({database, table_name});
if (it == map_of_mixed_conditions->end())
return {};
return it->second.policy_ids;
}
}

66
dbms/src/Access/RowPolicyContext.h Normal file
View File

@ -0,0 +1,66 @@
#pragma once
#include <Access/RowPolicy.h>
#include <Core/Types.h>
#include <Core/UUID.h>
#include <common/StringRef.h>
#include <memory>
#include <unordered_map>
namespace DB
{
class IAST;
using ASTPtr = std::shared_ptr<IAST>;
/// Provides fast access to row policies' conditions for a specific user and tables.
class RowPolicyContext
{
public:
/// Default constructor makes a row policy usage context which restricts nothing.
RowPolicyContext();
~RowPolicyContext();
using ConditionIndex = RowPolicy::ConditionIndex;
/// Returns prepared filter for a specific table and operations.
/// The function can return nullptr, that means there is no filters applied.
/// The returned filter can be a combination of the filters defined by multiple row policies.
ASTPtr getCondition(const String & database, const String & table_name, ConditionIndex index) const;
/// Returns IDs of all the policies used by the current user.
std::vector<UUID> getCurrentPolicyIDs() const;
/// Returns IDs of the policies used by a concrete table.
std::vector<UUID> getCurrentPolicyIDs(const String & database, const String & table_name) const;
private:
friend class RowPolicyContextFactory;
friend struct ext::shared_ptr_helper<RowPolicyContext>;
RowPolicyContext(const String & user_name_); /// RowPolicyContext should be created by RowPolicyContextFactory.
using DatabaseAndTableName = std::pair<String, String>;
using DatabaseAndTableNameRef = std::pair<StringRef, StringRef>;
struct Hash
{
size_t operator()(const DatabaseAndTableNameRef & database_and_table_name) const;
};
static constexpr size_t MAX_CONDITION_INDEX = RowPolicy::MAX_CONDITION_INDEX;
using ParsedConditions = std::array<ASTPtr, MAX_CONDITION_INDEX>;
struct MixedConditions
{
std::unique_ptr<DatabaseAndTableName> database_and_table_name_keeper;
ParsedConditions mixed_conditions;
std::vector<UUID> policy_ids;
};
using MapOfMixedConditions = std::unordered_map<DatabaseAndTableNameRef, MixedConditions, Hash>;
const String user_name;
std::shared_ptr<const MapOfMixedConditions> atomic_map_of_mixed_conditions; /// Changed atomically, not protected by `mutex`.
};
using RowPolicyContextPtr = std::shared_ptr<RowPolicyContext>;
}

314
dbms/src/Access/RowPolicyContextFactory.cpp Normal file
View File

@ -0,0 +1,314 @@
#include <Access/RowPolicyContextFactory.h>
#include <Access/RowPolicyContext.h>
#include <Access/AccessControlManager.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ExpressionListParsers.h>
#include <Parsers/parseQuery.h>
#include <Common/Exception.h>
#include <Common/quoteString.h>
#include <ext/range.h>
#include <boost/range/algorithm/copy.hpp>
#include <boost/range/algorithm_ext/erase.hpp>
namespace DB
{
namespace
{
bool tryGetLiteralBool(const IAST & ast, bool & value)
{
try
{
if (const ASTLiteral * literal = ast.as<ASTLiteral>())
{
value = !literal->value.isNull() && applyVisitor(FieldVisitorConvertToNumber<bool>(), literal->value);
return true;
}
return false;
}
catch (...)
{
return false;
}
}
ASTPtr applyFunctionAND(ASTs arguments)
{
bool const_arguments = true;
boost::range::remove_erase_if(arguments, [&](const ASTPtr & argument) -> bool
{
bool b;
if (!tryGetLiteralBool(*argument, b))
return false;
const_arguments &= b;
return true;
});
if (!const_arguments)
return std::make_shared<ASTLiteral>(Field{UInt8(0)});
if (arguments.empty())
return std::make_shared<ASTLiteral>(Field{UInt8(1)});
if (arguments.size() == 1)
return arguments[0];
auto function = std::make_shared<ASTFunction>();
auto exp_list = std::make_shared<ASTExpressionList>();
function->name = "and";
function->arguments = exp_list;
function->children.push_back(exp_list);
exp_list->children = std::move(arguments);
return function;
}
ASTPtr applyFunctionOR(ASTs arguments)
{
bool const_arguments = false;
boost::range::remove_erase_if(arguments, [&](const ASTPtr & argument) -> bool
{
bool b;
if (!tryGetLiteralBool(*argument, b))
return false;
const_arguments |= b;
return true;
});
if (const_arguments)
return std::make_shared<ASTLiteral>(Field{UInt8(1)});
if (arguments.empty())
return std::make_shared<ASTLiteral>(Field{UInt8(0)});
if (arguments.size() == 1)
return arguments[0];
auto function = std::make_shared<ASTFunction>();
auto exp_list = std::make_shared<ASTExpressionList>();
function->name = "or";
function->arguments = exp_list;
function->children.push_back(exp_list);
exp_list->children = std::move(arguments);
return function;
}
using ConditionIndex = RowPolicy::ConditionIndex;
static constexpr size_t MAX_CONDITION_INDEX = RowPolicy::MAX_CONDITION_INDEX;
/// Accumulates conditions from multiple row policies and joins them using the AND logical operation.
class ConditionsMixer
{
public:
void add(const ASTPtr & condition, bool is_restrictive)
{
if (!condition)
return;
if (is_restrictive)
restrictions.push_back(condition);
else
permissions.push_back(condition);
}
ASTPtr getResult() &&
{
/// Process permissive conditions.
if (!permissions.empty())
restrictions.push_back(applyFunctionOR(std::move(permissions)));
/// Process restrictive conditions.
if (!restrictions.empty())
return applyFunctionAND(std::move(restrictions));
return nullptr;
}
private:
ASTs permissions;
ASTs restrictions;
};
}
void RowPolicyContextFactory::PolicyInfo::setPolicy(const RowPolicyPtr & policy_)
{
policy = policy_;
boost::range::copy(policy->roles, std::inserter(roles, roles.end()));
all_roles = policy->all_roles;
boost::range::copy(policy->except_roles, std::inserter(except_roles, except_roles.end()));
for (auto index : ext::range_with_static_cast<ConditionIndex>(0, MAX_CONDITION_INDEX))
{
const String & condition = policy->conditions[index];
auto previous_range = std::pair(std::begin(policy->conditions), std::begin(policy->conditions) + index);
auto previous_it = std::find(previous_range.first, previous_range.second, condition);
if (previous_it != previous_range.second)
{
/// The condition is already parsed before.
parsed_conditions[index] = parsed_conditions[previous_it - previous_range.first];
}
else
{
/// Try to parse the condition.
try
{
ParserExpression parser;
parsed_conditions[index] = parseQuery(parser, condition, 0);
}
catch (...)
{
tryLogCurrentException(
&Poco::Logger::get("RowPolicy"),
String("Could not parse the condition ") + RowPolicy::conditionIndexToString(index) + " of row policy "
+ backQuote(policy->getFullName()));
}
}
}
}
bool RowPolicyContextFactory::PolicyInfo::canUseWithContext(const RowPolicyContext & context) const
{
if (roles.count(context.user_name))
return true;
if (all_roles && !except_roles.count(context.user_name))
return true;
return false;
}
RowPolicyContextFactory::RowPolicyContextFactory(const AccessControlManager & access_control_manager_)
: access_control_manager(access_control_manager_)
{
}
RowPolicyContextFactory::~RowPolicyContextFactory() = default;
RowPolicyContextPtr RowPolicyContextFactory::createContext(const String & user_name)
{
std::lock_guard lock{mutex};
ensureAllRowPoliciesRead();
auto context = ext::shared_ptr_helper<RowPolicyContext>::create(user_name);
contexts.push_back(context);
mixConditionsForContext(*context);
return context;
}
void RowPolicyContextFactory::ensureAllRowPoliciesRead()
{
/// `mutex` is already locked.
if (all_policies_read)
return;
all_policies_read = true;
subscription = access_control_manager.subscribeForChanges<RowPolicy>(
[&](const UUID & id, const AccessEntityPtr & entity)
{
if (entity)
rowPolicyAddedOrChanged(id, typeid_cast<RowPolicyPtr>(entity));
else
rowPolicyRemoved(id);
});
for (const UUID & id : access_control_manager.findAll<RowPolicy>())
{
auto quota = access_control_manager.tryRead<RowPolicy>(id);
if (quota)
all_policies.emplace(id, PolicyInfo(quota));
}
}
void RowPolicyContextFactory::rowPolicyAddedOrChanged(const UUID & policy_id, const RowPolicyPtr & new_policy)
{
std::lock_guard lock{mutex};
auto it = all_policies.find(policy_id);
if (it == all_policies.end())
{
it = all_policies.emplace(policy_id, PolicyInfo(new_policy)).first;
}
else
{
if (it->second.policy == new_policy)
return;
}
auto & info = it->second;
info.setPolicy(new_policy);
mixConditionsForAllContexts();
}
void RowPolicyContextFactory::rowPolicyRemoved(const UUID & policy_id)
{
std::lock_guard lock{mutex};
all_policies.erase(policy_id);
mixConditionsForAllContexts();
}
void RowPolicyContextFactory::mixConditionsForAllContexts()
{
/// `mutex` is already locked.
boost::range::remove_erase_if(
contexts,
[&](const std::weak_ptr<RowPolicyContext> & weak)
{
auto context = weak.lock();
if (!context)
return true; // remove from the `contexts` list.
mixConditionsForContext(*context);
return false; // keep in the `contexts` list.
});
}
void RowPolicyContextFactory::mixConditionsForContext(RowPolicyContext & context)
{
/// `mutex` is already locked.
struct Mixers
{
ConditionsMixer mixers[MAX_CONDITION_INDEX];
std::vector<UUID> policy_ids;
};
using MapOfMixedConditions = RowPolicyContext::MapOfMixedConditions;
using DatabaseAndTableName = RowPolicyContext::DatabaseAndTableName;
using DatabaseAndTableNameRef = RowPolicyContext::DatabaseAndTableNameRef;
using Hash = RowPolicyContext::Hash;
std::unordered_map<DatabaseAndTableName, Mixers, Hash> map_of_mixers;
for (const auto & [policy_id, info] : all_policies)
{
if (info.canUseWithContext(context))
{
const auto & policy = *info.policy;
auto & mixers = map_of_mixers[std::pair{policy.getDatabase(), policy.getTableName()}];
mixers.policy_ids.push_back(policy_id);
for (auto index : ext::range(0, MAX_CONDITION_INDEX))
mixers.mixers[index].add(info.parsed_conditions[index], policy.isRestrictive());
}
}
auto map_of_mixed_conditions = std::make_shared<MapOfMixedConditions>();
for (auto & [database_and_table_name, mixers] : map_of_mixers)
{
auto database_and_table_name_keeper = std::make_unique<DatabaseAndTableName>();
database_and_table_name_keeper->first = database_and_table_name.first;
database_and_table_name_keeper->second = database_and_table_name.second;
auto & mixed_conditions = (*map_of_mixed_conditions)[DatabaseAndTableNameRef{database_and_table_name_keeper->first,
database_and_table_name_keeper->second}];
mixed_conditions.database_and_table_name_keeper = std::move(database_and_table_name_keeper);
mixed_conditions.policy_ids = std::move(mixers.policy_ids);
for (auto index : ext::range(0, MAX_CONDITION_INDEX))
mixed_conditions.mixed_conditions[index] = std::move(mixers.mixers[index]).getResult();
}
std::atomic_store(&context.atomic_map_of_mixed_conditions, std::shared_ptr<const MapOfMixedConditions>{map_of_mixed_conditions});
}
}

54
dbms/src/Access/RowPolicyContextFactory.h Normal file
View File

@ -0,0 +1,54 @@
#pragma once
#include <Access/RowPolicyContext.h>
#include <Access/IAccessStorage.h>
#include <mutex>
#include <unordered_map>
#include <unordered_set>
namespace DB
{
class AccessControlManager;
/// Stores read and parsed row policies.
class RowPolicyContextFactory
{
public:
RowPolicyContextFactory(const AccessControlManager & access_control_manager_);
~RowPolicyContextFactory();
RowPolicyContextPtr createContext(const String & user_name);
private:
using ParsedConditions = RowPolicyContext::ParsedConditions;
struct PolicyInfo
{
PolicyInfo(const RowPolicyPtr & policy_) { setPolicy(policy_); }
void setPolicy(const RowPolicyPtr & policy_);
bool canUseWithContext(const RowPolicyContext & context) const;
RowPolicyPtr policy;
std::unordered_set<String> roles;
bool all_roles = false;
std::unordered_set<String> except_roles;
ParsedConditions parsed_conditions;
};
void ensureAllRowPoliciesRead();
void rowPolicyAddedOrChanged(const UUID & policy_id, const RowPolicyPtr & new_policy);
void rowPolicyRemoved(const UUID & policy_id);
void mixConditionsForAllContexts();
void mixConditionsForContext(RowPolicyContext & context);
const AccessControlManager & access_control_manager;
std::unordered_map<UUID, PolicyInfo> all_policies;
bool all_policies_read = false;
IAccessStorage::SubscriptionPtr subscription;
std::vector<std::weak_ptr<RowPolicyContext>> contexts;
std::mutex mutex;
};
}

56
dbms/src/Access/UsersConfigAccessStorage.cpp
View File

@ -1,5 +1,6 @@
#include <Access/UsersConfigAccessStorage.h>
#include <Access/Quota.h>
#include <Access/RowPolicy.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/quoteString.h>
#include <Poco/Util/AbstractConfiguration.h>
@ -15,6 +16,8 @@ namespace
{
if (type == typeid(Quota))
return 'Q';
if (type == typeid(RowPolicy))
return 'P';
return 0;
}
@ -112,6 +115,57 @@ namespace
}
return quotas;
}
std::vector<AccessEntityPtr> parseRowPolicies(const Poco::Util::AbstractConfiguration & config, Poco::Logger * log)
{
std::vector<AccessEntityPtr> policies;
Poco::Util::AbstractConfiguration::Keys user_names;
config.keys("users", user_names);
for (const String & user_name : user_names)
{
const String databases_config = "users." + user_name + ".databases";
if (config.has(databases_config))
{
Poco::Util::AbstractConfiguration::Keys databases;
config.keys(databases_config, databases);
/// Read tables within databases
for (const String & database : databases)
{
const String database_config = databases_config + "." + database;
Poco::Util::AbstractConfiguration::Keys table_names;
config.keys(database_config, table_names);
/// Read table properties
for (const String & table_name : table_names)
{
const auto filter_config = database_config + "." + table_name + ".filter";
if (config.has(filter_config))
{
try
{
auto policy = std::make_shared<RowPolicy>();
policy->setFullName(database, table_name, user_name);
policy->conditions[RowPolicy::SELECT_FILTER] = config.getString(filter_config);
policy->roles.push_back(user_name);
policies.push_back(policy);
}
catch (...)
{
tryLogCurrentException(
log,
"Could not parse row policy " + backQuote(user_name) + " on table " + backQuoteIfNeed(database) + "."
+ backQuoteIfNeed(table_name));
}
}
}
}
}
}
return policies;
}
}
@ -128,6 +182,8 @@ void UsersConfigAccessStorage::loadFromConfig(const Poco::Util::AbstractConfigur
std::vector<std::pair<UUID, AccessEntityPtr>> all_entities;
for (const auto & entity : parseQuotas(config, getLogger()))
all_entities.emplace_back(generateID(*entity), entity);
for (const auto & entity : parseRowPolicies(config, getLogger()))
all_entities.emplace_back(generateID(*entity), entity);
memory_storage.setAll(all_entities);
}

13
dbms/src/AggregateFunctions/AggregateFunctionArgMinMax.h
View File

@ -24,11 +24,16 @@ struct AggregateFunctionArgMinMaxData
ResultData result; // the argument at which the minimum/maximum value is reached.
ValueData value; // value for which the minimum/maximum is calculated.
static bool allocatesMemoryInArena()
{
return ResultData::allocatesMemoryInArena() || ValueData::allocatesMemoryInArena();
}
};
/// Returns the first arg value found for the minimum/maximum value. Example: argMax(arg, value).
template <typename Data, bool AllocatesMemoryInArena>
class AggregateFunctionArgMinMax final : public IAggregateFunctionDataHelper<Data, AggregateFunctionArgMinMax<Data, AllocatesMemoryInArena>>
template <typename Data>
class AggregateFunctionArgMinMax final : public IAggregateFunctionDataHelper<Data, AggregateFunctionArgMinMax<Data>>
{
private:
const DataTypePtr & type_res;
@ -36,7 +41,7 @@ private:
public:
AggregateFunctionArgMinMax(const DataTypePtr & type_res_, const DataTypePtr & type_val_)
: IAggregateFunctionDataHelper<Data, AggregateFunctionArgMinMax<Data, AllocatesMemoryInArena>>({type_res_, type_val_}, {}),
: IAggregateFunctionDataHelper<Data, AggregateFunctionArgMinMax<Data>>({type_res_, type_val_}, {}),
type_res(this->argument_types[0]), type_val(this->argument_types[1])
{
if (!type_val->isComparable())
@ -77,7 +82,7 @@ public:
bool allocatesMemoryInArena() const override
{
return AllocatesMemoryInArena;
return Data::allocatesMemoryInArena();
}
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override

23
dbms/src/AggregateFunctions/AggregateFunctionMinMaxAny.h
View File

@ -166,6 +166,11 @@ public:
{
return has() && assert_cast<const ColVecType &>(column).getData()[row_num] == value;
}
static bool allocatesMemoryInArena()
{
return false;
}
};
@ -384,6 +389,11 @@ public:
{
return has() && assert_cast<const ColumnString &>(column).getDataAtWithTerminatingZero(row_num) == getStringRef();
}
static bool allocatesMemoryInArena()
{
return true;
}
};
static_assert(
@ -555,6 +565,11 @@ public:
{
return has() && to.value == value;
}
static bool allocatesMemoryInArena()
{
return false;
}
};
@ -675,15 +690,15 @@ struct AggregateFunctionAnyHeavyData : Data
};
template <typename Data, bool use_arena>
class AggregateFunctionsSingleValue final : public IAggregateFunctionDataHelper<Data, AggregateFunctionsSingleValue<Data, use_arena>>
template <typename Data>
class AggregateFunctionsSingleValue final : public IAggregateFunctionDataHelper<Data, AggregateFunctionsSingleValue<Data>>
{
private:
DataTypePtr & type;
public:
AggregateFunctionsSingleValue(const DataTypePtr & type_)
: IAggregateFunctionDataHelper<Data, AggregateFunctionsSingleValue<Data, use_arena>>({type_}, {})
: IAggregateFunctionDataHelper<Data, AggregateFunctionsSingleValue<Data>>({type_}, {})
, type(this->argument_types[0])
{
if (StringRef(Data::name()) == StringRef("min")
@ -724,7 +739,7 @@ public:
bool allocatesMemoryInArena() const override
{
return use_arena;
return Data::allocatesMemoryInArena();
}
void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const override

41
dbms/src/AggregateFunctions/HelpersMinMaxAny.h
View File

@ -13,8 +13,8 @@
namespace DB
{
/// min, max, any, anyLast
template <template <typename, bool> class AggregateFunctionTemplate, template <typename> class Data>
/// min, max, any, anyLast, anyHeavy, etc...
template <template <typename> class AggregateFunctionTemplate, template <typename> class Data>
static IAggregateFunction * createAggregateFunctionSingleValue(const String & name, const DataTypes & argument_types, const Array & parameters)
{
assertNoParameters(name, parameters);
@ -24,26 +24,26 @@ static IAggregateFunction * createAggregateFunctionSingleValue(const String & na
WhichDataType which(argument_type);
#define DISPATCH(TYPE) \
if (which.idx == TypeIndex::TYPE) return new AggregateFunctionTemplate<Data<SingleValueDataFixed<TYPE>>, false>(argument_type);
if (which.idx == TypeIndex::TYPE) return new AggregateFunctionTemplate<Data<SingleValueDataFixed<TYPE>>>(argument_type);
FOR_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
if (which.idx == TypeIndex::Date)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DataTypeDate::FieldType>>, false>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DataTypeDate::FieldType>>>(argument_type);
if (which.idx == TypeIndex::DateTime)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DataTypeDateTime::FieldType>>, false>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DataTypeDateTime::FieldType>>>(argument_type);
if (which.idx == TypeIndex::DateTime64)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DateTime64>>, false>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<DateTime64>>>(argument_type);
if (which.idx == TypeIndex::Decimal32)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal32>>, false>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal32>>>(argument_type);
if (which.idx == TypeIndex::Decimal64)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal64>>, false>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal64>>>(argument_type);
if (which.idx == TypeIndex::Decimal128)
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal128>>, false>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataFixed<Decimal128>>>(argument_type);
if (which.idx == TypeIndex::String)
return new AggregateFunctionTemplate<Data<SingleValueDataString>, true>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataString>>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataGeneric>, false>(argument_type);
return new AggregateFunctionTemplate<Data<SingleValueDataGeneric>>(argument_type);
}
@ -52,28 +52,29 @@ template <template <typename> class MinMaxData, typename ResData>
static IAggregateFunction * createAggregateFunctionArgMinMaxSecond(const DataTypePtr & res_type, const DataTypePtr & val_type)
{
WhichDataType which(val_type);
#define DISPATCH(TYPE) \
if (which.idx == TypeIndex::TYPE) \
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<TYPE>>>, false>(res_type, val_type);
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<TYPE>>>>(res_type, val_type); \
FOR_NUMERIC_TYPES(DISPATCH)
#undef DISPATCH
if (which.idx == TypeIndex::Date)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DataTypeDate::FieldType>>>, false>(res_type, val_type);
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DataTypeDate::FieldType>>>>(res_type, val_type);
if (which.idx == TypeIndex::DateTime)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DataTypeDateTime::FieldType>>>, false>(res_type, val_type);
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DataTypeDateTime::FieldType>>>>(res_type, val_type);
if (which.idx == TypeIndex::DateTime64)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DateTime64>>>, false>(res_type, val_type);
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<DateTime64>>>>(res_type, val_type);
if (which.idx == TypeIndex::Decimal32)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal32>>>, false>(res_type, val_type);
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal32>>>>(res_type, val_type);
if (which.idx == TypeIndex::Decimal64)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal64>>>, false>(res_type, val_type);
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal64>>>>(res_type, val_type);
if (which.idx == TypeIndex::Decimal128)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal128>>>, false>(res_type, val_type);
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataFixed<Decimal128>>>>(res_type, val_type);
if (which.idx == TypeIndex::String)
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataString>>, true>(res_type, val_type);
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataString>>>(res_type, val_type);
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataGeneric>>, false>(res_type, val_type);
return new AggregateFunctionArgMinMax<AggregateFunctionArgMinMaxData<ResData, MinMaxData<SingleValueDataGeneric>>>(res_type, val_type);
}
template <template <typename> class MinMaxData>

5
dbms/src/Compression/CompressedReadBufferBase.cpp
View File

@ -32,6 +32,7 @@ namespace ErrorCodes
extern const int TOO_LARGE_SIZE_COMPRESSED;
extern const int CHECKSUM_DOESNT_MATCH;
extern const int CANNOT_DECOMPRESS;
extern const int CORRUPTED_DATA;
}
using Checksum = CityHash_v1_0_2::uint128;
@ -124,6 +125,10 @@ size_t CompressedReadBufferBase::readCompressedData(size_t & size_decompressed,
+ ". Most likely corrupted data.",
ErrorCodes::TOO_LARGE_SIZE_COMPRESSED);
if (size_compressed_without_checksum < header_size)
throw Exception("Can't decompress data: the compressed data size (" + toString(size_compressed_without_checksum)
+ ", this should include header size) is less than the header size (" + toString(header_size) + ")", ErrorCodes::CORRUPTED_DATA);
ProfileEvents::increment(ProfileEvents::ReadCompressedBytes, size_compressed_without_checksum + sizeof(Checksum));
/// Is whole compressed block located in 'compressed_in->' buffer?

14
dbms/src/Compression/CompressionCodecMultiple.cpp
View File

@ -5,6 +5,8 @@
#include <Compression/CompressionFactory.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <IO/WriteBufferFromString.h>
#include <IO/Operators.h>
#include <Common/hex.h>
#include <sstream>
@ -31,16 +33,15 @@ UInt8 CompressionCodecMultiple::getMethodByte() const
String CompressionCodecMultiple::getCodecDesc() const
{
std::ostringstream ss;
for (size_t idx = 0; idx < codecs.size(); idx++)
WriteBufferFromOwnString out;
for (size_t idx = 0; idx < codecs.size(); ++idx)
{
if (idx != 0)
ss << ',' << ' ';
out << ", ";
const auto codec = codecs[idx];
ss << codec->getCodecDesc();
out << codecs[idx]->getCodecDesc();
}
return ss.str();
return out.str();
}
UInt32 CompressionCodecMultiple::getMaxCompressedDataSize(UInt32 uncompressed_size) const
@ -55,7 +56,6 @@ UInt32 CompressionCodecMultiple::getMaxCompressedDataSize(UInt32 uncompressed_si
UInt32 CompressionCodecMultiple::doCompressData(const char * source, UInt32 source_size, char * dest) const
{
PODArray<char> compressed_buf;
PODArray<char> uncompressed_buf(source, source + source_size);

9
dbms/src/Compression/ICompressionCodec.cpp
View File

@ -8,6 +8,7 @@
#include <Common/typeid_cast.h>
#include <Compression/CompressionFactory.h>
namespace ProfileEvents
{
extern const Event ReadCompressedBytes;
@ -25,6 +26,7 @@ namespace ErrorCodes
extern const int UNKNOWN_COMPRESSION_METHOD;
extern const int CANNOT_DECOMPRESS;
extern const int SEEK_POSITION_OUT_OF_BOUND;
extern const int CORRUPTED_DATA;
}
@ -42,11 +44,16 @@ UInt32 ICompressionCodec::compress(const char * source, UInt32 source_size, char
UInt32 ICompressionCodec::decompress(const char * source, UInt32 source_size, char * dest) const
{
UInt8 header_size = getHeaderSize();
if (source_size < header_size)
throw Exception("Can't decompress data: the compressed data size (" + toString(source_size)
+ ", this should include header size) is less than the header size (" + toString(header_size) + ")", ErrorCodes::CORRUPTED_DATA);
UInt8 method = source[0];
if (method != getMethodByte())
throw Exception("Can't decompress data with codec byte " + toString(method) + " from codec with byte " + toString(method), ErrorCodes::CANNOT_DECOMPRESS);
UInt8 header_size = getHeaderSize();
UInt32 decompressed_size = readDecompressedBlockSize(source);
doDecompressData(&source[header_size], source_size - header_size, dest, decompressed_size);

6
dbms/src/Compression/ICompressionCodec.h
View File

@ -33,10 +33,10 @@ public:
virtual String getCodecDesc() const = 0;
/// Compressed bytes from uncompressed source to dest. Dest should preallocate memory
virtual UInt32 compress(const char * source, UInt32 source_size, char * dest) const;
UInt32 compress(const char * source, UInt32 source_size, char * dest) const;
/// Decompress bytes from compressed source to dest. Dest should preallocate memory
virtual UInt32 decompress(const char * source, UInt32 source_size, char * dest) const;
UInt32 decompress(const char * source, UInt32 source_size, char * dest) const;
/// Number of bytes, that will be used to compress uncompressed_size bytes with current codec
virtual UInt32 getCompressedReserveSize(UInt32 uncompressed_size) const { return getHeaderSize() + getMaxCompressedDataSize(uncompressed_size); }
@ -45,7 +45,7 @@ public:
virtual UInt32 getAdditionalSizeAtTheEndOfBuffer() const { return 0; }
/// Size of header in compressed data on disk
static UInt8 getHeaderSize() { return COMPRESSED_BLOCK_HEADER_SIZE; }
static constexpr UInt8 getHeaderSize() { return COMPRESSED_BLOCK_HEADER_SIZE; }
/// Read size of compressed block from compressed source
static UInt32 readCompressedBlockSize(const char * source);

9
dbms/src/Core/Settings.h
View File

@ -358,11 +358,8 @@ struct Settings : public SettingsCollection<Settings>
M(SettingBool, enable_unaligned_array_join, false, "Allow ARRAY JOIN with multiple arrays that have different sizes. When this settings is enabled, arrays will be resized to the longest one.", 0) \
M(SettingBool, optimize_read_in_order, true, "Enable ORDER BY optimization for reading data in corresponding order in MergeTree tables.", 0) \
M(SettingBool, low_cardinality_allow_in_native_format, true, "Use LowCardinality type in Native format. Otherwise, convert LowCardinality columns to ordinary for select query, and convert ordinary columns to required LowCardinality for insert query.", 0) \
M(SettingBool, allow_experimental_multiple_joins_emulation, true, "Emulate multiple joins using subselects", 0) \
M(SettingBool, allow_experimental_cross_to_join_conversion, true, "Convert CROSS JOIN to INNER JOIN if possible", 0) \
M(SettingBool, cancel_http_readonly_queries_on_client_close, false, "Cancel HTTP readonly queries when a client closes the connection without waiting for response.", 0) \
M(SettingBool, external_table_functions_use_nulls, true, "If it is set to true, external table functions will implicitly use Nullable type if needed. Otherwise NULLs will be substituted with default values. Currently supported only by 'mysql' and 'odbc' table functions.", 0) \
M(SettingBool, allow_experimental_data_skipping_indices, false, "If it is set to true, data skipping indices can be used in CREATE TABLE/ALTER TABLE queries.", 0) \
\
M(SettingBool, experimental_use_processors, false, "Use processors pipeline.", 0) \
\
@ -390,11 +387,13 @@ struct Settings : public SettingsCollection<Settings>
/** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
\
M(SettingBool, allow_experimental_low_cardinality_type, true, "Obsolete setting, does nothing. Will be removed after 2019-08-13", 0) \
M(SettingBool, compile, false, "Obsolete setting, does nothing. Will be removed after 2020-03-13", 0) \
M(SettingBool, compile, false, "Whether query compilation is enabled. Will be removed after 2020-03-13", 0) \
M(SettingUInt64, min_count_to_compile, 0, "Obsolete setting, does nothing. Will be removed after 2020-03-13", 0) \
M(SettingBool, allow_experimental_multiple_joins_emulation, true, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \
M(SettingBool, allow_experimental_cross_to_join_conversion, true, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \
M(SettingBool, allow_experimental_data_skipping_indices, true, "Obsolete setting, does nothing. Will be removed after 2020-05-31", 0) \
M(SettingBool, merge_tree_uniform_read_distribution, true, "Obsolete setting, does nothing. Will be removed after 2020-05-20", 0) \
DECLARE_SETTINGS_COLLECTION(LIST_OF_SETTINGS)
/** Set multiple settings from "profile" (in server configuration file (users.xml), profiles contain groups of multiple settings).

2
dbms/src/DataStreams/ParallelParsingBlockInputStream.cpp
View File

@ -65,7 +65,7 @@ void ParallelParsingBlockInputStream::parserThreadFunction(size_t current_unit_n
*/
ReadBuffer read_buffer(unit.segment.data(), unit.segment.size(), 0);
auto parser = std::make_unique<InputStreamFromInputFormat>(
input_processor_creator(read_buffer, header, context,
input_processor_creator(read_buffer, header,
row_input_format_params, format_settings));
unit.block_ext.block.clear();

4
dbms/src/DataStreams/ParallelParsingBlockInputStream.h
View File

@ -55,14 +55,12 @@ private:
using InputProcessorCreator = std::function<InputFormatPtr(
ReadBuffer & buf,
const Block & header,
const Context & context,
const RowInputFormatParams & params,
const FormatSettings & settings)>;
public:
struct InputCreatorParams
{
const Block & sample;
const Context &context;
const RowInputFormatParams & row_input_format_params;
const FormatSettings &settings;
};
@ -79,7 +77,6 @@ public:
explicit ParallelParsingBlockInputStream(const Params & params)
: header(params.input_creator_params.sample),
context(params.input_creator_params.context),
row_input_format_params(params.input_creator_params.row_input_format_params),
format_settings(params.input_creator_params.settings),
input_processor_creator(params.input_processor_creator),
@ -149,7 +146,6 @@ protected:
private:
const Block header;
const Context context;
const RowInputFormatParams row_input_format_params;
const FormatSettings format_settings;
const InputProcessorCreator input_processor_creator;

2
dbms/src/DataTypes/DataTypeAggregateFunction.cpp
View File

@ -339,7 +339,7 @@ bool DataTypeAggregateFunction::equals(const IDataType & rhs) const
}
static DataTypePtr create(const ASTPtr & arguments)
static DataTypePtr create(const String & /*type_name*/, const ASTPtr & arguments)
{
String function_name;
AggregateFunctionPtr function;

2
dbms/src/DataTypes/DataTypeArray.cpp
View File

@ -508,7 +508,7 @@ size_t DataTypeArray::getNumberOfDimensions() const
}
static DataTypePtr create(const ASTPtr & arguments)
static DataTypePtr create(const String & /*type_name*/, const ASTPtr & arguments)
{
if (!arguments || arguments->children.size() != 1)
throw Exception("Array data type family must have exactly one argument - type of elements", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);

4
dbms/src/DataTypes/DataTypeCustomIPv4AndIPv6.cpp
View File

@ -102,13 +102,13 @@ public:
void registerDataTypeDomainIPv4AndIPv6(DataTypeFactory & factory)
{
factory.registerSimpleDataTypeCustom("IPv4", []
factory.registerSimpleDataTypeCustom("IPv4", [&](const String & /*type_name*/)
{
return std::make_pair(DataTypeFactory::instance().get("UInt32"),
std::make_unique<DataTypeCustomDesc>(std::make_unique<DataTypeCustomFixedName>("IPv4"), std::make_unique<DataTypeCustomIPv4Serialization>()));
});
factory.registerSimpleDataTypeCustom("IPv6", []
factory.registerSimpleDataTypeCustom("IPv6", [&](const String & /*type_name*/)
{
return std::make_pair(DataTypeFactory::instance().get("FixedString(16)"),
std::make_unique<DataTypeCustomDesc>(std::make_unique<DataTypeCustomFixedName>("IPv6"), std::make_unique<DataTypeCustomIPv6Serialization>()));

2
dbms/src/DataTypes/DataTypeCustomSimpleAggregateFunction.cpp
View File

@ -58,7 +58,7 @@ String DataTypeCustomSimpleAggregateFunction::getName() const
}
static std::pair<DataTypePtr, DataTypeCustomDescPtr> create(const ASTPtr & arguments)
static std::pair<DataTypePtr, DataTypeCustomDescPtr> create(const String & /*type_name*/, const ASTPtr & arguments)
{
String function_name;
AggregateFunctionPtr function;

7
dbms/src/DataTypes/DataTypeDate.cpp
View File

@ -113,7 +113,12 @@ bool DataTypeDate::equals(const IDataType & rhs) const
void registerDataTypeDate(DataTypeFactory & factory)
{
factory.registerSimpleDataType("Date", [] { return DataTypePtr(std::make_shared<DataTypeDate>()); }, DataTypeFactory::CaseInsensitive);
const auto & creator = [&](const String & /*type_name*/)
{
return DataTypePtr(std::make_shared<DataTypeDate>());
};
factory.registerSimpleDataType("Date", creator, DataTypeFactory::CaseInsensitive);
}
}

14
dbms/src/DataTypes/DataTypeDateTime.cpp
View File

@ -43,8 +43,8 @@ TimezoneMixin::TimezoneMixin(const String & time_zone_name)
utc_time_zone(DateLUT::instance("UTC"))
{}
DataTypeDateTime::DataTypeDateTime(const String & time_zone_name)
: TimezoneMixin(time_zone_name)
DataTypeDateTime::DataTypeDateTime(const String & time_zone_name, const String & type_name_)
: TimezoneMixin(time_zone_name), type_name(type_name_)
{
}
@ -55,10 +55,10 @@ DataTypeDateTime::DataTypeDateTime(const TimezoneMixin & time_zone_)
String DataTypeDateTime::doGetName() const
{
if (!has_explicit_time_zone)
return "DateTime";
return type_name;
WriteBufferFromOwnString out;
out << "DateTime(" << quote << time_zone.getTimeZone() << ")";
out << type_name << "(" << quote << time_zone.getTimeZone() << ")";
return out.str();
}
@ -194,10 +194,10 @@ namespace ErrorCodes
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
static DataTypePtr create(const ASTPtr & arguments)
static DataTypePtr create(const String & type_name, const ASTPtr & arguments)
{
if (!arguments)
return std::make_shared<DataTypeDateTime>();
return std::make_shared<DataTypeDateTime>("", type_name);
if (arguments->children.size() != 1)
throw Exception("DateTime data type can optionally have only one argument - time zone name", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
@ -206,7 +206,7 @@ static DataTypePtr create(const ASTPtr & arguments)
if (!arg || arg->value.getType() != Field::Types::String)
throw Exception("Parameter for DateTime data type must be string literal", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
return std::make_shared<DataTypeDateTime>(arg->value.get<String>());
return std::make_shared<DataTypeDateTime>(arg->value.get<String>(), type_name);
}
void registerDataTypeDateTime(DataTypeFactory & factory)

4
dbms/src/DataTypes/DataTypeDateTime.h
View File

@ -49,7 +49,7 @@ protected:
class DataTypeDateTime final : public DataTypeNumberBase<UInt32>, public TimezoneMixin
{
public:
explicit DataTypeDateTime(const String & time_zone_name = "");
explicit DataTypeDateTime(const String & time_zone_name = "", const String & type_name_ = "DateTime");
explicit DataTypeDateTime(const TimezoneMixin & time_zone);
static constexpr auto family_name = "DateTime";
@ -75,6 +75,8 @@ public:
bool canBeInsideNullable() const override { return true; }
bool equals(const IDataType & rhs) const override;
private:
const String type_name;
};
}

2
dbms/src/DataTypes/DataTypeDateTime64.cpp
View File

@ -233,7 +233,7 @@ getArgument(const ASTPtr & arguments, size_t argument_index, const char * argume
return argument->value.get<NearestResultType>();
}
static DataTypePtr create64(const ASTPtr & arguments)
static DataTypePtr create64(const String & /*type_name*/, const ASTPtr & arguments)
{
if (!arguments || arguments->size() == 0)
return std::make_shared<DataTypeDateTime64>(DataTypeDateTime64::default_scale);

8
dbms/src/DataTypes/DataTypeDecimalBase.h
View File

@ -195,7 +195,7 @@ const DecimalType<U> decimalResultType(const DataTypeNumber<T> &, const DecimalT
}
template <template <typename> typename DecimalType>
DataTypePtr createDecimal(UInt64 precision_value, UInt64 scale_value)
DataTypePtr createDecimal(UInt64 precision_value, UInt64 scale_value, const String & type_name = "Decimal", bool only_scale = false)
{
if (precision_value < DecimalUtils::minPrecision() || precision_value > DecimalUtils::maxPrecision<Decimal128>())
throw Exception("Wrong precision", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
@ -204,10 +204,10 @@ DataTypePtr createDecimal(UInt64 precision_value, UInt64 scale_value)
throw Exception("Negative scales and scales larger than precision are not supported", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
if (precision_value <= DecimalUtils::maxPrecision<Decimal32>())
return std::make_shared<DecimalType<Decimal32>>(precision_value, scale_value);
return std::make_shared<DecimalType<Decimal32>>(precision_value, scale_value, type_name, only_scale);
else if (precision_value <= DecimalUtils::maxPrecision<Decimal64>())
return std::make_shared<DecimalType<Decimal64>>(precision_value, scale_value);
return std::make_shared<DecimalType<Decimal128>>(precision_value, scale_value);
return std::make_shared<DecimalType<Decimal64>>(precision_value, scale_value, type_name, only_scale);
return std::make_shared<DecimalType<Decimal128>>(precision_value, scale_value, type_name, only_scale);
}
}

8
dbms/src/DataTypes/DataTypeEnum.cpp
View File

@ -364,7 +364,7 @@ static void checkASTStructure(const ASTPtr & child)
}
template <typename DataTypeEnum>
static DataTypePtr createExact(const ASTPtr & arguments)
static DataTypePtr createExact(const String & /*type_name*/, const ASTPtr & arguments)
{
if (!arguments || arguments->children.empty())
throw Exception("Enum data type cannot be empty", ErrorCodes::EMPTY_DATA_PASSED);
@ -403,7 +403,7 @@ static DataTypePtr createExact(const ASTPtr & arguments)
return std::make_shared<DataTypeEnum>(values);
}
static DataTypePtr create(const ASTPtr & arguments)
static DataTypePtr create(const String & type_name, const ASTPtr & arguments)
{
if (!arguments || arguments->children.empty())
throw Exception("Enum data type cannot be empty", ErrorCodes::EMPTY_DATA_PASSED);
@ -424,10 +424,10 @@ static DataTypePtr create(const ASTPtr & arguments)
Int64 value = value_literal->value.get<Int64>();
if (value > std::numeric_limits<Int8>::max() || value < std::numeric_limits<Int8>::min())
return createExact<DataTypeEnum16>(arguments);
return createExact<DataTypeEnum16>(type_name, arguments);
}
return createExact<DataTypeEnum8>(arguments);
return createExact<DataTypeEnum8>(type_name, arguments);
}
void registerDataTypeEnum(DataTypeFactory & factory)

14
dbms/src/DataTypes/DataTypeFactory.cpp
View File

@ -74,7 +74,7 @@ DataTypePtr DataTypeFactory::get(const String & family_name_param, const ASTPtr
return get("LowCardinality", low_cardinality_params);
}
return findCreatorByName(family_name)(parameters);
return findCreatorByName(family_name)(family_name_param, parameters);
}
@ -107,19 +107,19 @@ void DataTypeFactory::registerSimpleDataType(const String & name, SimpleCreator
throw Exception("DataTypeFactory: the data type " + name + " has been provided "
" a null constructor", ErrorCodes::LOGICAL_ERROR);
registerDataType(name, [name, creator](const ASTPtr & ast)
registerDataType(name, [name, creator](const String & type_name, const ASTPtr & ast)
{
if (ast)
throw Exception("Data type " + name + " cannot have arguments", ErrorCodes::DATA_TYPE_CANNOT_HAVE_ARGUMENTS);
return creator();
return creator(type_name);
}, case_sensitiveness);
}
void DataTypeFactory::registerDataTypeCustom(const String & family_name, CreatorWithCustom creator, CaseSensitiveness case_sensitiveness)
{
registerDataType(family_name, [creator](const ASTPtr & ast)
registerDataType(family_name, [creator](const String & type_name, const ASTPtr & ast)
{
auto res = creator(ast);
auto res = creator(type_name, ast);
res.first->setCustomization(std::move(res.second));
return res.first;
@ -128,9 +128,9 @@ void DataTypeFactory::registerDataTypeCustom(const String & family_name, Creator
void DataTypeFactory::registerSimpleDataTypeCustom(const String & name, SimpleCreatorWithCustom creator, CaseSensitiveness case_sensitiveness)
{
registerDataTypeCustom(name, [creator](const ASTPtr & /*ast*/)
registerDataTypeCustom(name, [creator](const String & type_name, const ASTPtr & /*ast*/)
{
return creator();
return creator(type_name);
}, case_sensitiveness);
}

9
dbms/src/DataTypes/DataTypeFactory.h
View File

@ -16,16 +16,15 @@ namespace DB
class IDataType;
using DataTypePtr = std::shared_ptr<const IDataType>;
/** Creates a data type by name of data type family and parameters.
*/
class DataTypeFactory final : private boost::noncopyable, public IFactoryWithAliases<std::function<DataTypePtr(const ASTPtr & parameters)>>
class DataTypeFactory final : private boost::noncopyable, public IFactoryWithAliases<std::function<DataTypePtr(const String & type_name, const ASTPtr & parameters)>>
{
private:
using SimpleCreator = std::function<DataTypePtr()>;
using SimpleCreator = std::function<DataTypePtr(const String & type_name)>;
using DataTypesDictionary = std::unordered_map<String, Creator>;
using CreatorWithCustom = std::function<std::pair<DataTypePtr,DataTypeCustomDescPtr>(const ASTPtr & parameters)>;
using SimpleCreatorWithCustom = std::function<std::pair<DataTypePtr,DataTypeCustomDescPtr>()>;
using CreatorWithCustom = std::function<std::pair<DataTypePtr, DataTypeCustomDescPtr>(const String & type_name, const ASTPtr & parameters)>;
using SimpleCreatorWithCustom = std::function<std::pair<DataTypePtr, DataTypeCustomDescPtr>(const String & type_name)>;
public:
static DataTypeFactory & instance();

6
dbms/src/DataTypes/DataTypeFixedString.cpp
View File

@ -34,7 +34,7 @@ namespace ErrorCodes
std::string DataTypeFixedString::doGetName() const
{
return "FixedString(" + toString(n) + ")";
return type_name + "(" + toString(n) + ")";
}
@ -279,7 +279,7 @@ bool DataTypeFixedString::equals(const IDataType & rhs) const
}
static DataTypePtr create(const ASTPtr & arguments)
static DataTypePtr create(const String & type_name, const ASTPtr & arguments)
{
if (!arguments || arguments->children.size() != 1)
throw Exception("FixedString data type family must have exactly one argument - size in bytes", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
@ -288,7 +288,7 @@ static DataTypePtr create(const ASTPtr & arguments)
if (!argument || argument->value.getType() != Field::Types::UInt64 || argument->value.get<UInt64>() == 0)
throw Exception("FixedString data type family must have a number (positive integer) as its argument", ErrorCodes::UNEXPECTED_AST_STRUCTURE);
return std::make_shared<DataTypeFixedString>(argument->value.get<UInt64>());
return std::make_shared<DataTypeFixedString>(argument->value.get<UInt64>(), type_name);
}

5
dbms/src/DataTypes/DataTypeFixedString.h
View File

@ -22,7 +22,7 @@ private:
public:
static constexpr bool is_parametric = true;
DataTypeFixedString(size_t n_) : n(n_)
DataTypeFixedString(size_t n_, const String & type_name_ = "FixedString") : n(n_), type_name(type_name_)
{
if (n == 0)
throw Exception("FixedString size must be positive", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
@ -85,6 +85,9 @@ public:
bool isCategorial() const override { return true; }
bool canBeInsideNullable() const override { return true; }
bool canBeInsideLowCardinality() const override { return true; }
private:
const String type_name;
};
}

21
dbms/src/DataTypes/DataTypeInterval.cpp
View File

@ -10,17 +10,22 @@ bool DataTypeInterval::equals(const IDataType & rhs) const
return typeid(rhs) == typeid(*this) && kind == static_cast<const DataTypeInterval &>(rhs).kind;
}
template <IntervalKind::Kind kind>
static DataTypePtr create(const String & /*type_name*/)
{
return DataTypePtr(std::make_shared<DataTypeInterval>(kind));
}
void registerDataTypeInterval(DataTypeFactory & factory)
{
factory.registerSimpleDataType("IntervalSecond", [] { return DataTypePtr(std::make_shared<DataTypeInterval>(IntervalKind::Second)); });
factory.registerSimpleDataType("IntervalMinute", [] { return DataTypePtr(std::make_shared<DataTypeInterval>(IntervalKind::Minute)); });
factory.registerSimpleDataType("IntervalHour", [] { return DataTypePtr(std::make_shared<DataTypeInterval>(IntervalKind::Hour)); });
factory.registerSimpleDataType("IntervalDay", [] { return DataTypePtr(std::make_shared<DataTypeInterval>(IntervalKind::Day)); });
factory.registerSimpleDataType("IntervalWeek", [] { return DataTypePtr(std::make_shared<DataTypeInterval>(IntervalKind::Week)); });
factory.registerSimpleDataType("IntervalMonth", [] { return DataTypePtr(std::make_shared<DataTypeInterval>(IntervalKind::Month)); });
factory.registerSimpleDataType("IntervalQuarter", [] { return DataTypePtr(std::make_shared<DataTypeInterval>(IntervalKind::Quarter)); });
factory.registerSimpleDataType("IntervalYear", [] { return DataTypePtr(std::make_shared<DataTypeInterval>(IntervalKind::Year)); });
factory.registerSimpleDataType("IntervalSecond", create<IntervalKind::Second>);
factory.registerSimpleDataType("IntervalMinute", create<IntervalKind::Minute>);
factory.registerSimpleDataType("IntervalHour", create<IntervalKind::Hour>);
factory.registerSimpleDataType("IntervalDay", create<IntervalKind::Day>);
factory.registerSimpleDataType("IntervalWeek", create<IntervalKind::Week>);
factory.registerSimpleDataType("IntervalMonth", create<IntervalKind::Month>);
factory.registerSimpleDataType("IntervalQuarter", create<IntervalKind::Quarter>);
factory.registerSimpleDataType("IntervalYear", create<IntervalKind::Year>);
}
}

2
dbms/src/DataTypes/DataTypeLowCardinality.cpp
View File

@ -949,7 +949,7 @@ bool DataTypeLowCardinality::equals(const IDataType & rhs) const
}
static DataTypePtr create(const ASTPtr & arguments)
static DataTypePtr create(const String & /*type_name*/, const ASTPtr & arguments)
{
if (!arguments || arguments->children.size() != 1)
throw Exception("LowCardinality data type family must have single argument - type of elements",

4
dbms/src/DataTypes/DataTypeNothing.cpp
View File

@ -38,7 +38,9 @@ bool DataTypeNothing::equals(const IDataType & rhs) const
void registerDataTypeNothing(DataTypeFactory & factory)
{
factory.registerSimpleDataType("Nothing", [] { return DataTypePtr(std::make_shared<DataTypeNothing>()); });
const auto & creator = [&](const String & /*type_name*/) { return DataTypePtr(std::make_shared<DataTypeNothing>()); };
factory.registerSimpleDataType("Nothing", creator);
}
}

2
dbms/src/DataTypes/DataTypeNullable.cpp
View File

@ -505,7 +505,7 @@ bool DataTypeNullable::equals(const IDataType & rhs) const
}
static DataTypePtr create(const ASTPtr & arguments)
static DataTypePtr create(const String & /*type_name*/, const ASTPtr & arguments)
{
if (!arguments || arguments->children.size() != 1)
throw Exception("Nullable data type family must have exactly one argument - nested type", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);

2
dbms/src/DataTypes/DataTypeString.cpp
View File

@ -369,7 +369,7 @@ bool DataTypeString::equals(const IDataType & rhs) const
void registerDataTypeString(DataTypeFactory & factory)
{
auto creator = static_cast<DataTypePtr(*)()>([] { return DataTypePtr(std::make_shared<DataTypeString>()); });
const auto & creator = [&] (const String & type_name) { return std::make_shared<DataTypeString>(type_name); };
factory.registerSimpleDataType("String", creator);

7
dbms/src/DataTypes/DataTypeString.h
View File

@ -14,6 +14,10 @@ public:
using FieldType = String;
static constexpr bool is_parametric = false;
DataTypeString(const String & type_name_ = "String") : type_name(type_name_) {}
String doGetName() const override { return type_name; }
const char * getFamilyName() const override
{
return "String";
@ -63,6 +67,9 @@ public:
bool isCategorial() const override { return true; }
bool canBeInsideNullable() const override { return true; }
bool canBeInsideLowCardinality() const override { return true; }
private:
const String type_name;
};
}

4
dbms/src/DataTypes/DataTypeTuple.cpp
View File

@ -529,7 +529,7 @@ size_t DataTypeTuple::getSizeOfValueInMemory() const
}
static DataTypePtr create(const ASTPtr & arguments)
static DataTypePtr create(const String & /*type_name*/, const ASTPtr & arguments)
{
if (!arguments || arguments->children.empty())
throw Exception("Tuple cannot be empty", ErrorCodes::EMPTY_DATA_PASSED);
@ -568,7 +568,7 @@ void registerDataTypeTuple(DataTypeFactory & factory)
void registerDataTypeNested(DataTypeFactory & factory)
{
/// Nested(...) data type is just a sugar for Array(Tuple(...))
factory.registerDataType("Nested", [&factory](const ASTPtr & arguments)
factory.registerDataType("Nested", [&factory](const String & /*type_name*/, const ASTPtr & arguments)
{
return std::make_shared<DataTypeArray>(factory.get("Tuple", arguments));
});

4
dbms/src/DataTypes/DataTypeUUID.cpp
View File

@ -106,7 +106,9 @@ bool DataTypeUUID::equals(const IDataType & rhs) const
void registerDataTypeUUID(DataTypeFactory & factory)
{
factory.registerSimpleDataType("UUID", [] { return DataTypePtr(std::make_shared<DataTypeUUID>()); });
const auto & creator = [&] (const String & /*type_name*/) { return std::make_shared<DataTypeUUID>(); };
factory.registerSimpleDataType("UUID", creator);
}
}

23
dbms/src/DataTypes/DataTypesDecimal.cpp
View File

@ -14,6 +14,8 @@
#include <Parsers/IAST.h>
#include <type_traits>
#include "DataTypesDecimal.h"
namespace DB
{
@ -31,7 +33,12 @@ template <typename T>
std::string DataTypeDecimal<T>::doGetName() const
{
std::stringstream ss;
ss << "Decimal(" << this->precision << ", " << this->scale << ")";
ss << type_name << "(";
if (!only_scale)
ss << this->precision << ", ";
ss << this->scale << ")";
return ss.str();
}
@ -135,8 +142,14 @@ void DataTypeDecimal<T>::deserializeProtobuf(IColumn & column, ProtobufReader &
container.back() = decimal;
}
template<typename T>
DataTypeDecimal<T>::DataTypeDecimal(UInt32 precision_, UInt32 scale_, const String & type_name_, bool only_scale_)
: Base(precision_, scale_), type_name(type_name_), only_scale(only_scale_)
{
}
static DataTypePtr create(const ASTPtr & arguments)
static DataTypePtr create(const String & type_name, const ASTPtr & arguments)
{
if (!arguments || arguments->children.size() != 2)
throw Exception("Decimal data type family must have exactly two arguments: precision and scale",
@ -152,11 +165,11 @@ static DataTypePtr create(const ASTPtr & arguments)
UInt64 precision_value = precision->value.get<UInt64>();
UInt64 scale_value = scale->value.get<UInt64>();
return createDecimal<DataTypeDecimal>(precision_value, scale_value);
return createDecimal<DataTypeDecimal>(precision_value, scale_value, type_name);
}
template <typename T>
static DataTypePtr createExact(const ASTPtr & arguments)
static DataTypePtr createExact(const String & type_name, const ASTPtr & arguments)
{
if (!arguments || arguments->children.size() != 1)
throw Exception("Decimal data type family must have exactly two arguments: precision and scale",
@ -170,7 +183,7 @@ static DataTypePtr createExact(const ASTPtr & arguments)
UInt64 precision = DecimalUtils::maxPrecision<T>();
UInt64 scale = scale_arg->value.get<UInt64>();
return createDecimal<DataTypeDecimal>(precision, scale);
return createDecimal<DataTypeDecimal>(precision, scale, type_name, true);
}
void registerDataTypeDecimal(DataTypeFactory & factory)

8
dbms/src/DataTypes/DataTypesDecimal.h
View File

@ -35,6 +35,8 @@ public:
using typename Base::ColumnType;
using Base::Base;
DataTypeDecimal(UInt32 precision_, UInt32 scale_, const String & type_name_ = "Decimal", bool only_scale_ = false);
static constexpr auto family_name = "Decimal";
const char * getFamilyName() const override { return family_name; }
@ -57,6 +59,12 @@ public:
static void readText(T & x, ReadBuffer & istr, UInt32 precision_, UInt32 scale_, bool csv = false);
static bool tryReadText(T & x, ReadBuffer & istr, UInt32 precision_, UInt32 scale_);
private:
/// The name of data type how the user specified it. A single data type may be referenced by various synonims.
const String type_name;
/// If the user specified it only with scale parameter but without precision.
bool only_scale = false;
};
template <typename T>

26
dbms/src/DataTypes/DataTypesNumber.cpp
View File

@ -5,20 +5,26 @@
namespace DB
{
template <typename NumberType>
static DataTypePtr create(const String & type_name)
{
return DataTypePtr(std::make_shared<NumberType>(type_name));
}
void registerDataTypeNumbers(DataTypeFactory & factory)
{
factory.registerSimpleDataType("UInt8", [] { return DataTypePtr(std::make_shared<DataTypeUInt8>()); });
factory.registerSimpleDataType("UInt16", [] { return DataTypePtr(std::make_shared<DataTypeUInt16>()); });
factory.registerSimpleDataType("UInt32", [] { return DataTypePtr(std::make_shared<DataTypeUInt32>()); });
factory.registerSimpleDataType("UInt64", [] { return DataTypePtr(std::make_shared<DataTypeUInt64>()); });
factory.registerSimpleDataType("UInt8", create<DataTypeUInt8>);
factory.registerSimpleDataType("UInt16", create<DataTypeUInt16>);
factory.registerSimpleDataType("UInt32", create<DataTypeUInt32>);
factory.registerSimpleDataType("UInt64", create<DataTypeUInt64>);
factory.registerSimpleDataType("Int8", [] { return DataTypePtr(std::make_shared<DataTypeInt8>()); });
factory.registerSimpleDataType("Int16", [] { return DataTypePtr(std::make_shared<DataTypeInt16>()); });
factory.registerSimpleDataType("Int32", [] { return DataTypePtr(std::make_shared<DataTypeInt32>()); });
factory.registerSimpleDataType("Int64", [] { return DataTypePtr(std::make_shared<DataTypeInt64>()); });
factory.registerSimpleDataType("Int8", create<DataTypeInt8>);
factory.registerSimpleDataType("Int16", create<DataTypeInt16>);
factory.registerSimpleDataType("Int32", create<DataTypeInt32>);
factory.registerSimpleDataType("Int64", create<DataTypeInt64>);
factory.registerSimpleDataType("Float32", [] { return DataTypePtr(std::make_shared<DataTypeFloat32>()); });
factory.registerSimpleDataType("Float64", [] { return DataTypePtr(std::make_shared<DataTypeFloat64>()); });
factory.registerSimpleDataType("Float32", create<DataTypeFloat32>);
factory.registerSimpleDataType("Float64", create<DataTypeFloat64>);
/// These synonyms are added for compatibility.

7
dbms/src/DataTypes/DataTypesNumber.h
View File

@ -25,6 +25,13 @@ class DataTypeNumber final : public DataTypeNumberBase<T>
using PromotedType = DataTypeNumber<NearestFieldType<T>>;
return std::make_shared<PromotedType>();
}
public:
DataTypeNumber(const String & type_name_ = TypeName<T>::get()) : type_name(type_name_) {}
String doGetName() const override { return type_name; }
private:
const String type_name;
};
using DataTypeUInt8 = DataTypeNumber<UInt8>;

1
dbms/src/DataTypes/IDataType.h
View File

@ -534,6 +534,7 @@ struct WhichDataType
inline bool isDate(const DataTypePtr & data_type) { return WhichDataType(data_type).isDate(); }
inline bool isDateOrDateTime(const DataTypePtr & data_type) { return WhichDataType(data_type).isDateOrDateTime(); }
inline bool isDateTime(const DataTypePtr & data_type) { return WhichDataType(data_type).isDateTime(); }
inline bool isDateTime64(const DataTypePtr & data_type) { return WhichDataType(data_type).isDateTime64(); }
inline bool isEnum(const DataTypePtr & data_type) { return WhichDataType(data_type).isEnum(); }
inline bool isDecimal(const DataTypePtr & data_type) { return WhichDataType(data_type).isDecimal(); }

14
dbms/src/Databases/DatabaseLazy.cpp
View File

@ -30,22 +30,22 @@ namespace ErrorCodes
DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_, time_t expiration_time_, const Context & context)
DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_, time_t expiration_time_, const Context & context_)
: name(name_)
, metadata_path(metadata_path_)
, data_path(context.getPath() + "data/" + escapeForFileName(name) + "/")
, data_path("data/" + escapeForFileName(name) + "/")
, expiration_time(expiration_time_)
, log(&Logger::get("DatabaseLazy (" + name + ")"))
{
Poco::File(getDataPath()).createDirectories();
Poco::File(context_.getPath() + getDataPath()).createDirectories();
}
void DatabaseLazy::loadStoredObjects(
Context & /* context */,
Context & context,
bool /* has_force_restore_data_flag */)
{
DatabaseOnDisk::iterateMetadataFiles(*this, log, [this](const String & file_name)
DatabaseOnDisk::iterateMetadataFiles(*this, log, context, [this](const String & file_name)
{
const std::string table_name = file_name.substr(0, file_name.size() - 4);
attachTable(table_name, nullptr);
@ -185,9 +185,9 @@ void DatabaseLazy::alterTable(
}
void DatabaseLazy::drop()
void DatabaseLazy::drop(const Context & context)
{
DatabaseOnDisk::drop(*this);
DatabaseOnDisk::drop(*this, context);
}
bool DatabaseLazy::isTableExist(

4
dbms/src/Databases/DatabaseLazy.h
View File

@ -18,7 +18,7 @@ class DatabaseLazyIterator;
class DatabaseLazy : public IDatabase
{
public:
DatabaseLazy(const String & name_, const String & metadata_path_, time_t expiration_time_, const Context & context);
DatabaseLazy(const String & name_, const String & metadata_path_, time_t expiration_time_, const Context & context_);
String getEngineName() const override { return "Lazy"; }
@ -87,7 +87,7 @@ public:
String getMetadataPath() const override;
String getObjectMetadataPath(const String & table_name) const override;
void drop() override;
void drop(const Context & context) override;
bool isTableExist(
const Context & context,

2
dbms/src/Databases/DatabaseMySQL.cpp
View File

@ -332,7 +332,7 @@ void DatabaseMySQL::shutdown()
local_tables_cache.clear();
}
void DatabaseMySQL::drop()
void DatabaseMySQL::drop(const Context & /*context*/)
{
Poco::File(getMetadataPath()).remove(true);
}

2
dbms/src/Databases/DatabaseMySQL.h
View File

@ -60,7 +60,7 @@ public:
void shutdown() override;
void drop() override;
void drop(const Context & /*context*/) override;
String getMetadataPath() const override;

13
dbms/src/Databases/DatabaseOnDisk.cpp
View File

@ -129,7 +129,7 @@ ASTPtr parseCreateQueryFromMetadataFile(const String & filepath, Poco::Logger *
std::pair<String, StoragePtr> createTableFromAST(
ASTCreateQuery ast_create_query,
const String & database_name,
const String & database_data_path,
const String & database_data_path_relative,
Context & context,
bool has_force_restore_data_flag)
{
@ -152,12 +152,13 @@ std::pair<String, StoragePtr> createTableFromAST(
ColumnsDescription columns = InterpreterCreateQuery::getColumnsDescription(*ast_create_query.columns_list->columns, context);
ConstraintsDescription constraints = InterpreterCreateQuery::getConstraintsDescription(ast_create_query.columns_list->constraints);
String table_data_path_relative = database_data_path_relative + escapeForFileName(ast_create_query.table) + '/';
return
{
ast_create_query.table,
StorageFactory::instance().get(
ast_create_query,
database_data_path, ast_create_query.table, database_name, context, context.getGlobalContext(),
table_data_path_relative, ast_create_query.table, database_name, context, context.getGlobalContext(),
columns, constraints,
true, has_force_restore_data_flag)
};
@ -495,9 +496,9 @@ ASTPtr DatabaseOnDisk::getCreateDatabaseQuery(const IDatabase & database, const
return ast;
}
void DatabaseOnDisk::drop(const IDatabase & database)
void DatabaseOnDisk::drop(const IDatabase & database, const Context & context)
{
Poco::File(database.getDataPath()).remove(false);
Poco::File(context.getPath() + database.getDataPath()).remove(false);
Poco::File(database.getMetadataPath()).remove(false);
}
@ -519,7 +520,7 @@ time_t DatabaseOnDisk::getObjectMetadataModificationTime(
return static_cast<time_t>(0);
}
void DatabaseOnDisk::iterateMetadataFiles(const IDatabase & database, Poco::Logger * log, const IteratingFunction & iterating_function)
void DatabaseOnDisk::iterateMetadataFiles(const IDatabase & database, Poco::Logger * log, const Context & context, const IteratingFunction & iterating_function)
{
Poco::DirectoryIterator dir_end;
for (Poco::DirectoryIterator dir_it(database.getMetadataPath()); dir_it != dir_end; ++dir_it)
@ -537,7 +538,7 @@ void DatabaseOnDisk::iterateMetadataFiles(const IDatabase & database, Poco::Logg
if (endsWith(dir_it.name(), tmp_drop_ext))
{
const std::string object_name = dir_it.name().substr(0, dir_it.name().size() - strlen(tmp_drop_ext));
if (Poco::File(database.getDataPath() + '/' + object_name).exists())
if (Poco::File(context.getPath() + database.getDataPath() + '/' + object_name).exists())
{
/// TODO maybe complete table drop and remove all table data (including data on other volumes and metadata in ZK)
Poco::File(dir_it->path()).renameTo(database.getMetadataPath() + object_name + ".sql");

8
dbms/src/Databases/DatabaseOnDisk.h
View File

@ -24,7 +24,7 @@ ASTPtr parseCreateQueryFromMetadataFile(const String & filepath, Poco::Logger *
std::pair<String, StoragePtr> createTableFromAST(
ASTCreateQuery ast_create_query,
const String & database_name,
const String & database_data_path,
const String & database_data_path_relative,
Context & context,
bool has_force_restore_data_flag);
@ -98,7 +98,7 @@ public:
const IDatabase & database,
const Context & context);
static void drop(const IDatabase & database);
static void drop(const IDatabase & database, const Context & context);
static String getObjectMetadataPath(
const IDatabase & database,
@ -110,7 +110,7 @@ public:
using IteratingFunction = std::function<void(const String &)>;
static void iterateMetadataFiles(const IDatabase & database, Poco::Logger * log, const IteratingFunction & iterating_function);
static void iterateMetadataFiles(const IDatabase & database, Poco::Logger * log, const Context & context, const IteratingFunction & iterating_function);
private:
static ASTPtr getCreateTableQueryImpl(
@ -156,7 +156,7 @@ void DatabaseOnDisk::renameTable(
/// Notify the table that it is renamed. If the table does not support renaming, exception is thrown.
try
{
table->rename(context.getPath() + "/data/" + escapeForFileName(to_database_concrete->getDatabaseName()) + "/",
table->rename("/data/" + escapeForFileName(to_database_concrete->getDatabaseName()) + "/" + escapeForFileName(to_table_name) + '/',
to_database_concrete->getDatabaseName(),
to_table_name, lock);
}

17
dbms/src/Databases/DatabaseOrdinary.cpp
View File

@ -59,7 +59,6 @@ namespace
Context & context,
const ASTCreateQuery & query,
DatabaseOrdinary & database,
const String database_data_path,
const String & database_name,
bool has_force_restore_data_flag)
{
@ -69,7 +68,7 @@ namespace
String table_name;
StoragePtr table;
std::tie(table_name, table)
= createTableFromAST(query, database_name, database_data_path, context, has_force_restore_data_flag);
= createTableFromAST(query, database_name, database.getDataPath(), context, has_force_restore_data_flag);
database.attachTable(table_name, table);
}
catch (const Exception & e)
@ -115,13 +114,13 @@ namespace
}
DatabaseOrdinary::DatabaseOrdinary(String name_, const String & metadata_path_, const Context & context)
DatabaseOrdinary::DatabaseOrdinary(String name_, const String & metadata_path_, const Context & context_)
: DatabaseWithOwnTablesBase(std::move(name_))
, metadata_path(metadata_path_)
, data_path(context.getPath() + "data/" + escapeForFileName(name) + "/")
, data_path("data/" + escapeForFileName(name) + "/")
, log(&Logger::get("DatabaseOrdinary (" + name + ")"))
{
Poco::File(getDataPath()).createDirectories();
Poco::File(context_.getPath() + getDataPath()).createDirectories();
}
@ -138,7 +137,7 @@ void DatabaseOrdinary::loadStoredObjects(
FileNames file_names;
size_t total_dictionaries = 0;
DatabaseOnDisk::iterateMetadataFiles(*this, log, [&file_names, &total_dictionaries, this](const String & file_name)
DatabaseOnDisk::iterateMetadataFiles(*this, log, context, [&file_names, &total_dictionaries, this](const String & file_name)
{
String full_path = metadata_path + "/" + file_name;
try
@ -176,7 +175,7 @@ void DatabaseOrdinary::loadStoredObjects(
if (!create_query.is_dictionary)
pool.scheduleOrThrowOnError([&]()
{
tryAttachTable(context, create_query, *this, getDataPath(), getDatabaseName(), has_force_restore_data_flag);
tryAttachTable(context, create_query, *this, getDatabaseName(), has_force_restore_data_flag);
/// Messages, so that it's not boring to wait for the server to load for a long time.
logAboutProgress(log, ++tables_processed, total_tables, watch);
@ -374,9 +373,9 @@ void DatabaseOrdinary::alterTable(
}
void DatabaseOrdinary::drop()
void DatabaseOrdinary::drop(const Context & context)
{
DatabaseOnDisk::drop(*this);
DatabaseOnDisk::drop(*this, context);
}

2
dbms/src/Databases/DatabaseOrdinary.h
View File

@ -83,7 +83,7 @@ public:
String getMetadataPath() const override;
String getObjectMetadataPath(const String & table_name) const override;
void drop() override;
void drop(const Context & context) override;
private:
const String metadata_path;

9
dbms/src/Databases/DatabasesCommon.cpp
View File

@ -1,22 +1,13 @@
#include <Databases/DatabasesCommon.h>
#include <Interpreters/ExternalDictionariesLoader.h>
#include <Interpreters/ExternalLoaderDatabaseConfigRepository.h>
#include <Interpreters/Context.h>
#include <Interpreters/InterpreterCreateQuery.h>
#include <Parsers/ASTCreateQuery.h>
#include <Parsers/ParserCreateQuery.h>
#include <Parsers/ParserDictionary.h>
#include <Parsers/formatAST.h>
#include <Parsers/parseQuery.h>
#include <Storages/IStorage.h>
#include <Storages/StorageDictionary.h>
#include <Storages/StorageFactory.h>
#include <Common/typeid_cast.h>
#include <TableFunctions/TableFunctionFactory.h>
#include <Dictionaries/DictionaryFactory.h>
#include <sstream>
namespace DB

2
dbms/src/Databases/IDatabase.h
View File

@ -237,7 +237,7 @@ public:
virtual void shutdown() = 0;
/// Delete data and metadata stored inside the database, if exists.
virtual void drop() {}
virtual void drop(const Context & /*context*/) {}
virtual ~IDatabase() {}
};

62
dbms/src/Formats/FormatFactory.cpp
View File

@ -12,6 +12,8 @@
#include <Processors/Formats/OutputStreamToOutputFormat.h>
#include <DataStreams/SquashingBlockOutputStream.h>
#include <DataStreams/NativeBlockInputStream.h>
#include <Processors/Formats/Impl/ValuesBlockInputFormat.h>
#include <Processors/Formats/Impl/MySQLOutputFormat.h>
namespace DB
@ -34,7 +36,7 @@ const FormatFactory::Creators & FormatFactory::getCreators(const String & name)
}
static FormatSettings getInputFormatSetting(const Settings & settings)
static FormatSettings getInputFormatSetting(const Settings & settings, const Context & context)
{
FormatSettings format_settings;
format_settings.csv.delimiter = settings.format_csv_delimiter;
@ -56,11 +58,21 @@ static FormatSettings getInputFormatSetting(const Settings & settings)
format_settings.template_settings.row_format = settings.format_template_row;
format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter;
format_settings.tsv.empty_as_default = settings.input_format_tsv_empty_as_default;
format_settings.schema.format_schema = settings.format_schema;
format_settings.schema.format_schema_path = context.getFormatSchemaPath();
format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER);
format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter;
format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter;
format_settings.custom.escaping_rule = settings.format_custom_escaping_rule;
format_settings.custom.field_delimiter = settings.format_custom_field_delimiter;
format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter;
format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter;
format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter;
return format_settings;
}
static FormatSettings getOutputFormatSetting(const Settings & settings)
static FormatSettings getOutputFormatSetting(const Settings & settings, const Context & context)
{
FormatSettings format_settings;
format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers;
@ -77,6 +89,16 @@ static FormatSettings getOutputFormatSetting(const Settings & settings)
format_settings.template_settings.row_between_delimiter = settings.format_template_rows_between_delimiter;
format_settings.write_statistics = settings.output_format_write_statistics;
format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size;
format_settings.schema.format_schema = settings.format_schema;
format_settings.schema.format_schema_path = context.getFormatSchemaPath();
format_settings.schema.is_server = context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER);
format_settings.custom.result_before_delimiter = settings.format_custom_result_before_delimiter;
format_settings.custom.result_after_delimiter = settings.format_custom_result_after_delimiter;
format_settings.custom.escaping_rule = settings.format_custom_escaping_rule;
format_settings.custom.field_delimiter = settings.format_custom_field_delimiter;
format_settings.custom.row_before_delimiter = settings.format_custom_row_before_delimiter;
format_settings.custom.row_after_delimiter = settings.format_custom_row_after_delimiter;
format_settings.custom.row_between_delimiter = settings.format_custom_row_between_delimiter;
return format_settings;
}
@ -100,9 +122,9 @@ BlockInputStreamPtr FormatFactory::getInput(
throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT);
const Settings & settings = context.getSettingsRef();
FormatSettings format_settings = getInputFormatSetting(settings);
FormatSettings format_settings = getInputFormatSetting(settings, context);
return input_getter(buf, sample, context, max_block_size, callback ? callback : ReadCallback(), format_settings);
return input_getter(buf, sample, max_block_size, callback ? callback : ReadCallback(), format_settings);
}
const Settings & settings = context.getSettingsRef();
@ -118,7 +140,7 @@ BlockInputStreamPtr FormatFactory::getInput(
if (!input_getter)
throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT);
FormatSettings format_settings = getInputFormatSetting(settings);
FormatSettings format_settings = getInputFormatSetting(settings, context);
RowInputFormatParams row_input_format_params;
row_input_format_params.max_block_size = max_block_size;
@ -128,7 +150,7 @@ BlockInputStreamPtr FormatFactory::getInput(
row_input_format_params.max_execution_time = settings.max_execution_time;
row_input_format_params.timeout_overflow_mode = settings.timeout_overflow_mode;
auto input_creator_params = ParallelParsingBlockInputStream::InputCreatorParams{sample, context, row_input_format_params, format_settings};
auto input_creator_params = ParallelParsingBlockInputStream::InputCreatorParams{sample, row_input_format_params, format_settings};
ParallelParsingBlockInputStream::Params params{buf, input_getter,
input_creator_params, file_segmentation_engine,
static_cast<int>(settings.max_threads),
@ -164,16 +186,16 @@ BlockOutputStreamPtr FormatFactory::getOutput(
throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
const Settings & settings = context.getSettingsRef();
FormatSettings format_settings = getOutputFormatSetting(settings);
FormatSettings format_settings = getOutputFormatSetting(settings, context);
/** Materialization is needed, because formats can use the functions `IDataType`,
* which only work with full columns.
*/
return std::make_shared<MaterializingBlockOutputStream>(
output_getter(buf, sample, context, callback, format_settings), sample);
output_getter(buf, sample, std::move(callback), format_settings), sample);
}
auto format = getOutputFormat(name, buf, sample, context, callback);
auto format = getOutputFormat(name, buf, sample, context, std::move(callback));
return std::make_shared<MaterializingBlockOutputStream>(std::make_shared<OutputStreamToOutputFormat>(format), sample);
}
@ -191,7 +213,7 @@ InputFormatPtr FormatFactory::getInputFormat(
throw Exception("Format " + name + " is not suitable for input", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_INPUT);
const Settings & settings = context.getSettingsRef();
FormatSettings format_settings = getInputFormatSetting(settings);
FormatSettings format_settings = getInputFormatSetting(settings, context);
RowInputFormatParams params;
params.max_block_size = max_block_size;
@ -201,7 +223,13 @@ InputFormatPtr FormatFactory::getInputFormat(
params.max_execution_time = settings.max_execution_time;
params.timeout_overflow_mode = settings.timeout_overflow_mode;
return input_getter(buf, sample, context, params, format_settings);
auto format = input_getter(buf, sample, params, format_settings);
/// It's a kludge. Because I cannot remove context from values format.
if (auto * values = typeid_cast<ValuesBlockInputFormat *>(format.get()))
values->setContext(context);
return format;
}
@ -213,12 +241,18 @@ OutputFormatPtr FormatFactory::getOutputFormat(
throw Exception("Format " + name + " is not suitable for output", ErrorCodes::FORMAT_IS_NOT_SUITABLE_FOR_OUTPUT);
const Settings & settings = context.getSettingsRef();
FormatSettings format_settings = getOutputFormatSetting(settings);
FormatSettings format_settings = getOutputFormatSetting(settings, context);
/** TODO: Materialization is needed, because formats can use the functions `IDataType`,
* which only work with full columns.
*/
return output_getter(buf, sample, context, callback, format_settings);
auto format = output_getter(buf, sample, std::move(callback), format_settings);
/// It's a kludge. Because I cannot remove context from MySQL format.
if (auto * mysql = typeid_cast<MySQLOutputFormat *>(format.get()))
mysql->setContext(context);
return format;
}
@ -259,7 +293,7 @@ void FormatFactory::registerFileSegmentationEngine(const String & name, FileSegm
auto & target = dict[name].file_segmentation_engine;
if (target)
throw Exception("FormatFactory: File segmentation engine " + name + " is already registered", ErrorCodes::LOGICAL_ERROR);
target = file_segmentation_engine;
target = std::move(file_segmentation_engine);
}
FormatFactory::FormatFactory()

4
dbms/src/Formats/FormatFactory.h
View File

@ -59,7 +59,6 @@ private:
using InputCreator = std::function<BlockInputStreamPtr(
ReadBuffer & buf,
const Block & sample,
const Context & context,
UInt64 max_block_size,
ReadCallback callback,
const FormatSettings & settings)>;
@ -67,21 +66,18 @@ private:
using OutputCreator = std::function<BlockOutputStreamPtr(
WriteBuffer & buf,
const Block & sample,
const Context & context,
WriteCallback callback,
const FormatSettings & settings)>;
using InputProcessorCreator = std::function<InputFormatPtr(
ReadBuffer & buf,
const Block & header,
const Context & context,
const RowInputFormatParams & params,
const FormatSettings & settings)>;
using OutputProcessorCreator = std::function<OutputFormatPtr(
WriteBuffer & buf,
const Block & sample,
const Context & context,
WriteCallback callback,
const FormatSettings & settings)>;

14
dbms/src/Formats/FormatSchemaInfo.cpp
View File

@ -26,7 +26,7 @@ namespace
}
FormatSchemaInfo::FormatSchemaInfo(const Context & context, const String & format_schema, const String & format, bool require_message)
FormatSchemaInfo::FormatSchemaInfo(const String & format_schema, const String & format, bool require_message, bool is_server, const std::string & format_schema_path)
{
if (format_schema.empty())
throw Exception(
@ -54,29 +54,25 @@ FormatSchemaInfo::FormatSchemaInfo(const Context & context, const String & forma
else
path.assign(format_schema).makeFile().getFileName();
auto default_schema_directory = [&context]()
auto default_schema_directory = [&format_schema_path]()
{
static const String str = Poco::Path(context.getFormatSchemaPath()).makeAbsolute().makeDirectory().toString();
static const String str = Poco::Path(format_schema_path).makeAbsolute().makeDirectory().toString();
return str;
};
auto is_server = [&context]()
{
return context.hasGlobalContext() && (context.getGlobalContext().getApplicationType() == Context::ApplicationType::SERVER);
};
if (path.getExtension().empty() && !default_file_extension.empty())
path.setExtension(default_file_extension);
if (path.isAbsolute())
{
if (is_server())
if (is_server)
throw Exception("Absolute path in the 'format_schema' setting is prohibited: " + path.toString(), ErrorCodes::BAD_ARGUMENTS);
schema_path = path.getFileName();
schema_directory = path.makeParent().toString();
}
else if (path.depth() >= 1 && path.directory(0) == "..")
{
if (is_server())
if (is_server)
throw Exception(
"Path in the 'format_schema' setting shouldn't go outside the 'format_schema_path' directory: " + path.toString(),
ErrorCodes::BAD_ARGUMENTS);

2
dbms/src/Formats/FormatSchemaInfo.h
View File

@ -10,7 +10,7 @@ class Context;
class FormatSchemaInfo
{
public:
FormatSchemaInfo(const Context & context, const String & format_schema, const String & format, bool require_message);
FormatSchemaInfo(const String & format_schema, const String & format, bool require_message, bool is_server, const std::string & format_schema_path);
/// Returns path to the schema file.
const String & schemaPath() const { return schema_path; }

21
dbms/src/Formats/FormatSettings.h
View File

@ -89,6 +89,27 @@ struct FormatSettings
UInt64 row_group_size = 1000000;
} parquet;
struct Schema
{
std::string format_schema;
std::string format_schema_path;
bool is_server = false;
};
Schema schema;
struct Custom
{
std::string result_before_delimiter;
std::string result_after_delimiter;
std::string row_before_delimiter;
std::string row_after_delimiter;
std::string row_between_delimiter;
std::string field_delimiter;
std::string escaping_rule;
};
Custom custom;
};
}

2
dbms/src/Formats/NativeFormat.cpp
View File

@ -11,7 +11,6 @@ void registerInputFormatNative(FormatFactory & factory)
factory.registerInputFormat("Native", [](
ReadBuffer & buf,
const Block & sample,
const Context &,
UInt64 /* max_block_size */,
FormatFactory::ReadCallback /* callback */,
const FormatSettings &)
@ -25,7 +24,6 @@ void registerOutputFormatNative(FormatFactory & factory)
factory.registerOutputFormat("Native", [](
WriteBuffer & buf,
const Block & sample,
const Context &,
FormatFactory::WriteCallback,
const FormatSettings &)
{

1
dbms/src/Formats/NullFormat.cpp
View File

@ -10,7 +10,6 @@ void registerOutputFormatNull(FormatFactory & factory)
factory.registerOutputFormat("Null", [](
WriteBuffer &,
const Block & sample,
const Context &,
FormatFactory::WriteCallback,
const FormatSettings &)
{

18
dbms/src/Formats/ParsedTemplateFormatString.cpp
View File

@ -234,36 +234,32 @@ void ParsedTemplateFormatString::throwInvalidFormat(const String & message, size
ErrorCodes::INVALID_TEMPLATE_FORMAT);
}
ParsedTemplateFormatString ParsedTemplateFormatString::setupCustomSeparatedResultsetFormat(const Context & context)
ParsedTemplateFormatString ParsedTemplateFormatString::setupCustomSeparatedResultsetFormat(const FormatSettings::Custom & settings)
{
const Settings & settings = context.getSettingsRef();
/// Set resultset format to "result_before_delimiter ${data} result_after_delimiter"
ParsedTemplateFormatString resultset_format;
resultset_format.delimiters.emplace_back(settings.format_custom_result_before_delimiter);
resultset_format.delimiters.emplace_back(settings.format_custom_result_after_delimiter);
resultset_format.delimiters.emplace_back(settings.result_before_delimiter);
resultset_format.delimiters.emplace_back(settings.result_after_delimiter);
resultset_format.formats.emplace_back(ParsedTemplateFormatString::ColumnFormat::None);
resultset_format.format_idx_to_column_idx.emplace_back(0);
resultset_format.column_names.emplace_back("data");
return resultset_format;
}
ParsedTemplateFormatString ParsedTemplateFormatString::setupCustomSeparatedRowFormat(const Context & context, const Block & sample)
ParsedTemplateFormatString ParsedTemplateFormatString::setupCustomSeparatedRowFormat(const FormatSettings::Custom & settings, const Block & sample)
{
const Settings & settings = context.getSettingsRef();
/// Set row format to
/// "row_before_delimiter ${Col0:escaping} field_delimiter ${Col1:escaping} field_delimiter ... ${ColN:escaping} row_after_delimiter"
ParsedTemplateFormatString::ColumnFormat escaping = ParsedTemplateFormatString::stringToFormat(settings.format_custom_escaping_rule);
ParsedTemplateFormatString::ColumnFormat escaping = ParsedTemplateFormatString::stringToFormat(settings.escaping_rule);
ParsedTemplateFormatString row_format;
row_format.delimiters.emplace_back(settings.format_custom_row_before_delimiter);
row_format.delimiters.emplace_back(settings.row_before_delimiter);
for (size_t i = 0; i < sample.columns(); ++i)
{
row_format.formats.emplace_back(escaping);
row_format.format_idx_to_column_idx.emplace_back(i);
row_format.column_names.emplace_back(sample.getByPosition(i).name);
bool last_column = i == sample.columns() - 1;
row_format.delimiters.emplace_back(last_column ? settings.format_custom_row_after_delimiter : settings.format_custom_field_delimiter);
row_format.delimiters.emplace_back(last_column ? settings.row_after_delimiter : settings.field_delimiter);
}
return row_format;
}

5
dbms/src/Formats/ParsedTemplateFormatString.h
View File

@ -4,6 +4,7 @@
#include <functional>
#include <optional>
#include <Formats/FormatSchemaInfo.h>
#include <Formats/FormatSettings.h>
namespace DB
{
@ -49,8 +50,8 @@ struct ParsedTemplateFormatString
String dump() const;
[[noreturn]] void throwInvalidFormat(const String & message, size_t column) const;
static ParsedTemplateFormatString setupCustomSeparatedResultsetFormat(const Context & context);
static ParsedTemplateFormatString setupCustomSeparatedRowFormat(const Context & context, const Block & sample);
static ParsedTemplateFormatString setupCustomSeparatedResultsetFormat(const FormatSettings::Custom & settings);
static ParsedTemplateFormatString setupCustomSeparatedRowFormat(const FormatSettings::Custom & settings, const Block & sample);
};
}

10
dbms/src/Functions/FunctionBase64Conversion.h
View File

@ -165,4 +165,14 @@ public:
}
};
}
/** We must call it in advance from a single thread
* to avoid thread sanitizer report about data race in "codec_choose" function.
*/
inline void initializeBase64()
{
size_t outlen = 0;
base64_encode(nullptr, 0, nullptr, &outlen, 0);
}
#endif

56
dbms/src/Functions/FunctionHelpers.cpp
View File

@ -19,6 +19,7 @@ namespace ErrorCodes
extern const int ILLEGAL_COLUMN;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int SIZES_OF_ARRAYS_DOESNT_MATCH;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
const ColumnConst * checkAndGetColumnConstStringOrFixedString(const IColumn * column)
@ -124,9 +125,9 @@ namespace
void validateArgumentsImpl(const IFunction & func,
const ColumnsWithTypeAndName & arguments,
size_t argument_offset,
const FunctionArgumentTypeValidators & validators)
const FunctionArgumentDescriptors & descriptors)
{
for (size_t i = 0; i < validators.size(); ++i)
for (size_t i = 0; i < descriptors.size(); ++i)
{
const auto argument_index = i + argument_offset;
if (argument_index >= arguments.size())
@ -135,24 +136,36 @@ void validateArgumentsImpl(const IFunction & func,
}
const auto & arg = arguments[i + argument_offset];
const auto validator = validators[i];
if (!validator.validator_func(*arg.type))
throw Exception("Illegal type " + arg.type->getName() +
" of " + std::to_string(i) +
" argument of function " + func.getName() +
" expected " + validator.expected_type_description,
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
const auto descriptor = descriptors[i];
if (int errorCode = descriptor.isValid(arg.type, arg.column); errorCode != 0)
throw Exception("Illegal type of argument #" + std::to_string(i)
+ (descriptor.argument_name ? " '" + std::string(descriptor.argument_name) + "'" : String{})
+ " of function " + func.getName()
+ (descriptor.expected_type_description ? String(", expected ") + descriptor.expected_type_description : String{})
+ (arg.type ? ", got " + arg.type->getName() : String{}),
errorCode);
}
}
}
int FunctionArgumentDescriptor::isValid(const DataTypePtr & data_type, const ColumnPtr & column) const
{
if (type_validator_func && (data_type == nullptr || type_validator_func(*data_type) == false))
return ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT;
if (column_validator_func && (column == nullptr || column_validator_func(*column) == false))
return ErrorCodes::ILLEGAL_COLUMN;
return 0;
}
void validateFunctionArgumentTypes(const IFunction & func,
const ColumnsWithTypeAndName & arguments,
const FunctionArgumentTypeValidators & mandatory_args,
const FunctionArgumentTypeValidators & optional_args)
const FunctionArgumentDescriptors & mandatory_args,
const FunctionArgumentDescriptors & optional_args)
{
if (arguments.size() < mandatory_args.size())
if (arguments.size() < mandatory_args.size() || arguments.size() > mandatory_args.size() + optional_args.size())
{
auto joinArgumentTypes = [](const auto & args, const String sep = ", ") -> String
{
@ -160,8 +173,13 @@ void validateFunctionArgumentTypes(const IFunction & func,
for (const auto & a : args)
{
using A = std::decay_t<decltype(a)>;
if constexpr (std::is_same_v<A, FunctionArgumentTypeValidator>)
if constexpr (std::is_same_v<A, FunctionArgumentDescriptor>)
{
if (a.argument_name)
result += "'" + std::string(a.argument_name) + "' : ";
if (a.expected_type_description)
result += a.expected_type_description;
}
else if constexpr (std::is_same_v<A, ColumnWithTypeAndName>)
result += a.type->getName();
@ -174,10 +192,14 @@ void validateFunctionArgumentTypes(const IFunction & func,
return result;
};
throw Exception("Incorrect number of arguments of function " + func.getName()
+ " provided " + std::to_string(arguments.size()) + " (" + joinArgumentTypes(arguments) + ")"
+ " expected " + std::to_string(mandatory_args.size()) + (optional_args.size() ? " or " + std::to_string(mandatory_args.size() + optional_args.size()) : "")
+ " (" + joinArgumentTypes(mandatory_args) + (optional_args.size() ? ", [" + joinArgumentTypes(mandatory_args) + "]" : "") + ")",
throw Exception("Incorrect number of arguments for function " + func.getName()
+ " provided " + std::to_string(arguments.size())
+ (arguments.size() ? " (" + joinArgumentTypes(arguments) + ")" : String{})
+ ", expected " + std::to_string(mandatory_args.size())
+ (optional_args.size() ? " to " + std::to_string(mandatory_args.size() + optional_args.size()) : "")
+ " (" + joinArgumentTypes(mandatory_args)
+ (optional_args.size() ? ", [" + joinArgumentTypes(optional_args) + "]" : "")
+ ")",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}

47
dbms/src/Functions/FunctionHelpers.h
View File

@ -90,21 +90,46 @@ void validateArgumentType(const IFunction & func, const DataTypes & arguments,
size_t argument_index, bool (* validator_func)(const IDataType &),
const char * expected_type_description);
// Simple validator that is used in conjunction with validateFunctionArgumentTypes() to check if function arguments are as expected.
struct FunctionArgumentTypeValidator
/** Simple validator that is used in conjunction with validateFunctionArgumentTypes() to check if function arguments are as expected
*
* Also it is used to generate function description when arguments do not match expected ones.
* Any field can be null:
* `argument_name` - if not null, reported via type check errors.
* `expected_type_description` - if not null, reported via type check errors.
* `type_validator_func` - if not null, used to validate data type of function argument.
* `column_validator_func` - if not null, used to validate column of function argument.
*/
struct FunctionArgumentDescriptor
{
bool (* validator_func)(const IDataType &);
const char * argument_name;
bool (* type_validator_func)(const IDataType &);
bool (* column_validator_func)(const IColumn &);
const char * expected_type_description;
/** Validate argument type and column.
*
* Returns non-zero error code if:
* Validator != nullptr && (Value == nullptr || Validator(*Value) == false)
* For:
* Validator is either `type_validator_func` or `column_validator_func`
* Value is either `data_type` or `column` respectively.
* ILLEGAL_TYPE_OF_ARGUMENT if type validation fails
*
*/
int isValid(const DataTypePtr & data_type, const ColumnPtr & column) const;
};
using FunctionArgumentTypeValidators = std::vector<FunctionArgumentTypeValidator>;
using FunctionArgumentDescriptors = std::vector<FunctionArgumentDescriptor>;
/** Validate that function arguments match specification.
*
* Designed to simplify argument validation
* for functions with variable arguments (e.g. depending on result type or other trait).
* first, checks that mandatory args present and have valid type.
* second, checks optional arguents types, skipping ones that are missing.
* Designed to simplify argument validation for functions with variable arguments
* (e.g. depending on result type or other trait).
* First, checks that number of arguments is as expected (including optional arguments).
* Second, checks that mandatory args present and have valid type.
* Third, checks optional arguents types, skipping ones that are missing.
*
* Please note that if you have several optional arguments, like f([a, b, c]),
* only these calls are considered valid:
@ -113,11 +138,13 @@ using FunctionArgumentTypeValidators = std::vector<FunctionArgumentTypeValidator
* f(a, b, c)
*
* But NOT these: f(a, c), f(b, c)
* In other words you can't skip
* In other words you can't omit middle optional arguments (just like in regular C++).
*
* If any mandatory arg is missing, throw an exception, with explicit description of expected arguments.
*/
void validateFunctionArgumentTypes(const IFunction & func, const ColumnsWithTypeAndName & arguments, const FunctionArgumentTypeValidators & mandatory_args, const FunctionArgumentTypeValidators & optional_args = {});
void validateFunctionArgumentTypes(const IFunction & func, const ColumnsWithTypeAndName & arguments,
const FunctionArgumentDescriptors & mandatory_args,
const FunctionArgumentDescriptors & optional_args = {});
/// Checks if a list of array columns have equal offsets. Return a pair of nested columns and offsets if true, otherwise throw.
std::pair<std::vector<const IColumn *>, const ColumnArray::Offset *>

23
dbms/src/Functions/FunctionsConversion.cpp
View File

@ -5,29 +5,6 @@
namespace DB
{
void throwExceptionForIncompletelyParsedValue(
ReadBuffer & read_buffer, Block & block, size_t result)
{
const IDataType & to_type = *block.getByPosition(result).type;
WriteBufferFromOwnString message_buf;
message_buf << "Cannot parse string " << quote << String(read_buffer.buffer().begin(), read_buffer.buffer().size())
<< " as " << to_type.getName()
<< ": syntax error";
if (read_buffer.offset())
message_buf << " at position " << read_buffer.offset()
<< " (parsed just " << quote << String(read_buffer.buffer().begin(), read_buffer.offset()) << ")";
else
message_buf << " at begin of string";
if (isNativeNumber(to_type))
message_buf << ". Note: there are to" << to_type.getName() << "OrZero and to" << to_type.getName() << "OrNull functions, which returns zero/NULL instead of throwing exception.";
throw Exception(message_buf.str(), ErrorCodes::CANNOT_PARSE_TEXT);
}
void registerFunctionsConversion(FunctionFactory & factory)
{
factory.registerFunction<FunctionToUInt8>();

71
dbms/src/Functions/FunctionsConversion.h
View File

@ -501,7 +501,26 @@ inline bool tryParseImpl<DataTypeDateTime>(DataTypeDateTime::FieldType & x, Read
/** Throw exception with verbose message when string value is not parsed completely.
*/
[[noreturn]] void throwExceptionForIncompletelyParsedValue(ReadBuffer & read_buffer, Block & block, size_t result);
[[noreturn]] inline void throwExceptionForIncompletelyParsedValue(ReadBuffer & read_buffer, Block & block, size_t result)
{
const IDataType & to_type = *block.getByPosition(result).type;
WriteBufferFromOwnString message_buf;
message_buf << "Cannot parse string " << quote << String(read_buffer.buffer().begin(), read_buffer.buffer().size())
<< " as " << to_type.getName()
<< ": syntax error";
if (read_buffer.offset())
message_buf << " at position " << read_buffer.offset()
<< " (parsed just " << quote << String(read_buffer.buffer().begin(), read_buffer.offset()) << ")";
else
message_buf << " at begin of string";
if (isNativeNumber(to_type))
message_buf << ". Note: there are to" << to_type.getName() << "OrZero and to" << to_type.getName() << "OrNull functions, which returns zero/NULL instead of throwing exception.";
throw Exception(message_buf.str(), ErrorCodes::CANNOT_PARSE_TEXT);
}
enum class ConvertFromStringExceptionMode
@ -886,6 +905,7 @@ public:
static constexpr bool to_datetime64 = std::is_same_v<ToDataType, DataTypeDateTime64>;
static FunctionPtr create(const Context &) { return std::make_shared<FunctionConvert>(); }
static FunctionPtr create() { return std::make_shared<FunctionConvert>(); }
String getName() const override
{
@ -898,16 +918,25 @@ public:
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
FunctionArgumentTypeValidators mandatory_args = {{[](const auto &) {return true;}, "ANY TYPE"}};
FunctionArgumentTypeValidators optional_args;
FunctionArgumentDescriptors mandatory_args = {{"Value", nullptr, nullptr, nullptr}};
FunctionArgumentDescriptors optional_args;
if constexpr (to_decimal || to_datetime64)
{
mandatory_args.push_back(FunctionArgumentTypeValidator{&isNativeInteger, "Integer"}); // scale
mandatory_args.push_back({"scale", &isNativeInteger, &isColumnConst, "const Integer"});
}
else
// toString(DateTime or DateTime64, [timezone: String])
if ((std::is_same_v<Name, NameToString> && arguments.size() > 0 && (isDateTime64(arguments[0].type) || isDateTime(arguments[0].type)))
// toUnixTimestamp(value[, timezone : String])
|| std::is_same_v<Name, NameToUnixTimestamp>
// toDate(value[, timezone : String])
|| std::is_same_v<ToDataType, DataTypeDate> // TODO: shall we allow timestamp argument for toDate? DateTime knows nothing about timezones and this arument is ignored below.
// toDateTime(value[, timezone: String])
|| std::is_same_v<ToDataType, DataTypeDateTime>
// toDateTime64(value, scale : Integer[, timezone: String])
|| std::is_same_v<ToDataType, DataTypeDateTime64>)
{
optional_args.push_back(FunctionArgumentTypeValidator{&isString, "String"}); // timezone
optional_args.push_back({"timezone", &isString, &isColumnConst, "const String"});
}
validateFunctionArgumentTypes(*this, arguments, mandatory_args, optional_args);
@ -918,8 +947,8 @@ public:
}
else if constexpr (to_decimal)
{
if (!arguments[1].column)
throw Exception("Second argument for function " + getName() + " must be constant", ErrorCodes::ILLEGAL_COLUMN);
// if (!arguments[1].column)
// throw Exception("Second argument for function " + getName() + " must be constant", ErrorCodes::ILLEGAL_COLUMN);
UInt64 scale = extractToDecimalScale(arguments[1]);
@ -1083,6 +1112,7 @@ public:
std::is_same_v<ToDataType, DataTypeDecimal<Decimal128>>;
static FunctionPtr create(const Context &) { return std::make_shared<FunctionConvertFromString>(); }
static FunctionPtr create() { return std::make_shared<FunctionConvertFromString>(); }
String getName() const override
{
@ -1231,6 +1261,7 @@ class FunctionToFixedString : public IFunction
public:
static constexpr auto name = "toFixedString";
static FunctionPtr create(const Context &) { return std::make_shared<FunctionToFixedString>(); }
static FunctionPtr create() { return std::make_shared<FunctionToFixedString>(); }
String getName() const override
{
@ -1686,9 +1717,9 @@ public:
using WrapperType = std::function<void(Block &, const ColumnNumbers &, size_t, size_t)>;
using MonotonicityForRange = std::function<Monotonicity(const IDataType &, const Field &, const Field &)>;
FunctionCast(const Context & context_, const char * name_, MonotonicityForRange && monotonicity_for_range_
FunctionCast(const char * name_, MonotonicityForRange && monotonicity_for_range_
, const DataTypes & argument_types_, const DataTypePtr & return_type_)
: context(context_), name(name_), monotonicity_for_range(monotonicity_for_range_)
: name(name_), monotonicity_for_range(monotonicity_for_range_)
, argument_types(argument_types_), return_type(return_type_)
{
}
@ -1719,7 +1750,6 @@ public:
private:
const Context & context;
const char * name;
MonotonicityForRange monotonicity_for_range;
@ -1735,10 +1765,10 @@ private:
{
/// In case when converting to Nullable type, we apply different parsing rule,
/// that will not throw an exception but return NULL in case of malformed input.
function = FunctionConvertFromString<DataType, NameCast, ConvertFromStringExceptionMode::Null>::create(context);
function = FunctionConvertFromString<DataType, NameCast, ConvertFromStringExceptionMode::Null>::create();
}
else
function = FunctionTo<DataType>::Type::create(context);
function = FunctionTo<DataType>::Type::create();
auto function_adaptor =
FunctionOverloadResolverAdaptor(std::make_unique<DefaultOverloadResolver>(function))
@ -1752,7 +1782,7 @@ private:
WrapperType createStringWrapper(const DataTypePtr & from_type) const
{
FunctionPtr function = FunctionToString::create(context);
FunctionPtr function = FunctionToString::create();
auto function_adaptor =
FunctionOverloadResolverAdaptor(std::make_unique<DefaultOverloadResolver>(function))
@ -1780,7 +1810,7 @@ private:
if (requested_result_is_nullable)
throw Exception{"CAST AS Nullable(UUID) is not implemented", ErrorCodes::NOT_IMPLEMENTED};
FunctionPtr function = FunctionTo<DataTypeUUID>::Type::create(context);
FunctionPtr function = FunctionTo<DataTypeUUID>::Type::create();
auto function_adaptor =
FunctionOverloadResolverAdaptor(std::make_unique<DefaultOverloadResolver>(function))
@ -1985,7 +2015,7 @@ private:
return createStringToEnumWrapper<ColumnFixedString, EnumType>();
else if (isNativeNumber(from_type) || isEnum(from_type))
{
auto function = Function::create(context);
auto function = Function::create();
auto func_or_adaptor = FunctionOverloadResolverAdaptor(std::make_unique<DefaultOverloadResolver>(function))
.build(ColumnsWithTypeAndName{{nullptr, from_type, "" }});
@ -2337,9 +2367,10 @@ public:
using MonotonicityForRange = FunctionCast::MonotonicityForRange;
static constexpr auto name = "CAST";
static FunctionOverloadResolverImplPtr create(const Context & context) { return std::make_unique<CastOverloadResolver>(context); }
static FunctionOverloadResolverImplPtr create(const Context &) { return createImpl(); }
static FunctionOverloadResolverImplPtr createImpl() { return std::make_unique<CastOverloadResolver>(); }
CastOverloadResolver(const Context & context_) : context(context_) {}
CastOverloadResolver() {}
String getName() const override { return name; }
@ -2357,7 +2388,7 @@ protected:
data_types[i] = arguments[i].type;
auto monotonicity = getMonotonicityInformation(arguments.front().type, return_type.get());
return std::make_unique<FunctionCast>(context, name, std::move(monotonicity), data_types, return_type);
return std::make_unique<FunctionCast>(name, std::move(monotonicity), data_types, return_type);
}
DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments) const override
@ -2418,8 +2449,6 @@ private:
/// other types like Null, FixedString, Array and Tuple have no monotonicity defined
return {};
}
const Context & context;
};
}

5
dbms/src/Functions/GeoUtils.h
View File

@ -261,6 +261,10 @@ void PointInPolygonWithGrid<CoordinateType>::buildGrid()
for (size_t row = 0; row < grid_size; ++row)
{
#pragma GCC diagnostic push
#if !__clang__
#pragma GCC diagnostic ignored "-Wmaybe-uninitialized"
#endif
CoordinateType y_min = min_corner.y() + row * cell_height;
CoordinateType y_max = min_corner.y() + (row + 1) * cell_height;
@ -268,6 +272,7 @@ void PointInPolygonWithGrid<CoordinateType>::buildGrid()
{
CoordinateType x_min = min_corner.x() + col * cell_width;
CoordinateType x_max = min_corner.x() + (col + 1) * cell_width;
#pragma GCC diagnostic pop
Box cell_box(Point(x_min, y_min), Point(x_max, y_max));
Polygon cell_bound;

225
dbms/src/Functions/currentRowPolicies.cpp Normal file
View File

@ -0,0 +1,225 @@
#include <Functions/IFunction.h>
#include <Functions/FunctionFactory.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeString.h>
#include <DataTypes/DataTypeTuple.h>
#include <DataTypes/DataTypeUUID.h>
#include <Columns/ColumnArray.h>
#include <Columns/ColumnConst.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnTuple.h>
#include <Interpreters/Context.h>
#include <Access/RowPolicyContext.h>
#include <Access/AccessControlManager.h>
#include <ext/range.h>
namespace DB
{
namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
}
/// The currentRowPolicies() function can be called with 0..2 arguments:
/// currentRowPolicies() returns array of tuples (database, table_name, row_policy_name) for all the row policies applied for the current user;
/// currentRowPolicies(table_name) is equivalent to currentRowPolicies(currentDatabase(), table_name);
/// currentRowPolicies(database, table_name) returns array of names of the row policies applied to a specific table and for the current user.
class FunctionCurrentRowPolicies : public IFunction
{
public:
static constexpr auto name = "currentRowPolicies";
static FunctionPtr create(const Context & context_) { return std::make_shared<FunctionCurrentRowPolicies>(context_); }
explicit FunctionCurrentRowPolicies(const Context & context_) : context(context_) {}
String getName() const override { return name; }
size_t getNumberOfArguments() const override { return 0; }
bool isVariadic() const override { return true; }
void checkNumberOfArgumentsIfVariadic(size_t number_of_arguments) const override
{
if (number_of_arguments > 2)
throw Exception("Number of arguments for function " + String(name) + " doesn't match: passed "
+ toString(number_of_arguments) + ", should be 0..2",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (arguments.empty())
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeTuple>(
DataTypes{std::make_shared<DataTypeString>(), std::make_shared<DataTypeString>(), std::make_shared<DataTypeString>()}));
else
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
}
bool isDeterministic() const override { return false; }
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result_pos, size_t input_rows_count) override
{
if (arguments.empty())
{
auto database_column = ColumnString::create();
auto table_name_column = ColumnString::create();
auto policy_name_column = ColumnString::create();
for (const auto & policy_id : context.getRowPolicy()->getCurrentPolicyIDs())
{
const auto policy = context.getAccessControlManager().tryRead<RowPolicy>(policy_id);
if (policy)
{
const String database = policy->getDatabase();
const String table_name = policy->getTableName();
const String policy_name = policy->getName();
database_column->insertData(database.data(), database.length());
table_name_column->insertData(table_name.data(), table_name.length());
policy_name_column->insertData(policy_name.data(), policy_name.length());
}
}
auto offset_column = ColumnArray::ColumnOffsets::create();
offset_column->insertValue(policy_name_column->size());
block.getByPosition(result_pos).column = ColumnConst::create(
ColumnArray::create(
ColumnTuple::create(Columns{std::move(database_column), std::move(table_name_column), std::move(policy_name_column)}),
std::move(offset_column)),
input_rows_count);
return;
}
const IColumn * database_column = nullptr;
if (arguments.size() == 2)
{
const auto & database_column_with_type = block.getByPosition(arguments[0]);
if (!isStringOrFixedString(database_column_with_type.type))
throw Exception{"The first argument of function " + String(name)
+ " should be a string containing database name, illegal type: "
+ database_column_with_type.type->getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
database_column = database_column_with_type.column.get();
}
const auto & table_name_column_with_type = block.getByPosition(arguments[arguments.size() - 1]);
if (!isStringOrFixedString(table_name_column_with_type.type))
throw Exception{"The" + String(database_column ? " last" : "") + " argument of function " + String(name)
+ " should be a string containing table name, illegal type: " + table_name_column_with_type.type->getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
const IColumn * table_name_column = table_name_column_with_type.column.get();
auto policy_name_column = ColumnString::create();
auto offset_column = ColumnArray::ColumnOffsets::create();
for (const auto i : ext::range(0, input_rows_count))
{
String database = database_column ? database_column->getDataAt(i).toString() : context.getCurrentDatabase();
String table_name = table_name_column->getDataAt(i).toString();
for (const auto & policy_id : context.getRowPolicy()->getCurrentPolicyIDs(database, table_name))
{
const auto policy = context.getAccessControlManager().tryRead<RowPolicy>(policy_id);
if (policy)
{
const String policy_name = policy->getName();
policy_name_column->insertData(policy_name.data(), policy_name.length());
}
}
offset_column->insertValue(policy_name_column->size());
}
block.getByPosition(result_pos).column = ColumnArray::create(std::move(policy_name_column), std::move(offset_column));
}
private:
const Context & context;
};
/// The currentRowPolicyIDs() function can be called with 0..2 arguments:
/// currentRowPolicyIDs() returns array of IDs of all the row policies applied for the current user;
/// currentRowPolicyIDs(table_name) is equivalent to currentRowPolicyIDs(currentDatabase(), table_name);
/// currentRowPolicyIDs(database, table_name) returns array of IDs of the row policies applied to a specific table and for the current user.
class FunctionCurrentRowPolicyIDs : public IFunction
{
public:
static constexpr auto name = "currentRowPolicyIDs";
static FunctionPtr create(const Context & context_) { return std::make_shared<FunctionCurrentRowPolicyIDs>(context_); }
explicit FunctionCurrentRowPolicyIDs(const Context & context_) : context(context_) {}
String getName() const override { return name; }
size_t getNumberOfArguments() const override { return 0; }
bool isVariadic() const override { return true; }
void checkNumberOfArgumentsIfVariadic(size_t number_of_arguments) const override
{
if (number_of_arguments > 2)
throw Exception("Number of arguments for function " + String(name) + " doesn't match: passed "
+ toString(number_of_arguments) + ", should be 0..2",
ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH);
}
DataTypePtr getReturnTypeImpl(const DataTypes & /* arguments */) const override
{
return std::make_shared<DataTypeArray>(std::make_shared<DataTypeUUID>());
}
bool isDeterministic() const override { return false; }
void executeImpl(Block & block, const ColumnNumbers & arguments, size_t result_pos, size_t input_rows_count) override
{
if (arguments.empty())
{
auto policy_id_column = ColumnVector<UInt128>::create();
for (const auto & policy_id : context.getRowPolicy()->getCurrentPolicyIDs())
policy_id_column->insertValue(policy_id);
auto offset_column = ColumnArray::ColumnOffsets::create();
offset_column->insertValue(policy_id_column->size());
block.getByPosition(result_pos).column
= ColumnConst::create(ColumnArray::create(std::move(policy_id_column), std::move(offset_column)), input_rows_count);
return;
}
const IColumn * database_column = nullptr;
if (arguments.size() == 2)
{
const auto & database_column_with_type = block.getByPosition(arguments[0]);
if (!isStringOrFixedString(database_column_with_type.type))
throw Exception{"The first argument of function " + String(name)
+ " should be a string containing database name, illegal type: "
+ database_column_with_type.type->getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
database_column = database_column_with_type.column.get();
}
const auto & table_name_column_with_type = block.getByPosition(arguments[arguments.size() - 1]);
if (!isStringOrFixedString(table_name_column_with_type.type))
throw Exception{"The" + String(database_column ? " last" : "") + " argument of function " + String(name)
+ " should be a string containing table name, illegal type: " + table_name_column_with_type.type->getName(),
ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT};
const IColumn * table_name_column = table_name_column_with_type.column.get();
auto policy_id_column = ColumnVector<UInt128>::create();
auto offset_column = ColumnArray::ColumnOffsets::create();
for (const auto i : ext::range(0, input_rows_count))
{
String database = database_column ? database_column->getDataAt(i).toString() : context.getCurrentDatabase();
String table_name = table_name_column->getDataAt(i).toString();
for (const auto & policy_id : context.getRowPolicy()->getCurrentPolicyIDs(database, table_name))
policy_id_column->insertValue(policy_id);
offset_column->insertValue(policy_id_column->size());
}
block.getByPosition(result_pos).column = ColumnArray::create(std::move(policy_id_column), std::move(offset_column));
}
private:
const Context & context;
};
void registerFunctionCurrentRowPolicies(FunctionFactory & factory)
{
factory.registerFunction<FunctionCurrentRowPolicies>();
factory.registerFunction<FunctionCurrentRowPolicyIDs>();
}
}

1
dbms/src/Functions/registerFunctions.h
View File

@ -9,6 +9,7 @@ class FunctionFactory;
void registerFunctionCurrentDatabase(FunctionFactory &);
void registerFunctionCurrentUser(FunctionFactory &);
void registerFunctionCurrentQuota(FunctionFactory &);
void registerFunctionCurrentRowPolicies(FunctionFactory &);
void registerFunctionHostName(FunctionFactory &);
void registerFunctionFQDN(FunctionFactory &);
void registerFunctionVisibleWidth(FunctionFactory &);

1
dbms/src/Functions/registerFunctionsMiscellaneous.cpp
View File

@ -8,6 +8,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory)
registerFunctionCurrentDatabase(factory);
registerFunctionCurrentUser(factory);
registerFunctionCurrentQuota(factory);
registerFunctionCurrentRowPolicies(factory);
registerFunctionHostName(factory);
registerFunctionFQDN(factory);
registerFunctionVisibleWidth(factory);

12
dbms/src/Interpreters/BloomFilterHash.h
View File

@ -85,8 +85,16 @@ struct BloomFilterHash
throw Exception("Unexpected type " + data_type->getName() + " of bloom filter index.", ErrorCodes::LOGICAL_ERROR);
const auto & offsets = array_col->getOffsets();
size_t offset = (pos == 0) ? 0 : offsets[pos - 1];
limit = std::max(array_col->getData().size() - offset, limit);
limit = offsets[pos + limit - 1] - offsets[pos - 1]; /// PaddedPODArray allows access on index -1.
pos = offsets[pos - 1];
if (limit == 0)
{
auto index_column = ColumnUInt64::create(1);
ColumnUInt64::Container & index_column_vec = index_column->getData();
index_column_vec[0] = 0;
return index_column;
}
}
const ColumnPtr actual_col = BloomFilter::getPrimitiveColumn(column);

39
dbms/src/Interpreters/Context.cpp
View File

@ -28,6 +28,7 @@
#include <Access/AccessControlManager.h>
#include <Access/SettingsConstraints.h>
#include <Access/QuotaContext.h>
#include <Access/RowPolicyContext.h>
#include <Interpreters/ExpressionJIT.h>
#include <Interpreters/UsersManager.h>
#include <Dictionaries/Embedded/GeoDictionariesLoader.h>
@ -333,6 +334,7 @@ Context Context::createGlobal()
{
Context res;
res.quota = std::make_shared<QuotaContext>();
res.row_policy = std::make_shared<RowPolicyContext>();
res.shared = std::make_shared<ContextShared>();
return res;
}
@ -625,6 +627,13 @@ void Context::checkQuotaManagementIsAllowed()
"User " + client_info.current_user + " doesn't have enough privileges to manage quotas", ErrorCodes::NOT_ENOUGH_PRIVILEGES);
}
void Context::checkRowPolicyManagementIsAllowed()
{
if (!is_row_policy_management_allowed)
throw Exception(
"User " + client_info.current_user + " doesn't have enough privileges to manage row policies", ErrorCodes::NOT_ENOUGH_PRIVILEGES);
}
void Context::setUsersConfig(const ConfigurationPtr & config)
{
auto lock = getLock();
@ -639,34 +648,6 @@ ConfigurationPtr Context::getUsersConfig()
return shared->users_config;
}
bool Context::hasUserProperty(const String & database, const String & table, const String & name) const
{
auto lock = getLock();
// No user - no properties.
if (client_info.current_user.empty())
return false;
const auto & props = shared->users_manager->getUser(client_info.current_user)->table_props;
auto db = props.find(database);
if (db == props.end())
return false;
auto table_props = db->second.find(table);
if (table_props == db->second.end())
return false;
return !!table_props->second.count(name);
}
const String & Context::getUserProperty(const String & database, const String & table, const String & name) const
{
auto lock = getLock();
const auto & props = shared->users_manager->getUser(client_info.current_user)->table_props;
return props.at(database).at(table).at(name);
}
void Context::calculateUserSettings()
{
auto lock = getLock();
@ -691,6 +672,8 @@ void Context::calculateUserSettings()
quota = getAccessControlManager().createQuotaContext(
client_info.current_user, client_info.current_address.host(), client_info.quota_key);
is_quota_management_allowed = user->is_quota_management_allowed;
row_policy = getAccessControlManager().getRowPolicyContext(client_info.current_user);
is_row_policy_management_allowed = user->is_row_policy_management_allowed;
}

9
dbms/src/Interpreters/Context.h
View File

@ -45,6 +45,7 @@ namespace DB
struct ContextShared;
class Context;
class QuotaContext;
class RowPolicyContext;
class EmbeddedDictionaries;
class ExternalDictionariesLoader;
class ExternalModelsLoader;
@ -140,6 +141,8 @@ private:
std::shared_ptr<QuotaContext> quota; /// Current quota. By default - empty quota, that have no limits.
bool is_quota_management_allowed = false; /// Whether the current user is allowed to manage quotas via SQL commands.
std::shared_ptr<RowPolicyContext> row_policy;
bool is_row_policy_management_allowed = false; /// Whether the current user is allowed to manage row policies via SQL commands.
String current_database;
Settings settings; /// Setting for query execution.
std::shared_ptr<const SettingsConstraints> settings_constraints;
@ -209,6 +212,8 @@ public:
const AccessControlManager & getAccessControlManager() const;
std::shared_ptr<QuotaContext> getQuota() const { return quota; }
void checkQuotaManagementIsAllowed();
std::shared_ptr<RowPolicyContext> getRowPolicy() const { return row_policy; }
void checkRowPolicyManagementIsAllowed();
/** Take the list of users, quotas and configuration profiles from this config.
* The list of users is completely replaced.
@ -217,10 +222,6 @@ public:
void setUsersConfig(const ConfigurationPtr & config);
ConfigurationPtr getUsersConfig();
// User property is a key-value pair from the configuration entry: users.<username>.databases.<db_name>.<table_name>.<key_name>
bool hasUserProperty(const String & database, const String & table, const String & name) const;
const String & getUserProperty(const String & database, const String & table, const String & name) const;
/// Must be called before getClientInfo.
void setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address, const String & quota_key);

3
dbms/src/Interpreters/DDLWorker.cpp
View File

@ -938,8 +938,6 @@ void DDLWorker::runMainThread()
bool initialized = false;
do
{
try
{
try
{
@ -957,7 +955,6 @@ void DDLWorker::runMainThread()
/// Avoid busy loop when ZooKeeper is not available.
sleepForSeconds(1);
}
}
catch (...)
{
tryLogCurrentException(log, "Terminating. Cannot initialize DDL queue.");

2
dbms/src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -632,7 +632,7 @@ bool InterpreterCreateQuery::doCreateTable(const ASTCreateQuery & create,
else
{
res = StorageFactory::instance().get(create,
data_path,
data_path + escapeForFileName(table_name) + "/",
table_name,
create.database,
context,

93
dbms/src/Interpreters/InterpreterCreateRowPolicyQuery.cpp Normal file
View File

@ -0,0 +1,93 @@
#include <Interpreters/InterpreterCreateRowPolicyQuery.h>
#include <Parsers/ASTCreateRowPolicyQuery.h>
#include <Parsers/ASTRoleList.h>
#include <Parsers/formatAST.h>
#include <Interpreters/Context.h>
#include <Access/AccessControlManager.h>
#include <boost/range/algorithm/sort.hpp>
namespace DB
{
BlockIO InterpreterCreateRowPolicyQuery::execute()
{
context.checkRowPolicyManagementIsAllowed();
const auto & query = query_ptr->as<const ASTCreateRowPolicyQuery &>();
auto & access_control = context.getAccessControlManager();
if (query.alter)
{
auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr
{
auto updated_policy = typeid_cast<std::shared_ptr<RowPolicy>>(entity->clone());
updateRowPolicyFromQuery(*updated_policy, query);
return updated_policy;
};
String full_name = query.name_parts.getFullName(context);
if (query.if_exists)
{
if (auto id = access_control.find<RowPolicy>(full_name))
access_control.tryUpdate(*id, update_func);
}
else
access_control.update(access_control.getID<RowPolicy>(full_name), update_func);
}
else
{
auto new_policy = std::make_shared<RowPolicy>();
updateRowPolicyFromQuery(*new_policy, query);
if (query.if_not_exists)
access_control.tryInsert(new_policy);
else if (query.or_replace)
access_control.insertOrReplace(new_policy);
else
access_control.insert(new_policy);
}
return {};
}
void InterpreterCreateRowPolicyQuery::updateRowPolicyFromQuery(RowPolicy & policy, const ASTCreateRowPolicyQuery & query)
{
if (query.alter)
{
if (!query.new_policy_name.empty())
policy.setName(query.new_policy_name);
}
else
{
policy.setDatabase(query.name_parts.database.empty() ? context.getCurrentDatabase() : query.name_parts.database);
policy.setTableName(query.name_parts.table_name);
policy.setName(query.name_parts.policy_name);
}
if (query.is_restrictive)
policy.setRestrictive(*query.is_restrictive);
for (const auto & [index, condition] : query.conditions)
policy.conditions[index] = condition ? serializeAST(*condition) : String{};
if (query.roles)
{
const auto & query_roles = *query.roles;
/// We keep `roles` sorted.
policy.roles = query_roles.roles;
if (query_roles.current_user)
policy.roles.push_back(context.getClientInfo().current_user);
boost::range::sort(policy.roles);
policy.roles.erase(std::unique(policy.roles.begin(), policy.roles.end()), policy.roles.end());
policy.all_roles = query_roles.all_roles;
/// We keep `except_roles` sorted.
policy.except_roles = query_roles.except_roles;
if (query_roles.except_current_user)
policy.except_roles.push_back(context.getClientInfo().current_user);
boost::range::sort(policy.except_roles);
policy.except_roles.erase(std::unique(policy.except_roles.begin(), policy.except_roles.end()), policy.except_roles.end());
}
}
}

26
dbms/src/Interpreters/InterpreterCreateRowPolicyQuery.h Normal file
View File

@ -0,0 +1,26 @@
#pragma once
#include <Interpreters/IInterpreter.h>
#include <Parsers/IAST_fwd.h>
namespace DB
{
class ASTCreateRowPolicyQuery;
struct RowPolicy;
class InterpreterCreateRowPolicyQuery : public IInterpreter
{
public:
InterpreterCreateRowPolicyQuery(const ASTPtr & query_ptr_, Context & context_) : query_ptr(query_ptr_), context(context_) {}
BlockIO execute() override;
private:
void updateRowPolicyFromQuery(RowPolicy & policy, const ASTCreateRowPolicyQuery & query);
ASTPtr query_ptr;
Context & context;
};
}

Some files were not shown because too many files have changed in this diff Show More