diff --git a/.gitmodules b/.gitmodules index f6990fed41..e5be5438cc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -97,9 +97,6 @@ [submodule "contrib/rapidjson"] path = contrib/rapidjson url = https://github.com/Tencent/rapidjson -[submodule "contrib/mimalloc"] - path = contrib/mimalloc - url = https://github.com/ClickHouse-Extras/mimalloc [submodule "contrib/fastops"] path = contrib/fastops url = https://github.com/ClickHouse-Extras/fastops diff --git a/CMakeLists.txt b/CMakeLists.txt index fb4ca18126..5330c8daeb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -340,7 +340,6 @@ include (cmake/find_consistent-hashing.cmake) include (cmake/find_base64.cmake) include (cmake/find_parquet.cmake) include (cmake/find_hyperscan.cmake) -include (cmake/find_mimalloc.cmake) include (cmake/find_simdjson.cmake) include (cmake/find_rapidjson.cmake) include (cmake/find_fastops.cmake) diff --git a/cmake/find_mimalloc.cmake b/cmake/find_mimalloc.cmake deleted file mode 100644 index 1820421379..0000000000 --- a/cmake/find_mimalloc.cmake +++ /dev/null @@ -1,17 +0,0 @@ -if (OS_LINUX AND NOT SANITIZE AND NOT ARCH_ARM AND NOT ARCH_32 AND NOT ARCH_PPC64LE) - option (ENABLE_MIMALLOC "Set to FALSE to disable usage of mimalloc for internal ClickHouse caches" FALSE) -endif () - -if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include/mimalloc.h") - message (WARNING "submodule contrib/mimalloc is missing. to fix try run: \n git submodule update --init --recursive") - return() -endif () - -if (ENABLE_MIMALLOC) - message (FATAL_ERROR "Mimalloc is not production ready. (Disable with cmake -D ENABLE_MIMALLOC=0). If you want to use mimalloc, you must manually remove this message.") - - set (MIMALLOC_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/mimalloc/include) - set (USE_MIMALLOC 1) - set (MIMALLOC_LIBRARY mimalloc-static) - message (STATUS "Using mimalloc: ${MIMALLOC_INCLUDE_DIR} : ${MIMALLOC_LIBRARY}") -endif () diff --git a/contrib/mimalloc b/contrib/mimalloc deleted file mode 160000 index a787bdebce..0000000000 --- a/contrib/mimalloc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit a787bdebce94bf3776dc0d1ad597917f479ab8d5 diff --git a/dbms/CMakeLists.txt b/dbms/CMakeLists.txt index 98eb23809d..e7cc084237 100644 --- a/dbms/CMakeLists.txt +++ b/dbms/CMakeLists.txt @@ -256,11 +256,6 @@ if(RE2_INCLUDE_DIR) target_include_directories(clickhouse_common_io SYSTEM BEFORE PUBLIC ${RE2_INCLUDE_DIR}) endif() -if (USE_MIMALLOC) - target_include_directories (clickhouse_common_io SYSTEM BEFORE PUBLIC ${MIMALLOC_INCLUDE_DIR}) - target_link_libraries (clickhouse_common_io PRIVATE ${MIMALLOC_LIBRARY}) -endif () - if(CPUID_LIBRARY) target_link_libraries(clickhouse_common_io PRIVATE ${CPUID_LIBRARY}) endif() diff --git a/dbms/src/Common/MiAllocator.cpp b/dbms/src/Common/MiAllocator.cpp deleted file mode 100644 index 04e61a5de1..0000000000 --- a/dbms/src/Common/MiAllocator.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include "MiAllocator.h" - -#if USE_MIMALLOC -#include - -#include -#include -#include - -namespace DB -{ -namespace ErrorCodes -{ - extern const int CANNOT_ALLOCATE_MEMORY; -} - -void * MiAllocator::alloc(size_t size, size_t alignment) -{ - void * ptr; - if (alignment == 0) - { - ptr = mi_malloc(size); - if (!ptr) - DB::throwFromErrno("MiAllocator: Cannot allocate in mimalloc " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - } - else - { - ptr = mi_malloc_aligned(size, alignment); - if (!ptr) - DB::throwFromErrno("MiAllocator: Cannot allocate in mimalloc (mi_malloc_aligned) " + formatReadableSizeWithBinarySuffix(size) + " with alignment " + toString(alignment) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - } - return ptr; -} - -void MiAllocator::free(void * buf, size_t) -{ - mi_free(buf); -} - -void * MiAllocator::realloc(void * old_ptr, size_t, size_t new_size, size_t alignment) -{ - if (old_ptr == nullptr) - return alloc(new_size, alignment); - - if (new_size == 0) - { - mi_free(old_ptr); - return nullptr; - } - - void * ptr; - - if (alignment == 0) - { - ptr = mi_realloc(old_ptr, alignment); - if (!ptr) - DB::throwFromErrno("MiAllocator: Cannot reallocate in mimalloc " + formatReadableSizeWithBinarySuffix(size) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - } - else - { - ptr = mi_realloc_aligned(old_ptr, new_size, alignment); - if (!ptr) - DB::throwFromErrno("MiAllocator: Cannot reallocate in mimalloc (mi_realloc_aligned) " + formatReadableSizeWithBinarySuffix(size) + " with alignment " + toString(alignment) + ".", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - } - return ptr; -} - -} - -#endif diff --git a/dbms/src/Common/MiAllocator.h b/dbms/src/Common/MiAllocator.h deleted file mode 100644 index 127be82434..0000000000 --- a/dbms/src/Common/MiAllocator.h +++ /dev/null @@ -1,27 +0,0 @@ -#pragma once - -#include - -#if USE_MIMALLOC -#include - -namespace DB -{ - -/* - * This is a different allocator that is based on mimalloc (Microsoft malloc). - * It can be used separately from main allocator to catch heap corruptions and vulnerabilities (for example, for caches). - * We use MI_SECURE mode in mimalloc to achieve such behaviour. - */ -struct MiAllocator -{ - static void * alloc(size_t size, size_t alignment = 0); - - static void free(void * buf, size_t); - - static void * realloc(void * old_ptr, size_t, size_t new_size, size_t alignment = 0); -}; - -} - -#endif diff --git a/dbms/src/Common/config.h.in b/dbms/src/Common/config.h.in index b6a5b6de2b..7804068e5c 100644 --- a/dbms/src/Common/config.h.in +++ b/dbms/src/Common/config.h.in @@ -8,6 +8,5 @@ #cmakedefine01 USE_CPUID #cmakedefine01 USE_CPUINFO #cmakedefine01 USE_BROTLI -#cmakedefine01 USE_MIMALLOC #cmakedefine01 USE_UNWIND #cmakedefine01 CLICKHOUSE_SPLIT_BINARY diff --git a/dbms/src/Common/tests/CMakeLists.txt b/dbms/src/Common/tests/CMakeLists.txt index 2c99c85bae..67c0e376f7 100644 --- a/dbms/src/Common/tests/CMakeLists.txt +++ b/dbms/src/Common/tests/CMakeLists.txt @@ -76,8 +76,5 @@ target_link_libraries (cow_compositions PRIVATE clickhouse_common_io) add_executable (stopwatch stopwatch.cpp) target_link_libraries (stopwatch PRIVATE clickhouse_common_io) -add_executable (mi_malloc_test mi_malloc_test.cpp) -target_link_libraries (mi_malloc_test PRIVATE clickhouse_common_io) - add_executable (symbol_index symbol_index.cpp) target_link_libraries (symbol_index PRIVATE clickhouse_common_io) diff --git a/dbms/src/Common/tests/mi_malloc_test.cpp b/dbms/src/Common/tests/mi_malloc_test.cpp deleted file mode 100644 index ce1e4a3a77..0000000000 --- a/dbms/src/Common/tests/mi_malloc_test.cpp +++ /dev/null @@ -1,118 +0,0 @@ -/** In addition to ClickHouse (Apache 2) license, this file can be also used under MIT license: - -MIT License - -Copyright (c) 2019 Yandex LLC, Alexey Milovidov - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -*/ - -#include -#include -#include -#include -#include -#include - -#include - -//#undef USE_MIMALLOC -//#define USE_MIMALLOC 0 - -#if USE_MIMALLOC - -#include -#define malloc mi_malloc -#define free mi_free - -#else - -#include - -#endif - - -size_t total_size{0}; - -struct Allocation -{ - void * ptr = nullptr; - size_t size = 0; - - Allocation() {} - - Allocation(size_t size_) - : size(size_) - { - ptr = malloc(size); - if (!ptr) - throw std::runtime_error("Cannot allocate memory"); - total_size += size; - } - - ~Allocation() - { - if (ptr) - { - free(ptr); - total_size -= size; - } - ptr = nullptr; - } - - Allocation(const Allocation &) = delete; - - Allocation(Allocation && rhs) - { - ptr = rhs.ptr; - size = rhs.size; - rhs.ptr = nullptr; - rhs.size = 0; - } -}; - - -int main(int, char **) -{ - std::vector allocations; - - constexpr size_t limit = 100000000; - constexpr size_t min_alloc_size = 65536; - constexpr size_t max_alloc_size = 10000000; - - std::mt19937 rng; - auto distribution = std::uniform_int_distribution(min_alloc_size, max_alloc_size); - - size_t total_allocations = 0; - - while (true) - { - size_t size = distribution(rng); - - while (total_size + size > limit) - allocations.pop_back(); - - allocations.emplace_back(size); - - ++total_allocations; - if (total_allocations % (1ULL << 20) == 0) - std::cerr << "Total allocations: " << total_allocations << "\n"; - } -} diff --git a/dbms/src/DataStreams/MarkInCompressedFile.h b/dbms/src/DataStreams/MarkInCompressedFile.h index a5970a8973..46d078f2b7 100644 --- a/dbms/src/DataStreams/MarkInCompressedFile.h +++ b/dbms/src/DataStreams/MarkInCompressedFile.h @@ -6,10 +6,6 @@ #include #include -#include -#if USE_MIMALLOC -#include -#endif namespace DB { @@ -43,9 +39,7 @@ struct MarkInCompressedFile } }; -#if USE_MIMALLOC -using MarksInCompressedFile = PODArray; -#else + using MarksInCompressedFile = PODArray; -#endif + } diff --git a/dbms/src/IO/UncompressedCache.h b/dbms/src/IO/UncompressedCache.h index 1f17c5e61b..86f1530e5b 100644 --- a/dbms/src/IO/UncompressedCache.h +++ b/dbms/src/IO/UncompressedCache.h @@ -6,11 +6,6 @@ #include #include -#include -#if USE_MIMALLOC -#include -#endif - namespace ProfileEvents { @@ -25,11 +20,7 @@ namespace DB struct UncompressedCacheCell { -#if USE_MIMALLOC - Memory data; -#else Memory<> data; -#endif size_t compressed_size; UInt32 additional_bytes; };