maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Use u16 to store Count/MaxCount 

The Count/MaxCount used in TransferBatch and PerClass can be fit in u16 in
current configurations and it's also reasonable to have a u16 limit. The
spare 16 bits will be used for additional status like pages mapping
status in a TransferBatch.

Reviewed By: cryptoad, cferris, vitalybuka

Differential Revision: https://reviews.llvm.org/D133145
This commit is contained in:
Chia-hung Duan 2022-10-13 22:03:06 +00:00
parent 3d8d9c9884
commit 0fb2aeef53
10 changed files with 44 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
compiler-rt/lib/scudo/standalone/local_cache.h
View File

@ -10,6 +10,7 @@
#define SCUDO_LOCAL_CACHE_H_ #define SCUDO_LOCAL_CACHE_H_
#include "internal_defs.h" #include "internal_defs.h"
#include "platform.h"
#include "report.h" #include "report.h"
#include "stats.h" #include "stats.h"
@ -20,8 +21,8 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
typedef typename SizeClassAllocator::CompactPtrT CompactPtrT; typedef typename SizeClassAllocator::CompactPtrT CompactPtrT;
struct TransferBatch { struct TransferBatch {
static const u32 MaxNumCached = SizeClassMap::MaxNumCachedHint; static const u16 MaxNumCached = SizeClassMap::MaxNumCachedHint;
void setFromArray(CompactPtrT *Array, u32 N) { void setFromArray(CompactPtrT *Array, u16 N) {
DCHECK_LE(N, MaxNumCached); DCHECK_LE(N, MaxNumCached);
Count = N; Count = N;
memcpy(Batch, Array, sizeof(Batch[0]) * Count); memcpy(Batch, Array, sizeof(Batch[0]) * Count);
@ -34,19 +35,19 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
void copyToArray(CompactPtrT *Array) const { void copyToArray(CompactPtrT *Array) const {
memcpy(Array, Batch, sizeof(Batch[0]) * Count); memcpy(Array, Batch, sizeof(Batch[0]) * Count);
} }
u32 getCount() const { return Count; } u16 getCount() const { return Count; }
CompactPtrT get(u32 I) const { CompactPtrT get(u16 I) const {
DCHECK_LE(I, Count); DCHECK_LE(I, Count);
return Batch[I]; return Batch[I];
} }
static u32 getMaxCached(uptr Size) { static u16 getMaxCached(uptr Size) {
return Min(MaxNumCached, SizeClassMap::getMaxCachedHint(Size)); return Min(MaxNumCached, SizeClassMap::getMaxCachedHint(Size));
} }
TransferBatch *Next; TransferBatch *Next;
private: private:
u32 Count;
CompactPtrT Batch[MaxNumCached]; CompactPtrT Batch[MaxNumCached];
u16 Count;
}; };
void init(GlobalStats *S, SizeClassAllocator *A) { void init(GlobalStats *S, SizeClassAllocator *A) {
@ -128,9 +129,9 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
private: private:
static const uptr NumClasses = SizeClassMap::NumClasses; static const uptr NumClasses = SizeClassMap::NumClasses;
static const uptr BatchClassId = SizeClassMap::BatchClassId; static const uptr BatchClassId = SizeClassMap::BatchClassId;
struct PerClass { struct alignas(SCUDO_CACHE_LINE_SIZE) PerClass {
u32 Count; u16 Count;
u32 MaxCount; u16 MaxCount;
// Note: ClassSize is zero for the transfer batch. // Note: ClassSize is zero for the transfer batch.
uptr ClassSize; uptr ClassSize;
CompactPtrT Chunks[2 * TransferBatch::MaxNumCached]; CompactPtrT Chunks[2 * TransferBatch::MaxNumCached];
@ -180,7 +181,7 @@ private:
} }
NOINLINE void drain(PerClass *C, uptr ClassId) { NOINLINE void drain(PerClass *C, uptr ClassId) {
const u32 Count = Min(C->MaxCount / 2, C->Count); const u16 Count = Min(static_cast<u16>(C->MaxCount / 2), C->Count);
TransferBatch *B = TransferBatch *B =
createBatch(ClassId, Allocator->decompactPtr(ClassId, C->Chunks[0])); createBatch(ClassId, Allocator->decompactPtr(ClassId, C->Chunks[0]));
if (UNLIKELY(!B)) if (UNLIKELY(!B))

5
compiler-rt/lib/scudo/standalone/primary32.h
View File

@ -351,7 +351,7 @@ private:
} }
const uptr Size = getSizeByClassId(ClassId); const uptr Size = getSizeByClassId(ClassId);
const u32 MaxCount = TransferBatch::getMaxCached(Size); const u16 MaxCount = TransferBatch::getMaxCached(Size);
DCHECK_GT(MaxCount, 0U); DCHECK_GT(MaxCount, 0U);
// The maximum number of blocks we should carve in the region is dictated // The maximum number of blocks we should carve in the region is dictated
// by the maximum number of batches we want to fill, and the amount of // by the maximum number of batches we want to fill, and the amount of
@ -382,7 +382,8 @@ private:
C->createBatch(ClassId, reinterpret_cast<void *>(ShuffleArray[I])); C->createBatch(ClassId, reinterpret_cast<void *>(ShuffleArray[I]));
if (UNLIKELY(!B)) if (UNLIKELY(!B))
return nullptr; return nullptr;
const u32 N = Min(MaxCount, NumberOfBlocks - I); // `MaxCount` is u16 so the result will also fit in u16.
const u16 N = static_cast<u16>(Min<u32>(MaxCount, NumberOfBlocks - I));
B->setFromArray(&ShuffleArray[I], N); B->setFromArray(&ShuffleArray[I], N);
Sci->FreeList.push_back(B); Sci->FreeList.push_back(B);
I += N; I += N;

5
compiler-rt/lib/scudo/standalone/primary64.h
View File

@ -333,7 +333,7 @@ private:
NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId, NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId,
RegionInfo *Region) { RegionInfo *Region) {
const uptr Size = getSizeByClassId(ClassId); const uptr Size = getSizeByClassId(ClassId);
const u32 MaxCount = TransferBatch::getMaxCached(Size); const u16 MaxCount = TransferBatch::getMaxCached(Size);
const uptr RegionBeg = Region->RegionBeg; const uptr RegionBeg = Region->RegionBeg;
const uptr MappedUser = Region->MappedUser; const uptr MappedUser = Region->MappedUser;
@ -392,7 +392,8 @@ private:
CompactPtrBase, ShuffleArray[I]))); CompactPtrBase, ShuffleArray[I])));
if (UNLIKELY(!B)) if (UNLIKELY(!B))
return nullptr; return nullptr;
const u32 N = Min(MaxCount, NumberOfBlocks - I); // `MaxCount` is u16 so the result will also fit in u16.
const u16 N = static_cast<u16>(Min<u32>(MaxCount, NumberOfBlocks - I));
B->setFromArray(&ShuffleArray[I], N); B->setFromArray(&ShuffleArray[I], N);
Region->FreeList.push_back(B); Region->FreeList.push_back(B);
I += N; I += N;

4
compiler-rt/lib/scudo/standalone/release.h
View File

@ -242,7 +242,7 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList,
if (BlockSize <= PageSize && PageSize % BlockSize == 0) { if (BlockSize <= PageSize && PageSize % BlockSize == 0) {
// Each chunk affects one page only. // Each chunk affects one page only.
for (const auto &It : FreeList) { for (const auto &It : FreeList) {
for (u32 I = 0; I < It.getCount(); I++) { for (u16 I = 0; I < It.getCount(); I++) {
const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase(); const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase();
if (P >= RoundedSize) if (P >= RoundedSize)
continue; continue;
@ -256,7 +256,7 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList,
DCHECK_GE(RegionSize, BlockSize); DCHECK_GE(RegionSize, BlockSize);
const uptr LastBlockInRegion = ((RegionSize / BlockSize) - 1U) * BlockSize; const uptr LastBlockInRegion = ((RegionSize / BlockSize) - 1U) * BlockSize;
for (const auto &It : FreeList) { for (const auto &It : FreeList) {
for (u32 I = 0; I < It.getCount(); I++) { for (u16 I = 0; I < It.getCount(); I++) {
const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase(); const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase();
if (P >= RoundedSize) if (P >= RoundedSize)
continue; continue;

29
compiler-rt/lib/scudo/standalone/size_class_map.h
View File

@ -23,7 +23,7 @@ inline uptr scaledLog2(uptr Size, uptr ZeroLog, uptr LogBits) {
} }
template <typename Config> struct SizeClassMapBase { template <typename Config> struct SizeClassMapBase {
static u32 getMaxCachedHint(uptr Size) { static u16 getMaxCachedHint(uptr Size) {
DCHECK_NE(Size, 0); DCHECK_NE(Size, 0);
u32 N; u32 N;
// Force a 32-bit division if the template parameters allow for it. // Force a 32-bit division if the template parameters allow for it.
@ -31,7 +31,10 @@ template <typename Config> struct SizeClassMapBase {
N = static_cast<u32>((1UL << Config::MaxBytesCachedLog) / Size); N = static_cast<u32>((1UL << Config::MaxBytesCachedLog) / Size);
else else
N = (1U << Config::MaxBytesCachedLog) / static_cast<u32>(Size); N = (1U << Config::MaxBytesCachedLog) / static_cast<u32>(Size);
return Max(1U, Min(Config::MaxNumCachedHint, N));
// Note that Config::MaxNumCachedHint is u16 so the result is guaranteed to
// fit in u16.
return static_cast<u16>(Max(1U, Min<u32>(Config::MaxNumCachedHint, N)));
} }
}; };
@ -65,7 +68,7 @@ class FixedSizeClassMap : public SizeClassMapBase<Config> {
static const uptr M = (1UL << S) - 1; static const uptr M = (1UL << S) - 1;
public: public:
static const u32 MaxNumCachedHint = Config::MaxNumCachedHint; static const u16 MaxNumCachedHint = Config::MaxNumCachedHint;
static const uptr MaxSize = (1UL << Config::MaxSizeLog) + Config::SizeDelta; static const uptr MaxSize = (1UL << Config::MaxSizeLog) + Config::SizeDelta;
static const uptr NumClasses = static const uptr NumClasses =
@ -99,7 +102,7 @@ public:
return MidClass + 1 + scaledLog2(Size - 1, Config::MidSizeLog, S); return MidClass + 1 + scaledLog2(Size - 1, Config::MidSizeLog, S);
} }
static u32 getMaxCachedHint(uptr Size) { static u16 getMaxCachedHint(uptr Size) {
DCHECK_LE(Size, MaxSize); DCHECK_LE(Size, MaxSize);
return Base::getMaxCachedHint(Size); return Base::getMaxCachedHint(Size);
} }
@ -178,7 +181,7 @@ class TableSizeClassMap : public SizeClassMapBase<Config> {
static constexpr LSBTable LTable = {}; static constexpr LSBTable LTable = {};
public: public:
static const u32 MaxNumCachedHint = Config::MaxNumCachedHint; static const u16 MaxNumCachedHint = Config::MaxNumCachedHint;
static const uptr NumClasses = ClassesSize + 1; static const uptr NumClasses = ClassesSize + 1;
static_assert(NumClasses < 256, ""); static_assert(NumClasses < 256, "");
@ -212,7 +215,7 @@ public:
return SzTable.Tab[scaledLog2(Size - 1, Config::MidSizeLog, S)]; return SzTable.Tab[scaledLog2(Size - 1, Config::MidSizeLog, S)];
} }
static u32 getMaxCachedHint(uptr Size) { static u16 getMaxCachedHint(uptr Size) {
DCHECK_LE(Size, MaxSize); DCHECK_LE(Size, MaxSize);
return Base::getMaxCachedHint(Size); return Base::getMaxCachedHint(Size);
} }
@ -223,7 +226,7 @@ struct DefaultSizeClassConfig {
static const uptr MinSizeLog = 5; static const uptr MinSizeLog = 5;
static const uptr MidSizeLog = 8; static const uptr MidSizeLog = 8;
static const uptr MaxSizeLog = 17; static const uptr MaxSizeLog = 17;
static const u32 MaxNumCachedHint = 14; static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 10; static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = 0; static const uptr SizeDelta = 0;
}; };
@ -235,7 +238,7 @@ struct FuchsiaSizeClassConfig {
static const uptr MinSizeLog = 5; static const uptr MinSizeLog = 5;
static const uptr MidSizeLog = 8; static const uptr MidSizeLog = 8;
static const uptr MaxSizeLog = 17; static const uptr MaxSizeLog = 17;
static const u32 MaxNumCachedHint = 12; static const u16 MaxNumCachedHint = 12;
static const uptr MaxBytesCachedLog = 10; static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = Chunk::getHeaderSize(); static const uptr SizeDelta = Chunk::getHeaderSize();
}; };
@ -248,7 +251,7 @@ struct AndroidSizeClassConfig {
static const uptr MinSizeLog = 4; static const uptr MinSizeLog = 4;
static const uptr MidSizeLog = 6; static const uptr MidSizeLog = 6;
static const uptr MaxSizeLog = 16; static const uptr MaxSizeLog = 16;
static const u32 MaxNumCachedHint = 13; static const u16 MaxNumCachedHint = 13;
static const uptr MaxBytesCachedLog = 13; static const uptr MaxBytesCachedLog = 13;
static constexpr u32 Classes[] = { static constexpr u32 Classes[] = {
@ -263,7 +266,7 @@ struct AndroidSizeClassConfig {
static const uptr MinSizeLog = 4; static const uptr MinSizeLog = 4;
static const uptr MidSizeLog = 7; static const uptr MidSizeLog = 7;
static const uptr MaxSizeLog = 16; static const uptr MaxSizeLog = 16;
static const u32 MaxNumCachedHint = 14; static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 13; static const uptr MaxBytesCachedLog = 13;
static constexpr u32 Classes[] = { static constexpr u32 Classes[] = {
@ -292,7 +295,7 @@ struct SvelteSizeClassConfig {
static const uptr MinSizeLog = 4; static const uptr MinSizeLog = 4;
static const uptr MidSizeLog = 8; static const uptr MidSizeLog = 8;
static const uptr MaxSizeLog = 14; static const uptr MaxSizeLog = 14;
static const u32 MaxNumCachedHint = 13; static const u16 MaxNumCachedHint = 13;
static const uptr MaxBytesCachedLog = 10; static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = Chunk::getHeaderSize(); static const uptr SizeDelta = Chunk::getHeaderSize();
#else #else
@ -300,7 +303,7 @@ struct SvelteSizeClassConfig {
static const uptr MinSizeLog = 3; static const uptr MinSizeLog = 3;
static const uptr MidSizeLog = 7; static const uptr MidSizeLog = 7;
static const uptr MaxSizeLog = 14; static const uptr MaxSizeLog = 14;
static const u32 MaxNumCachedHint = 14; static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 10; static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = Chunk::getHeaderSize(); static const uptr SizeDelta = Chunk::getHeaderSize();
#endif #endif
@ -315,7 +318,7 @@ struct TrustySizeClassConfig {
static const uptr MinSizeLog = 7; static const uptr MinSizeLog = 7;
static const uptr MidSizeLog = 7; static const uptr MidSizeLog = 7;
static const uptr MaxSizeLog = 7; static const uptr MaxSizeLog = 7;
static const u32 MaxNumCachedHint = 8; static const u16 MaxNumCachedHint = 8;
static const uptr MaxBytesCachedLog = 10; static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = 0; static const uptr SizeDelta = 0;
}; };

2
compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
View File

@ -506,7 +506,7 @@ struct DeathSizeClassConfig {
static const scudo::uptr MinSizeLog = 10; static const scudo::uptr MinSizeLog = 10;
static const scudo::uptr MidSizeLog = 10; static const scudo::uptr MidSizeLog = 10;
static const scudo::uptr MaxSizeLog = 13; static const scudo::uptr MaxSizeLog = 13;
static const scudo::u32 MaxNumCachedHint = 4; static const scudo::u16 MaxNumCachedHint = 4;
static const scudo::uptr MaxBytesCachedLog = 12; static const scudo::uptr MaxBytesCachedLog = 12;
static const scudo::uptr SizeDelta = 0; static const scudo::uptr SizeDelta = 0;
}; };

2
compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
View File

@ -176,7 +176,7 @@ TEST(ScudoPrimaryTest, Primary64OOM) {
AllocationFailed = true; AllocationFailed = true;
break; break;
} }
for (scudo::u32 J = 0; J < B->getCount(); J++) for (scudo::u16 J = 0; J < B->getCount(); J++)
memset(Allocator.decompactPtr(ClassId, B->get(J)), 'B', Size); memset(Allocator.decompactPtr(ClassId, B->get(J)), 'B', Size);
Batches.push_back(B); Batches.push_back(B);
} }

8
compiler-rt/lib/scudo/standalone/tests/release_test.cpp
View File

@ -130,22 +130,22 @@ public:
// Simplified version of a TransferBatch. // Simplified version of a TransferBatch.
template <class SizeClassMap> struct FreeBatch { template <class SizeClassMap> struct FreeBatch {
static const scudo::u32 MaxCount = SizeClassMap::MaxNumCachedHint; static const scudo::u16 MaxCount = SizeClassMap::MaxNumCachedHint;
void clear() { Count = 0; } void clear() { Count = 0; }
void add(scudo::uptr P) { void add(scudo::uptr P) {
DCHECK_LT(Count, MaxCount); DCHECK_LT(Count, MaxCount);
Batch[Count++] = P; Batch[Count++] = P;
} }
scudo::u32 getCount() const { return Count; } scudo::u16 getCount() const { return Count; }
scudo::uptr get(scudo::u32 I) const { scudo::uptr get(scudo::u16 I) const {
DCHECK_LE(I, Count); DCHECK_LE(I, Count);
return Batch[I]; return Batch[I];
} }
FreeBatch *Next; FreeBatch *Next;
private: private:
scudo::u32 Count;
scudo::uptr Batch[MaxCount]; scudo::uptr Batch[MaxCount];
scudo::u16 Count;
}; };
template <class SizeClassMap> void testReleaseFreeMemoryToOS() { template <class SizeClassMap> void testReleaseFreeMemoryToOS() {

4
compiler-rt/lib/scudo/standalone/tests/size_class_map_test.cpp
View File

@ -33,7 +33,7 @@ struct OneClassSizeClassConfig {
static const scudo::uptr MinSizeLog = 5; static const scudo::uptr MinSizeLog = 5;
static const scudo::uptr MidSizeLog = 5; static const scudo::uptr MidSizeLog = 5;
static const scudo::uptr MaxSizeLog = 5; static const scudo::uptr MaxSizeLog = 5;
static const scudo::u32 MaxNumCachedHint = 0; static const scudo::u16 MaxNumCachedHint = 0;
static const scudo::uptr MaxBytesCachedLog = 0; static const scudo::uptr MaxBytesCachedLog = 0;
static const scudo::uptr SizeDelta = 0; static const scudo::uptr SizeDelta = 0;
}; };
@ -48,7 +48,7 @@ struct LargeMaxSizeClassConfig {
static const scudo::uptr MinSizeLog = 4; static const scudo::uptr MinSizeLog = 4;
static const scudo::uptr MidSizeLog = 8; static const scudo::uptr MidSizeLog = 8;
static const scudo::uptr MaxSizeLog = 63; static const scudo::uptr MaxSizeLog = 63;
static const scudo::u32 MaxNumCachedHint = 128; static const scudo::u16 MaxNumCachedHint = 128;
static const scudo::uptr MaxBytesCachedLog = 16; static const scudo::uptr MaxBytesCachedLog = 16;
static const scudo::uptr SizeDelta = 0; static const scudo::uptr SizeDelta = 0;
}; };

2
compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp
View File

@ -140,7 +140,7 @@ struct MySizeClassConfig {
static const uptr MinSizeLog = %zu; static const uptr MinSizeLog = %zu;
static const uptr MidSizeLog = %zu; static const uptr MidSizeLog = %zu;
static const uptr MaxSizeLog = %zu; static const uptr MaxSizeLog = %zu;
static const u32 MaxNumCachedHint = 14; static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 14; static const uptr MaxBytesCachedLog = 14;
static constexpr u32 Classes[] = {)", static constexpr u32 Classes[] = {)",