maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Use u16 to store Count/MaxCount 

The Count/MaxCount used in TransferBatch and PerClass can be fit in u16 in
current configurations and it's also reasonable to have a u16 limit. The
spare 16 bits will be used for additional status like pages mapping
status in a TransferBatch.

Reviewed By: cryptoad, cferris, vitalybuka

Differential Revision: https://reviews.llvm.org/D133145
This commit is contained in:
Chia-hung Duan 2022-10-13 22:03:06 +00:00
parent 3d8d9c9884
commit 0fb2aeef53
10 changed files with 44 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
compiler-rt/lib/scudo/standalone/local_cache.h
View File

@ -10,6 +10,7 @@
#define SCUDO_LOCAL_CACHE_H_
#include "internal_defs.h"
#include "platform.h"
#include "report.h"
#include "stats.h"
@ -20,8 +21,8 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
typedef typename SizeClassAllocator::CompactPtrT CompactPtrT;
struct TransferBatch {
static const u32 MaxNumCached = SizeClassMap::MaxNumCachedHint;
void setFromArray(CompactPtrT *Array, u32 N) {
static const u16 MaxNumCached = SizeClassMap::MaxNumCachedHint;
void setFromArray(CompactPtrT *Array, u16 N) {
DCHECK_LE(N, MaxNumCached);
Count = N;
memcpy(Batch, Array, sizeof(Batch[0]) * Count);
@ -34,19 +35,19 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
void copyToArray(CompactPtrT *Array) const {
memcpy(Array, Batch, sizeof(Batch[0]) * Count);
}
u32 getCount() const { return Count; }
CompactPtrT get(u32 I) const {
u16 getCount() const { return Count; }
CompactPtrT get(u16 I) const {
DCHECK_LE(I, Count);
return Batch[I];
}
static u32 getMaxCached(uptr Size) {
static u16 getMaxCached(uptr Size) {
return Min(MaxNumCached, SizeClassMap::getMaxCachedHint(Size));
}
TransferBatch *Next;
private:
u32 Count;
CompactPtrT Batch[MaxNumCached];
u16 Count;
};
void init(GlobalStats *S, SizeClassAllocator *A) {
@ -128,9 +129,9 @@ template <class SizeClassAllocator> struct SizeClassAllocatorLocalCache {
private:
static const uptr NumClasses = SizeClassMap::NumClasses;
static const uptr BatchClassId = SizeClassMap::BatchClassId;
struct PerClass {
u32 Count;
u32 MaxCount;
struct alignas(SCUDO_CACHE_LINE_SIZE) PerClass {
u16 Count;
u16 MaxCount;
// Note: ClassSize is zero for the transfer batch.
uptr ClassSize;
CompactPtrT Chunks[2 * TransferBatch::MaxNumCached];
@ -180,7 +181,7 @@ private:
}
NOINLINE void drain(PerClass *C, uptr ClassId) {
const u32 Count = Min(C->MaxCount / 2, C->Count);
const u16 Count = Min(static_cast<u16>(C->MaxCount / 2), C->Count);
TransferBatch *B =
createBatch(ClassId, Allocator->decompactPtr(ClassId, C->Chunks[0]));
if (UNLIKELY(!B))

5
compiler-rt/lib/scudo/standalone/primary32.h
View File

@ -351,7 +351,7 @@ private:
}
const uptr Size = getSizeByClassId(ClassId);
const u32 MaxCount = TransferBatch::getMaxCached(Size);
const u16 MaxCount = TransferBatch::getMaxCached(Size);
DCHECK_GT(MaxCount, 0U);
// The maximum number of blocks we should carve in the region is dictated
// by the maximum number of batches we want to fill, and the amount of
@ -382,7 +382,8 @@ private:
C->createBatch(ClassId, reinterpret_cast<void *>(ShuffleArray[I]));
if (UNLIKELY(!B))
return nullptr;
const u32 N = Min(MaxCount, NumberOfBlocks - I);
// `MaxCount` is u16 so the result will also fit in u16.
const u16 N = static_cast<u16>(Min<u32>(MaxCount, NumberOfBlocks - I));
B->setFromArray(&ShuffleArray[I], N);
Sci->FreeList.push_back(B);
I += N;

5
compiler-rt/lib/scudo/standalone/primary64.h
View File

@ -333,7 +333,7 @@ private:
NOINLINE TransferBatch *populateFreeList(CacheT *C, uptr ClassId,
RegionInfo *Region) {
const uptr Size = getSizeByClassId(ClassId);
const u32 MaxCount = TransferBatch::getMaxCached(Size);
const u16 MaxCount = TransferBatch::getMaxCached(Size);
const uptr RegionBeg = Region->RegionBeg;
const uptr MappedUser = Region->MappedUser;
@ -392,7 +392,8 @@ private:
CompactPtrBase, ShuffleArray[I])));
if (UNLIKELY(!B))
return nullptr;
const u32 N = Min(MaxCount, NumberOfBlocks - I);
// `MaxCount` is u16 so the result will also fit in u16.
const u16 N = static_cast<u16>(Min<u32>(MaxCount, NumberOfBlocks - I));
B->setFromArray(&ShuffleArray[I], N);
Region->FreeList.push_back(B);
I += N;

4
compiler-rt/lib/scudo/standalone/release.h
View File

@ -242,7 +242,7 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList,
if (BlockSize <= PageSize && PageSize % BlockSize == 0) {
// Each chunk affects one page only.
for (const auto &It : FreeList) {
for (u32 I = 0; I < It.getCount(); I++) {
for (u16 I = 0; I < It.getCount(); I++) {
const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase();
if (P >= RoundedSize)
continue;
@ -256,7 +256,7 @@ releaseFreeMemoryToOS(const IntrusiveList<TransferBatchT> &FreeList,
DCHECK_GE(RegionSize, BlockSize);
const uptr LastBlockInRegion = ((RegionSize / BlockSize) - 1U) * BlockSize;
for (const auto &It : FreeList) {
for (u32 I = 0; I < It.getCount(); I++) {
for (u16 I = 0; I < It.getCount(); I++) {
const uptr P = DecompactPtr(It.get(I)) - Recorder->getBase();
if (P >= RoundedSize)
continue;

29
compiler-rt/lib/scudo/standalone/size_class_map.h
View File

@ -23,7 +23,7 @@ inline uptr scaledLog2(uptr Size, uptr ZeroLog, uptr LogBits) {
}
template <typename Config> struct SizeClassMapBase {
static u32 getMaxCachedHint(uptr Size) {
static u16 getMaxCachedHint(uptr Size) {
DCHECK_NE(Size, 0);
u32 N;
// Force a 32-bit division if the template parameters allow for it.
@ -31,7 +31,10 @@ template <typename Config> struct SizeClassMapBase {
N = static_cast<u32>((1UL << Config::MaxBytesCachedLog) / Size);
else
N = (1U << Config::MaxBytesCachedLog) / static_cast<u32>(Size);
return Max(1U, Min(Config::MaxNumCachedHint, N));
// Note that Config::MaxNumCachedHint is u16 so the result is guaranteed to
// fit in u16.
return static_cast<u16>(Max(1U, Min<u32>(Config::MaxNumCachedHint, N)));
}
};
@ -65,7 +68,7 @@ class FixedSizeClassMap : public SizeClassMapBase<Config> {
static const uptr M = (1UL << S) - 1;
public:
static const u32 MaxNumCachedHint = Config::MaxNumCachedHint;
static const u16 MaxNumCachedHint = Config::MaxNumCachedHint;
static const uptr MaxSize = (1UL << Config::MaxSizeLog) + Config::SizeDelta;
static const uptr NumClasses =
@ -99,7 +102,7 @@ public:
return MidClass + 1 + scaledLog2(Size - 1, Config::MidSizeLog, S);
}
static u32 getMaxCachedHint(uptr Size) {
static u16 getMaxCachedHint(uptr Size) {
DCHECK_LE(Size, MaxSize);
return Base::getMaxCachedHint(Size);
}
@ -178,7 +181,7 @@ class TableSizeClassMap : public SizeClassMapBase<Config> {
static constexpr LSBTable LTable = {};
public:
static const u32 MaxNumCachedHint = Config::MaxNumCachedHint;
static const u16 MaxNumCachedHint = Config::MaxNumCachedHint;
static const uptr NumClasses = ClassesSize + 1;
static_assert(NumClasses < 256, "");
@ -212,7 +215,7 @@ public:
return SzTable.Tab[scaledLog2(Size - 1, Config::MidSizeLog, S)];
}
static u32 getMaxCachedHint(uptr Size) {
static u16 getMaxCachedHint(uptr Size) {
DCHECK_LE(Size, MaxSize);
return Base::getMaxCachedHint(Size);
}
@ -223,7 +226,7 @@ struct DefaultSizeClassConfig {
static const uptr MinSizeLog = 5;
static const uptr MidSizeLog = 8;
static const uptr MaxSizeLog = 17;
static const u32 MaxNumCachedHint = 14;
static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = 0;
};
@ -235,7 +238,7 @@ struct FuchsiaSizeClassConfig {
static const uptr MinSizeLog = 5;
static const uptr MidSizeLog = 8;
static const uptr MaxSizeLog = 17;
static const u32 MaxNumCachedHint = 12;
static const u16 MaxNumCachedHint = 12;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = Chunk::getHeaderSize();
};
@ -248,7 +251,7 @@ struct AndroidSizeClassConfig {
static const uptr MinSizeLog = 4;
static const uptr MidSizeLog = 6;
static const uptr MaxSizeLog = 16;
static const u32 MaxNumCachedHint = 13;
static const u16 MaxNumCachedHint = 13;
static const uptr MaxBytesCachedLog = 13;
static constexpr u32 Classes[] = {
@ -263,7 +266,7 @@ struct AndroidSizeClassConfig {
static const uptr MinSizeLog = 4;
static const uptr MidSizeLog = 7;
static const uptr MaxSizeLog = 16;
static const u32 MaxNumCachedHint = 14;
static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 13;
static constexpr u32 Classes[] = {
@ -292,7 +295,7 @@ struct SvelteSizeClassConfig {
static const uptr MinSizeLog = 4;
static const uptr MidSizeLog = 8;
static const uptr MaxSizeLog = 14;
static const u32 MaxNumCachedHint = 13;
static const u16 MaxNumCachedHint = 13;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = Chunk::getHeaderSize();
#else
@ -300,7 +303,7 @@ struct SvelteSizeClassConfig {
static const uptr MinSizeLog = 3;
static const uptr MidSizeLog = 7;
static const uptr MaxSizeLog = 14;
static const u32 MaxNumCachedHint = 14;
static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = Chunk::getHeaderSize();
#endif
@ -315,7 +318,7 @@ struct TrustySizeClassConfig {
static const uptr MinSizeLog = 7;
static const uptr MidSizeLog = 7;
static const uptr MaxSizeLog = 7;
static const u32 MaxNumCachedHint = 8;
static const u16 MaxNumCachedHint = 8;
static const uptr MaxBytesCachedLog = 10;
static const uptr SizeDelta = 0;
};

2
compiler-rt/lib/scudo/standalone/tests/combined_test.cpp
View File

@ -506,7 +506,7 @@ struct DeathSizeClassConfig {
static const scudo::uptr MinSizeLog = 10;
static const scudo::uptr MidSizeLog = 10;
static const scudo::uptr MaxSizeLog = 13;
static const scudo::u32 MaxNumCachedHint = 4;
static const scudo::u16 MaxNumCachedHint = 4;
static const scudo::uptr MaxBytesCachedLog = 12;
static const scudo::uptr SizeDelta = 0;
};

2
compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
View File

@ -176,7 +176,7 @@ TEST(ScudoPrimaryTest, Primary64OOM) {
AllocationFailed = true;
break;
}
for (scudo::u32 J = 0; J < B->getCount(); J++)
for (scudo::u16 J = 0; J < B->getCount(); J++)
memset(Allocator.decompactPtr(ClassId, B->get(J)), 'B', Size);
Batches.push_back(B);
}

8
compiler-rt/lib/scudo/standalone/tests/release_test.cpp
View File

@ -130,22 +130,22 @@ public:
// Simplified version of a TransferBatch.
template <class SizeClassMap> struct FreeBatch {
static const scudo::u32 MaxCount = SizeClassMap::MaxNumCachedHint;
static const scudo::u16 MaxCount = SizeClassMap::MaxNumCachedHint;
void clear() { Count = 0; }
void add(scudo::uptr P) {
DCHECK_LT(Count, MaxCount);
Batch[Count++] = P;
}
scudo::u32 getCount() const { return Count; }
scudo::uptr get(scudo::u32 I) const {
scudo::u16 getCount() const { return Count; }
scudo::uptr get(scudo::u16 I) const {
DCHECK_LE(I, Count);
return Batch[I];
}
FreeBatch *Next;
private:
scudo::u32 Count;
scudo::uptr Batch[MaxCount];
scudo::u16 Count;
};
template <class SizeClassMap> void testReleaseFreeMemoryToOS() {

4
compiler-rt/lib/scudo/standalone/tests/size_class_map_test.cpp
View File

@ -33,7 +33,7 @@ struct OneClassSizeClassConfig {
static const scudo::uptr MinSizeLog = 5;
static const scudo::uptr MidSizeLog = 5;
static const scudo::uptr MaxSizeLog = 5;
static const scudo::u32 MaxNumCachedHint = 0;
static const scudo::u16 MaxNumCachedHint = 0;
static const scudo::uptr MaxBytesCachedLog = 0;
static const scudo::uptr SizeDelta = 0;
};
@ -48,7 +48,7 @@ struct LargeMaxSizeClassConfig {
static const scudo::uptr MinSizeLog = 4;
static const scudo::uptr MidSizeLog = 8;
static const scudo::uptr MaxSizeLog = 63;
static const scudo::u32 MaxNumCachedHint = 128;
static const scudo::u16 MaxNumCachedHint = 128;
static const scudo::uptr MaxBytesCachedLog = 16;
static const scudo::uptr SizeDelta = 0;
};

2
compiler-rt/lib/scudo/standalone/tools/compute_size_class_config.cpp
View File

@ -140,7 +140,7 @@ struct MySizeClassConfig {
static const uptr MinSizeLog = %zu;
static const uptr MidSizeLog = %zu;
static const uptr MaxSizeLog = %zu;
static const u32 MaxNumCachedHint = 14;
static const u16 MaxNumCachedHint = 14;
static const uptr MaxBytesCachedLog = 14;
static constexpr u32 Classes[] = {)",