diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h index e41b949d8d0e..0a985fb67bea 100644 --- a/compiler-rt/lib/scudo/standalone/primary32.h +++ b/compiler-rt/lib/scudo/standalone/primary32.h @@ -483,12 +483,15 @@ private: } } uptr TotalReleasedBytes = 0; + auto SkipRegion = [this, First, ClassId](uptr RegionIndex) { + return (PossibleRegions[First + RegionIndex] - 1U) != ClassId; + }; if (First && Last) { const uptr Base = First * RegionSize; const uptr NumberOfRegions = Last - First + 1U; ReleaseRecorder Recorder(Base); releaseFreeMemoryToOS(Sci->FreeList, Base, RegionSize, NumberOfRegions, - BlockSize, &Recorder); + BlockSize, &Recorder, SkipRegion); if (Recorder.getReleasedRangesCount() > 0) { Sci->ReleaseInfo.PushedBlocksAtLastRelease = Sci->Stats.PushedBlocks; Sci->ReleaseInfo.RangesReleased += Recorder.getReleasedRangesCount(); diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h index ad92ae250e1f..933b1ee7c967 100644 --- a/compiler-rt/lib/scudo/standalone/primary64.h +++ b/compiler-rt/lib/scudo/standalone/primary64.h @@ -479,9 +479,11 @@ private: } } + auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; }; ReleaseRecorder Recorder(Region->RegionBeg, &Region->Data); releaseFreeMemoryToOS(Region->FreeList, Region->RegionBeg, - Region->AllocatedUser, 1U, BlockSize, &Recorder); + Region->AllocatedUser, 1U, BlockSize, &Recorder, + SkipRegion); if (Recorder.getReleasedRangesCount() > 0) { Region->ReleaseInfo.PushedBlocksAtLastRelease = diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h index 748e1c001153..cd9e66d63b36 100644 --- a/compiler-rt/lib/scudo/standalone/release.h +++ b/compiler-rt/lib/scudo/standalone/release.h @@ -156,6 +156,11 @@ public: CurrentPage++; } + void skipPages(uptr N) { + closeOpenedRange(); + CurrentPage += N; + } + void finish() { closeOpenedRange(); } private: @@ -174,11 +179,11 @@ private: uptr CurrentRangeStatePage = 0; }; -template +template NOINLINE void releaseFreeMemoryToOS(const IntrusiveList &FreeList, uptr Base, uptr RegionSize, uptr NumberOfRegions, uptr BlockSize, - ReleaseRecorderT *Recorder) { + ReleaseRecorderT *Recorder, SkipRegionT SkipRegion) { const uptr PageSize = getPageSizeCached(); // Figure out the number of chunks per page and whether we can take a fast @@ -283,10 +288,15 @@ releaseFreeMemoryToOS(const IntrusiveList &FreeList, uptr Base, FreePagesRangeTracker RangeTracker(Recorder); if (SameBlockCountPerPage) { // Fast path, every page has the same number of chunks affecting it. - for (uptr I = 0; I < NumberOfRegions; I++) + for (uptr I = 0; I < NumberOfRegions; I++) { + if (SkipRegion(I)) { + RangeTracker.skipPages(PagesCount); + continue; + } for (uptr J = 0; J < PagesCount; J++) RangeTracker.processNextPage(Counters.get(I, J) == FullPagesBlockCountMax); + } } else { // Slow path, go through the pages keeping count how many chunks affect // each page. @@ -298,6 +308,10 @@ releaseFreeMemoryToOS(const IntrusiveList &FreeList, uptr Base, // up the number of chunks on the current page and checking on every step // whether the page boundary was crossed. for (uptr I = 0; I < NumberOfRegions; I++) { + if (SkipRegion(I)) { + RangeTracker.skipPages(PagesCount); + continue; + } uptr PrevPageBoundary = 0; uptr CurrentBoundary = 0; for (uptr J = 0; J < PagesCount; J++) { diff --git a/compiler-rt/lib/scudo/standalone/tests/release_test.cpp b/compiler-rt/lib/scudo/standalone/tests/release_test.cpp index a693b97f80da..9e991a7054ed 100644 --- a/compiler-rt/lib/scudo/standalone/tests/release_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/release_test.cpp @@ -190,9 +190,10 @@ template void testReleaseFreeMemoryToOS() { } // Release the memory. + auto SkipRegion = [](UNUSED scudo::uptr RegionIndex) { return false; }; ReleasedPagesRecorder Recorder; releaseFreeMemoryToOS(FreeList, 0, MaxBlocks * BlockSize, 1U, BlockSize, - &Recorder); + &Recorder, SkipRegion); // Verify that there are no released pages touched by used chunks and all // ranges of free chunks big enough to contain the entire memory pages had