diff --git a/tcmalloc/common.h b/tcmalloc/common.h index 4e7165c73..f7fae9bf5 100644 --- a/tcmalloc/common.h +++ b/tcmalloc/common.h @@ -35,6 +35,7 @@ #include "tcmalloc/internal/logging.h" #include "tcmalloc/internal/memory_tag.h" #include "tcmalloc/internal/optimization.h" +#include "tcmalloc/internal/range_tracker.h" #include "tcmalloc/malloc_extension.h" GOOGLE_MALLOC_SECTION_BEGIN @@ -208,6 +209,7 @@ inline constexpr size_t kMinObjectsToMove = 2; inline constexpr size_t kMaxObjectsToMove = 128; inline constexpr size_t kPageSize = 1 << kPageShift; +using PageBitmap = Bitmap; inline constexpr std::align_val_t kAlignment{8}; // log2 (kAlignment) diff --git a/tcmalloc/huge_page_filler.h b/tcmalloc/huge_page_filler.h index 853705e98..c823eaa20 100644 --- a/tcmalloc/huge_page_filler.h +++ b/tcmalloc/huge_page_filler.h @@ -72,6 +72,47 @@ enum class HugePageTreatmentType : uint8_t { kCollapse = 1 << 1, }; +// Reduction operations for Scale(). In the case of contracting bitmaps, +// we need to reduce multiple (n) bits to a single bit. +// +// - kAny: If any of the n bits are set, the resulting bit is set. +// - kAll: If all of the n bits are set, the resulting bit is set. +enum class ReductionOp { + kAny, + kAll, +}; + +// Scales `src` bitmap from `src_len` bits to `M` bits. If it's a contraction, +// applies `op` to reduce multiple bits to one. If it's an expansion (or +// identity), `op` is a no-op. +template +Bitmap Scale(const Bitmap& src, size_t src_len, ReductionOp op) { + TC_ASSERT_LE(src_len, N); + TC_ASSERT(src_len % M == 0 || M % src_len == 0); + + Bitmap res; + const size_t src_per_dst = std::max(1, src_len / M); + const size_t dst_per_src = std::max(1, M / src_len); + + const size_t required_count = (op == ReductionOp::kAll) ? src_per_dst : 1; + + size_t dst_idx = 0; + for (size_t src_idx = 0; src_idx < src_len; src_idx += src_per_dst) { + size_t count = 0; + for (size_t j = 0; j < src_per_dst; ++j) { + if (src.GetBit(src_idx + j)) { + count++; + if (count == required_count) { + res.SetRange(dst_idx, dst_per_src); + break; + } + } + } + dst_idx += dst_per_src; + } + return res; +} + // PageTracker keeps track of the allocation status of every page in a HugePage. // It allows allocation and deallocation of a contiguous run of pages. // @@ -218,8 +259,7 @@ class PageTracker : public TList::Elem { Length ReleaseFree(MemoryModifyFunction& unback) ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock); - Length MarkSubreleased(Bitmap unbacked, - int native_pages_per_tcmalloc_page) + Length MarkSubreleased(PageBitmap unbacked) ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock); Bitmap released_by_page() const { @@ -248,9 +288,15 @@ class PageTracker : public TList::Elem { // This records the trackers that are currently being collapsed. This is // used to avoid subreleasing the pages that are being collapsed. bool being_collapsed = false; - // Records swap and unbacked bitmaps for this hugepage. - Residency::SinglePageBitmaps bitmaps; - ResidencyBitmap stale; + // Records the unbacked bitmap for this hugepage. In terms of TCMalloc + // pages. scaled via `ReductionOp::kAll`. + PageBitmap unbacked; + // Records the swapped bitmap for this hugepage. In terms of TCMalloc + // pages. scaled via `ReductionOp::kAny`. + PageBitmap swapped; + // Records the stale bitmap for this hugepage. In terms of TCMalloc + // pages. scaled via `ReductionOp::kAny`. + PageBitmap stale; // Records whether collapse was skipped due to threshold constraints. bool collapse_skipped = false; // Records whether collapse was skipped due to backoff. @@ -326,8 +372,9 @@ class PageTracker : public TList::Elem { size_t n_used_stale; }; - HardwarePageResidencyInfo CountInfoInHugePage( - Residency::SinglePageBitmaps bitmaps, ResidencyBitmap stale) const; + HardwarePageResidencyInfo CountInfoInHugePage(PageBitmap unbacked, + PageBitmap swapped, + PageBitmap stale) const; private: HugePage location_; @@ -702,16 +749,18 @@ class UsageInfo { } else { records.num_free_non_hugepage_backed += (free - pt.released_pages()); records.num_used_non_hugepage_backed += pt.used_pages(); - Residency::SinglePageBitmaps bitmaps = hugepage_residency_state.bitmaps; - ++records.unbacked_histo[HardwarePageBucketNum( - bitmaps.unbacked.CountBits())]; - ++records.swapped_histo[HardwarePageBucketNum( - bitmaps.swapped.CountBits())]; - ++records.stale_histo[HardwarePageBucketNum( - hugepage_residency_state.stale.CountBits())]; - - PageTracker::HardwarePageResidencyInfo info = - pt.CountInfoInHugePage(bitmaps, hugepage_residency_state.stale); + PageTracker::HardwarePageResidencyInfo info = pt.CountInfoInHugePage( + hugepage_residency_state.unbacked, hugepage_residency_state.swapped, + hugepage_residency_state.stale); + + auto unbacked_bits = info.n_used_unbacked + info.n_free_unbacked; + auto swapped_bits = info.n_used_swapped + info.n_free_swapped; + auto stale_bits = info.n_used_stale + info.n_free_stale; + + ++records.unbacked_histo[HardwarePageBucketNum(unbacked_bits)]; + ++records.swapped_histo[HardwarePageBucketNum(swapped_bits)]; + ++records.stale_histo[HardwarePageBucketNum(stale_bits)]; + ++records .free_unbacked_histo[HardwarePageBucketNum(info.n_free_unbacked)]; ++records @@ -1311,8 +1360,7 @@ class HugePageFiller { // Utility function to handle a non-hugepage backed `page_tracker` and // mark its unmapped pages appropriately. - Length HandleUnbackedHugePage(PageTracker* page_tracker, - Bitmap unbacked) + Length HandleUnbackedHugePage(PageTracker* page_tracker, PageBitmap unbacked) ABSL_EXCLUSIVE_LOCKS_REQUIRED(pageheap_lock); private: @@ -1477,7 +1525,6 @@ class HugePageFiller { int current_backoff_delay_ ABSL_GUARDED_BY(pageheap_lock) = 0; uintptr_t rng_ = 0; SubreleaseUnbackedMode subrelease_unbacked_mode_; - const int native_pages_per_tcmalloc_page_; }; inline typename PageTracker::PageAllocation PageTracker::Get( @@ -1511,7 +1558,7 @@ inline void PageTracker::SetAnonVmaName(MemoryTagFunction& set_anon_vma_name, } inline PageTracker::HardwarePageResidencyInfo PageTracker::CountInfoInHugePage( - Residency::SinglePageBitmaps bitmaps, ResidencyBitmap stale) const { + PageBitmap unbacked, PageBitmap swapped, PageBitmap stale) const { // TODO(b/424551232): Add support for the scenario when native page size is // larger than TCMalloc page size. const size_t kHardwarePagesInHugePage = kHugePageSize / GetPageSize(); @@ -1520,9 +1567,7 @@ inline PageTracker::HardwarePageResidencyInfo PageTracker::CountInfoInHugePage( } TC_ASSERT_LE(kHardwarePagesInHugePage, kMaxResidencyBits); - Bitmap unbacked = bitmaps.unbacked; - Bitmap swapped = bitmaps.swapped; - Bitmap free = free_.bits(); + const Bitmap free = free_.bits(); TC_ASSERT_EQ(kHardwarePagesInHugePage % kPagesPerHugePage.raw_num(), 0); const int shift = kHardwarePagesInHugePage / kPagesPerHugePage.raw_num(); @@ -1534,14 +1579,12 @@ inline PageTracker::HardwarePageResidencyInfo PageTracker::CountInfoInHugePage( size_t n_swapped[2] = {0, 0}; size_t n_stale[2] = {0, 0}; - for (size_t i = 0; i < kHardwarePagesInHugePage; ++i) { - const int scaled_idx = i >> shift_bits; - bool is_free = !free.GetBit(scaled_idx); - - n_swapped[is_free] += swapped.GetBit(i); - n_unbacked[is_free] += unbacked.GetBit(i); - n_stale[is_free] += stale.GetBit(i); - } + n_unbacked[0] = (free & unbacked).CountBits() * shift; + n_unbacked[1] = (~free & unbacked).CountBits() * shift; + n_swapped[0] = (free & swapped).CountBits() * shift; + n_swapped[1] = (~free & swapped).CountBits() * shift; + n_stale[0] = (free & stale).CountBits() * shift; + n_stale[1] = (~free & stale).CountBits() * shift; return {.n_free_swapped = n_swapped[1], .n_used_swapped = n_swapped[0], @@ -1609,60 +1652,26 @@ inline Length PageTracker::ReleaseFree(MemoryModifyFunction& unback) { return Length(count); } -inline Length PageTracker::MarkSubreleased(Bitmap unbacked, - int native_pages_per_tcmalloc_page) { - // TODO(b/525422238): pass the bitmap in TCMalloc pages directly to avoid this - // conversion. +inline Length PageTracker::MarkSubreleased(PageBitmap unbacked) { + PageBitmap free = free_.bits(); - // When TCMalloc page size is smaller than the native page size, mark the page - // as broken. We should additionally mark unbacked pages as subreleased. The - // later case isn't done currently. - if (native_pages_per_tcmalloc_page == 0) { - unbroken_ = false; - return Length(0); - } - TC_ASSERT_GT(native_pages_per_tcmalloc_page, 0); - Bitmap free = free_.bits(); + // TODO(b/525422238): The residency bitmap was captured outside of the + // lock. So, in a rare case, it's possible that the page was allocated, + // backed and then freed. So, the free page here is actually backed. + // While we currently ignore this case (resulting in underestimating + // RSS), we can potentially fix this by re-investigating the bitmaps + // and marking the pages back to backed to eventually fix this. + auto to_release = (~free) & (~released_by_page_) & unbacked; + released_by_page_ = released_by_page_ | to_release; - size_t count = 0; - for (size_t i = 0; i < kPagesPerHugePage.raw_num(); ++i) { - if (!free.GetBit(i) && !released_by_page_.GetBit(i)) { - // Check if all native pages corresponding to this TCMalloc page are - // unbacked. - bool is_unbacked = true; - for (int j = 0; j < native_pages_per_tcmalloc_page; ++j) { - const int bitmap_index = i * native_pages_per_tcmalloc_page + j; - TC_ASSERT_LT(bitmap_index, kMaxResidencyBits); - if (!unbacked.GetBit(bitmap_index)) { - is_unbacked = false; - break; - } - } - // If all native pages corresponding to this TCMalloc page are unbacked, - // and as the TCMalloc page was freed but not released, mark this page - // as released now. - // - // TODO(b/525422238): The residency bitmap was captured outside of the - // lock. So, in a rare case, it's possible that the page was allocated, - // backed and then freed. So, the free page here is actually backed. - // While we currently ignore this case (resulting in underestimating - // RSS), we can potentially fix this by re-investigating the bitmaps - // and marking the pages back to backed to eventually fix this. - if (is_unbacked) { - released_by_page_.SetBit(i); - ++count; - } - } - } - - released_count_ += count; + released_count_ += to_release.CountBits(); // Mark this is unbroken regardless of whether it had any unbacked free // TCMalloc pages. Marking this will move this tracker to one of the // released lists. unbroken_ = false; TC_ASSERT_LE(Length(released_count_), kPagesPerHugePage); TC_ASSERT_EQ(released_by_page_.CountBits(), released_count_); - return Length(count); + return Length(to_release.CountBits()); } inline MemoryModifyStatus PageTracker::Collapse( @@ -1763,9 +1772,7 @@ inline HugePageFiller::HugePageFiller( unback_without_lock_(unback_without_lock), collapse_(collapse), set_anon_vma_name_(set_anon_vma_name), - subrelease_unbacked_mode_(subrelease_unbacked_mode), - native_pages_per_tcmalloc_page_(kHugePageSize / GetPageSize() / - kPagesPerHugePage.raw_num()) { + subrelease_unbacked_mode_(subrelease_unbacked_mode) { lifetime_bucket_bounds_[0] = 0; lifetime_bucket_bounds_[1] = 1; for (int i = 2; i <= kLifetimeBuckets; ++i) { @@ -2698,6 +2705,7 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment { // kMaxSwappedPagesForCollapse respectively. const double max_collapse_cycles = absl::ToDoubleSeconds(kMaxCollapseLatencyThreshold) * clock_.freq(); + const size_t pages_per_huge_page = kHugePageSize / GetPageSize(); for (int i = 0; i < num_valid_trackers_; ++i) { PageTracker::HugePageResidencyState state; PageTracker* tracker = selected_trackers_[i]; @@ -2713,22 +2721,27 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment { // information. state.maybe_hugepage_backed = is_hugepage; if (!is_hugepage) { - state.bitmaps = + auto bitmaps = res->GetUnbackedAndSwappedBitmaps(tracker->location().start_addr()); + state.unbacked = Scale( + bitmaps.unbacked, pages_per_huge_page, ReductionOp::kAll); + state.swapped = Scale( + bitmaps.swapped, pages_per_huge_page, ReductionOp::kAny); if (pf) { - pf->GetSinglePageBitmaps(tracker->location().start_addr(), - state.stale); + ResidencyBitmap b; + pf->GetSinglePageBitmaps(tracker->location().start_addr(), b); + state.stale = Scale( + b, pages_per_huge_page, ReductionOp::kAny); } const bool backoff = treatment_stats_.collapse_time_max_cycles > max_collapse_cycles; if (enable_collapse_ == EnableCollapse::kEnabled && !backoff) { - bool should_collapse = enable_unfiltered_collapse_ == - EnableUnfilteredCollapse::kEnabled || - (state.bitmaps.swapped.CountBits() < - kMaxSwappedPagesForCollapse && - state.bitmaps.unbacked.CountBits() < - kMaxUnbackedPagesForCollapse); + bool should_collapse = + enable_unfiltered_collapse_ == + EnableUnfilteredCollapse::kEnabled || + (bitmaps.swapped.CountBits() < kMaxSwappedPagesForCollapse && + bitmaps.unbacked.CountBits() < kMaxUnbackedPagesForCollapse); if (should_collapse) { state.maybe_hugepage_backed = TryUserspaceCollapse(tracker); } else { @@ -2765,7 +2778,7 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment { // released the pageheap lock. Check that the longest free range is less // than kPagesPerHugePage to make sure it's valid to release from that // tracker. - if (!residency_states_[i].tracker_state.bitmaps.swapped.IsZero()) { + if (!residency_states_[i].tracker_state.swapped.IsZero()) { // TODO: b/425749361 - Clear swapped bit for pages that were freed. Length released_length = page_filler_.HandleReleaseFree(tracker); if (released_length > Length(0)) { @@ -2776,7 +2789,7 @@ class HugePageUnbackedTrackerTreatment final : public HugePageTreatment { if (subrelease_unbacked_mode_ == SubreleaseUnbackedMode::kEnabled) { Length released_length = page_filler_.HandleUnbackedHugePage( - tracker, residency_states_[i].tracker_state.bitmaps.unbacked); + tracker, residency_states_[i].tracker_state.unbacked); if (released_length > Length(0)) { treatment_stats_.treated_pages_unbacked_subreleased += released_length.raw_num(); @@ -2991,10 +3004,9 @@ inline void HugePageFiller::OnCollapseSuccess(TrackerType* pt) { template inline Length HugePageFiller::HandleUnbackedHugePage( - PageTracker* tracker, Bitmap unbacked) { + PageTracker* tracker, PageBitmap unbacked) { RemoveFromFillerList(tracker); - Length unmapped_length = - tracker->MarkSubreleased(unbacked, native_pages_per_tcmalloc_page_); + Length unmapped_length = tracker->MarkSubreleased(unbacked); subrelease_stats_.total_pages_subreleased += unmapped_length; unmapped_ += unmapped_length; unmapping_unaccounted_ += unmapped_length; diff --git a/tcmalloc/huge_page_filler_test.cc b/tcmalloc/huge_page_filler_test.cc index 071155ae5..d778a3b64 100644 --- a/tcmalloc/huge_page_filler_test.cc +++ b/tcmalloc/huge_page_filler_test.cc @@ -5325,7 +5325,7 @@ HugePageFiller: 130 of sparsely-accessed released used native pages are unbacked EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_free_unbacked: 126")); EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_used_unbacked: 130")); EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_free_stale: 0")); - EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_used_stale: 1")); + EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_used_stale: 2")); EXPECT_THAT(buffer_pbtxt, testing::HasSubstr("num_pages_treated: 1")); DeleteVector(p1); DeleteVector(p2); @@ -6430,14 +6430,16 @@ TEST_F(FillerTest, StaleHistograms) { }); EXPECT_THAT(buffer, - testing::HasSubstr("HugePageFiller: # of sparsely-accessed " - "regular hps with a <= # of stale < b\n" - "HugePageFiller: < 0<= 0 < 1<= 3")); + testing::HasSubstr( + "HugePageFiller: # of sparsely-accessed " + "regular hps with a <= # of stale < b\n" + "HugePageFiller: < 0<= 0 < 1<= 0 < 2<= 3")); - EXPECT_THAT(buffer, testing::HasSubstr( - "HugePageFiller: # of sparsely-accessed regular hps " - "with a <= # of free AND stale < b\n" - "HugePageFiller: < 0<= 2 < 1<= 1")); + EXPECT_THAT(buffer, + testing::HasSubstr( + "HugePageFiller: # of sparsely-accessed regular hps " + "with a <= # of free AND stale < b\n" + "HugePageFiller: < 0<= 2 < 1<= 0 < 2<= 1")); Delete(b); Delete(c); @@ -6822,6 +6824,95 @@ TEST(SkipSubreleaseIntervalsTest, EmptyIsNotEnabled) { EXPECT_FALSE(SkipSubreleaseIntervals{}.SkipSubreleaseEnabled()); } +TEST(BitmapScaleTest, ScaleAssertionFailures) { +#ifdef NDEBUG + GTEST_SKIP() << "Requires debug mode"; +#endif + Bitmap<64> map1; + // 1. src_len (64) doesn't divide M (10) and vice versa + EXPECT_DEATH(Scale<10>(map1, 64, ReductionOp::kAll), ""); + // 2. src_len > N (64) + EXPECT_DEATH(Scale<256>(map1, 128, ReductionOp::kAll), ""); +} + +TEST(BitmapScaleTest, Scale) { + // Test case 1: N == M (identity) + { + Bitmap<64> map1; + map1.SetBit(3); + map1.SetBit(17); + map1.SetBit(63); + + auto map_all = Scale<64>(map1, 64, ReductionOp::kAll); + auto map_any = Scale<64>(map1, 64, ReductionOp::kAny); + for (size_t i = 0; i < 64; ++i) { + EXPECT_EQ(map1.GetBit(i), map_all.GetBit(i)); + EXPECT_EQ(map1.GetBit(i), map_any.GetBit(i)); + } + } + + // Test case 2: N > M (contracting) + { + Bitmap<64> map1; + // We group by 2 bits. + // Group 0: bits 0, 1 -> both must be set + map1.SetBit(0); + map1.SetBit(1); + + // Group 1: bits 2, 3 -> only one set + // Should not be set for kAll, should be set for kAny + map1.SetBit(2); + + // Group 7: bits 14, 15 -> both set + map1.SetBit(14); + map1.SetBit(15); + + auto map_all = Scale<8>(map1, 16, ReductionOp::kAll); + auto map_any = Scale<8>(map1, 16, ReductionOp::kAny); + + EXPECT_TRUE(map_all.GetBit(0)); + EXPECT_TRUE(map_any.GetBit(0)); + + EXPECT_FALSE(map_all.GetBit(1)); + EXPECT_TRUE(map_any.GetBit(1)); + + for (size_t i = 2; i <= 6; ++i) { + EXPECT_FALSE(map_all.GetBit(i)); + EXPECT_FALSE(map_any.GetBit(i)); + } + + EXPECT_TRUE(map_all.GetBit(7)); + EXPECT_TRUE(map_any.GetBit(7)); + } + + // Test case 3: N < M (expanding) + { + Bitmap<64> map1; + // We expand by 2 bits. + map1.SetBit(0); + map1.SetBit(7); + + auto map_all = Scale<16>(map1, 8, ReductionOp::kAll); + auto map_any = Scale<16>(map1, 8, ReductionOp::kAny); + + // Both operations behave identically for expanding. + EXPECT_TRUE(map_all.GetBit(0)); + EXPECT_TRUE(map_all.GetBit(1)); + EXPECT_TRUE(map_any.GetBit(0)); + EXPECT_TRUE(map_any.GetBit(1)); + + EXPECT_FALSE(map_all.GetBit(2)); + EXPECT_FALSE(map_all.GetBit(3)); + EXPECT_FALSE(map_any.GetBit(2)); + EXPECT_FALSE(map_any.GetBit(3)); + + EXPECT_TRUE(map_all.GetBit(14)); + EXPECT_TRUE(map_all.GetBit(15)); + EXPECT_TRUE(map_any.GetBit(14)); + EXPECT_TRUE(map_any.GetBit(15)); + } +} + } // namespace } // namespace tcmalloc_internal } // namespace tcmalloc