diff --git a/src/runtime/mgc.go b/src/runtime/mgc.go index 56ef1c4e384..6f8463c2536 100644 --- a/src/runtime/mgc.go +++ b/src/runtime/mgc.go @@ -1460,7 +1460,7 @@ func gcSweep(mode gcMode) { mheap_.pagesSwept.Store(0) mheap_.sweepArenas = mheap_.allArenas mheap_.reclaimIndex.Store(0) - mheap_.reclaimCredit = 0 + mheap_.reclaimCredit.Store(0) unlock(&mheap_.lock) sweep.centralIndex.clear() diff --git a/src/runtime/mgcsweep.go b/src/runtime/mgcsweep.go index aedd6c316e2..9c7f9d340d2 100644 --- a/src/runtime/mgcsweep.go +++ b/src/runtime/mgcsweep.go @@ -291,7 +291,7 @@ func sweepone() uintptr { // Whole span was freed. Count it toward the // page reclaimer credit since these pages can // now be used for span allocation. - atomic.Xadduintptr(&mheap_.reclaimCredit, npages) + mheap_.reclaimCredit.Add(npages) } else { // Span is still in-use, so this returned no // pages to the heap and the span needs to diff --git a/src/runtime/mheap.go b/src/runtime/mheap.go index fc86023f4db..0e7694aab77 100644 --- a/src/runtime/mheap.go +++ b/src/runtime/mheap.go @@ -124,13 +124,12 @@ type mheap struct { // If this is >= 1<<63, the page reclaimer is done scanning // the page marks. reclaimIndex atomic.Uint64 + // reclaimCredit is spare credit for extra pages swept. Since // the page reclaimer works in large chunks, it may reclaim // more than requested. Any spare pages released go to this // credit pool. - // - // This is accessed atomically. - reclaimCredit uintptr + reclaimCredit atomic.Uintptr // arenas is the heap arena map. It points to the metadata for // the heap for every arena frame of the entire usable virtual @@ -754,13 +753,13 @@ func (h *mheap) reclaim(npage uintptr) { locked := false for npage > 0 { // Pull from accumulated credit first. - if credit := atomic.Loaduintptr(&h.reclaimCredit); credit > 0 { + if credit := h.reclaimCredit.Load(); credit > 0 { take := credit if take > npage { // Take only what we need. take = npage } - if atomic.Casuintptr(&h.reclaimCredit, credit, credit-take) { + if h.reclaimCredit.CompareAndSwap(credit, credit-take) { npage -= take } continue @@ -786,7 +785,7 @@ func (h *mheap) reclaim(npage uintptr) { npage -= nfound } else { // Put spare pages toward global credit. - atomic.Xadduintptr(&h.reclaimCredit, nfound-npage) + h.reclaimCredit.Add(nfound - npage) npage = 0 } }