runtime: make the memory limit heap goal headroom proportional

Currently if GOGC=off and GOMEMLIMIT is set, then the synchronous
scavenger is likely to work fairly often to maintain the limit, since
the heap goal goes right up to the edge of the memory limit (minus a
fixed 1 MiB of headroom).

If the application's allocation rate is high, and page-level
fragmentation is high, then most allocations will scavenge.

This change mitigates this problem by adding a proportional component
to constant headroom added to the memory-limit-based heap goal. This
means the runtime will have much more headroom before fragmentation
forces memory to be eagerly scavenged.

The proportional headroom in this case is 3%, or ~30 MiB for a 1 GiB
heap. This technically will increase GC frequency in the GOGC=off case
by a tiny amount, but will likely have a positive impact on both
allocation throughput and latency that outweighs this difference.

I wrote a small program to reproduce this issue and confirmed that the
issue is resolved by this patch:

https://github.com/golang/go/issues/57069#issuecomment-1551746565

This value of 3% is chosen as it seems to be a inflection point in this
particular small program. 2% still resulted in quite a bit of eager
scavenging work. I confirmed this results in a GC frequency increase of
about 3%.

This choice is still somewhat arbitrary because the program is
arbitrary, so perhaps worth revisiting in the future. Still, it should
help a good number of programs.

Fixes #57069.

Change-Id: Icb9829db0dfefb4fe42a0cabc5aa8d35970dd7d5
Reviewed-on: https://go-review.googlesource.com/c/go/+/460375
Reviewed-by: Michael Pratt <mpratt@google.com>
Auto-Submit: Michael Knyszek <mknyszek@google.com>
Run-TryBot: Michael Knyszek <mknyszek@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>

src/runtime/export_test.go

@@ -1345,10 +1345,11 @@ func GCTestPointerClass(p unsafe.Pointer) string {
 const Raceenabled = raceenabled
 
 const (
-	GCBackgroundUtilization     = gcBackgroundUtilization
+	GCBackgroundUtilization            = gcBackgroundUtilization
-	GCGoalUtilization           = gcGoalUtilization
+	GCGoalUtilization                  = gcGoalUtilization
-	DefaultHeapMinimum          = defaultHeapMinimum
+	DefaultHeapMinimum                 = defaultHeapMinimum
-	MemoryLimitHeapGoalHeadroom = memoryLimitHeapGoalHeadroom
+	MemoryLimitHeapGoalHeadroomPercent = memoryLimitHeapGoalHeadroomPercent
+	MemoryLimitMinHeapGoalHeadroom     = memoryLimitMinHeapGoalHeadroom
 )
 
 type GCController struct {

src/runtime/mgcpacer.go

@@ -61,11 +61,16 @@ const (
 	// that can accumulate on a P before updating gcController.stackSize.
 	maxStackScanSlack = 8 << 10
 
-	// memoryLimitHeapGoalHeadroom is the amount of headroom the pacer gives to
+	// memoryLimitMinHeapGoalHeadroom is the minimum amount of headroom the
-	// the heap goal when operating in the memory-limited regime. That is,
+	// pacer gives to the heap goal when operating in the memory-limited regime.
-	// it'll reduce the heap goal by this many extra bytes off of the base
+	// That is, it'll reduce the heap goal by this many extra bytes off of the
-	// calculation.
+	// base calculation, at minimum.
-	memoryLimitHeapGoalHeadroom = 1 << 20
+	memoryLimitMinHeapGoalHeadroom = 1 << 20
+
+	// memoryLimitHeapGoalHeadroomPercent is how headroom the memory-limit-based
+	// heap goal should have as a percent of the maximum possible heap goal allowed
+	// to maintain the memory limit.
+	memoryLimitHeapGoalHeadroomPercent = 3
 )
 
 // gcController implements the GC pacing controller that determines
@@ -968,8 +973,10 @@ func (c *gcControllerState) memoryLimitHeapGoal() uint64 {
 	//
 	// In practice this computation looks like the following:
 	//
-	//    memoryLimit - ((mappedReady - heapFree - heapAlloc) + max(mappedReady - memoryLimit, 0)) - memoryLimitHeapGoalHeadroom
+	//    goal := memoryLimit - ((mappedReady - heapFree - heapAlloc) + max(mappedReady - memoryLimit, 0))
-	//                    ^1                                    ^2                                   ^3
+	//                    ^1                                    ^2
+	//    goal -= goal / 100 * memoryLimitHeapGoalHeadroomPercent
+	//    ^3
 	//
 	// Let's break this down.
 	//
@@ -1001,11 +1008,14 @@ func (c *gcControllerState) memoryLimitHeapGoal() uint64 {
 	// terms of heap objects, but it takes more than X bytes (e.g. due to fragmentation) to store
 	// X bytes worth of objects.
 	//
-	// The third term (marker 3) subtracts an additional memoryLimitHeapGoalHeadroom bytes from the
+	// The final adjustment (marker 3) reduces the maximum possible memory limit heap goal by
-	// heap goal. As the name implies, this is to provide additional headroom in the face of pacing
+	// memoryLimitHeapGoalPercent. As the name implies, this is to provide additional headroom in
-	// inaccuracies. This is a fixed number of bytes because these inaccuracies disproportionately
+	// the face of pacing inaccuracies, and also to leave a buffer of unscavenged memory so the
-	// affect small heaps: as heaps get smaller, the pacer's inputs get fuzzier. Shorter GC cycles
+	// allocator isn't constantly scavenging. The reduction amount also has a fixed minimum
-	// and less GC work means noisy external factors like the OS scheduler have a greater impact.
+	// (memoryLimitMinHeapGoalHeadroom, not pictured) because the aforementioned pacing inaccuracies
+	// disproportionately affect small heaps: as heaps get smaller, the pacer's inputs get fuzzier.
+	// Shorter GC cycles and less GC work means noisy external factors like the OS scheduler have a
+	// greater impact.
 
 	memoryLimit := uint64(c.memoryLimit.Load())
 
@@ -1029,12 +1039,19 @@ func (c *gcControllerState) memoryLimitHeapGoal() uint64 {
 	// Compute the goal.
 	goal := memoryLimit - (nonHeapMemory + overage)
 
-	// Apply some headroom to the goal to account for pacing inaccuracies.
+	// Apply some headroom to the goal to account for pacing inaccuracies and to reduce
-	// Be careful about small limits.
+	// the impact of scavenging at allocation time in response to a high allocation rate
-	if goal < memoryLimitHeapGoalHeadroom || goal-memoryLimitHeapGoalHeadroom < memoryLimitHeapGoalHeadroom {
+	// when GOGC=off. See issue #57069. Also, be careful about small limits.
-		goal = memoryLimitHeapGoalHeadroom
+	headroom := goal / 100 * memoryLimitHeapGoalHeadroomPercent
+	if headroom < memoryLimitMinHeapGoalHeadroom {
+		// Set a fixed minimum to deal with the particularly large effect pacing inaccuracies
+		// have for smaller heaps.
+		headroom = memoryLimitMinHeapGoalHeadroom
+	}
+	if goal < headroom || goal-headroom < headroom {
+		goal = headroom
 	} else {
-		goal = goal - memoryLimitHeapGoalHeadroom
+		goal = goal - headroom
 	}
 	// Don't let us go below the live heap. A heap goal below the live heap doesn't make sense.
 	if goal < c.heapMarked {

src/runtime/mgcpacer_test.go

@@ -417,7 +417,7 @@ func TestGcPacer(t *testing.T) {
 			length:        50,
 			checker: func(t *testing.T, c []gcCycleResult) {
 				n := len(c)
-				if peak := c[n-1].heapPeak; peak >= (512<<20)-MemoryLimitHeapGoalHeadroom {
+				if peak := c[n-1].heapPeak; peak >= applyMemoryLimitHeapGoalHeadroom(512<<20) {
 					t.Errorf("peak heap size reaches heap limit: %d", peak)
 				}
 				if n >= 25 {
@@ -446,7 +446,7 @@ func TestGcPacer(t *testing.T) {
 			length:        50,
 			checker: func(t *testing.T, c []gcCycleResult) {
 				n := len(c)
-				if goal := c[n-1].heapGoal; goal != (512<<20)-MemoryLimitHeapGoalHeadroom {
+				if goal := c[n-1].heapGoal; goal != applyMemoryLimitHeapGoalHeadroom(512<<20) {
 					t.Errorf("heap goal is not the heap limit: %d", goal)
 				}
 				if n >= 25 {
@@ -510,7 +510,7 @@ func TestGcPacer(t *testing.T) {
 			checker: func(t *testing.T, c []gcCycleResult) {
 				n := len(c)
 				if n < 10 {
-					if goal := c[n-1].heapGoal; goal != (512<<20)-MemoryLimitHeapGoalHeadroom {
+					if goal := c[n-1].heapGoal; goal != applyMemoryLimitHeapGoalHeadroom(512<<20) {
 						t.Errorf("heap goal is not the heap limit: %d", goal)
 					}
 				}
@@ -550,7 +550,7 @@ func TestGcPacer(t *testing.T) {
 				n := len(c)
 				if n > 12 {
 					// We're trying to saturate the memory limit.
-					if goal := c[n-1].heapGoal; goal != (512<<20)-MemoryLimitHeapGoalHeadroom {
+					if goal := c[n-1].heapGoal; goal != applyMemoryLimitHeapGoalHeadroom(512<<20) {
 						t.Errorf("heap goal is not the heap limit: %d", goal)
 					}
 				}
@@ -581,7 +581,7 @@ func TestGcPacer(t *testing.T) {
 			length:        50,
 			checker: func(t *testing.T, c []gcCycleResult) {
 				n := len(c)
-				if goal := c[n-1].heapGoal; goal != (512<<20)-MemoryLimitHeapGoalHeadroom {
+				if goal := c[n-1].heapGoal; goal != applyMemoryLimitHeapGoalHeadroom(512<<20) {
 					t.Errorf("heap goal is not the heap limit: %d", goal)
 				}
 				if n >= 25 {
@@ -1019,6 +1019,19 @@ func (f float64Stream) limit(min, max float64) float64Stream {
 	}
 }
 
+func applyMemoryLimitHeapGoalHeadroom(goal uint64) uint64 {
+	headroom := goal / 100 * MemoryLimitHeapGoalHeadroomPercent
+	if headroom < MemoryLimitMinHeapGoalHeadroom {
+		headroom = MemoryLimitMinHeapGoalHeadroom
+	}
+	if goal < headroom || goal-headroom < headroom {
+		goal = headroom
+	} else {
+		goal -= headroom
+	}
+	return goal
+}
+
 func TestIdleMarkWorkerCount(t *testing.T) {
 	const workers = 10
 	c := NewGCController(100, math.MaxInt64)