runtime: avoid zeroing scavenged memory

On Linux, memory returned to the kernel via MADV_DONTNEED is guaranteed
to be zero-filled on its next use.

This commit leverages this kernel behavior to avoid a redundant software
zeroing pass in the runtime, improving performance.

Change-Id: Ia14343b447a2cec7af87644fe8050e23e983c787
GitHub-Last-Rev: 6c8df32283
GitHub-Pull-Request: golang/go#76063
Reviewed-on: https://go-review.googlesource.com/c/go/+/715160
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Michael Knyszek <mknyszek@google.com>
Reviewed-by: David Chase <drchase@google.com>

src/runtime/arena.go

@@ -1051,7 +1051,11 @@ func (h *mheap) allocUserArenaChunk() *mspan {
 
 	// Model the user arena as a heap span for a large object.
 	spc := makeSpanClass(0, false)
-	h.initSpan(s, spanAllocHeap, spc, base, userArenaChunkPages)
+	// A user arena chunk is always fresh from the OS. It's either newly allocated
+	// via sysAlloc() or reused from the readyList after a sysFault(). The memory is
+	// then re-mapped via sysMap(), so we can safely treat it as scavenged; the
+	// kernel guarantees it will be zero-filled on its next use.
+	h.initSpan(s, spanAllocHeap, spc, base, userArenaChunkPages, userArenaChunkBytes)
 	s.isUserArenaChunk = true
 	s.elemsize -= userArenaChunkReserveBytes()
 	s.freeindex = 1

src/runtime/mem.go

@@ -70,6 +70,12 @@ func sysUnused(v unsafe.Pointer, n uintptr) {
 	sysUnusedOS(v, n)
 }
 
+// needZeroAfterSysUnused reports whether memory returned by sysUnused must be
+// zeroed for use.
+func needZeroAfterSysUnused() bool {
+	return needZeroAfterSysUnusedOS()
+}
+
 // sysUsed transitions a memory region from Prepared to Ready. It notifies the
 // operating system that the memory region is needed and ensures that the region
 // may be safely accessed. This is typically a no-op on systems that don't have

src/runtime/mem_aix.go

@@ -79,3 +79,7 @@ func sysMapOS(v unsafe.Pointer, n uintptr, _ string) {
 		throw("runtime: cannot map pages in arena address space")
 	}
 }
+
+func needZeroAfterSysUnusedOS() bool {
+	return true
+}

src/runtime/mem_bsd.go

@@ -85,3 +85,7 @@ func sysMapOS(v unsafe.Pointer, n uintptr, _ string) {
 		throw("runtime: cannot map pages in arena address space")
 	}
 }
+
+func needZeroAfterSysUnusedOS() bool {
+	return true
+}

src/runtime/mem_darwin.go

@@ -74,3 +74,7 @@ func sysMapOS(v unsafe.Pointer, n uintptr, _ string) {
 		throw("runtime: cannot map pages in arena address space")
 	}
 }
+
+func needZeroAfterSysUnusedOS() bool {
+	return true
+}

src/runtime/mem_linux.go

@@ -188,3 +188,7 @@ func sysMapOS(v unsafe.Pointer, n uintptr, vmaName string) {
 		sysNoHugePageOS(v, n)
 	}
 }
+
+func needZeroAfterSysUnusedOS() bool {
+	return debug.madvdontneed == 0
+}

src/runtime/mem_sbrk.go

@@ -296,3 +296,7 @@ func sysReserveAlignedSbrk(size, align uintptr) (unsafe.Pointer, uintptr) {
 	})
 	return unsafe.Pointer(p), size
 }
+
+func needZeroAfterSysUnusedOS() bool {
+	return true
+}

src/runtime/mem_windows.go

@@ -132,3 +132,7 @@ func sysReserveOS(v unsafe.Pointer, n uintptr, _ string) unsafe.Pointer {
 
 func sysMapOS(v unsafe.Pointer, n uintptr, _ string) {
 }
+
+func needZeroAfterSysUnusedOS() bool {
+	return true
+}

src/runtime/mheap.go

@@ -1394,7 +1394,7 @@ HaveSpan:
 	}
 
 	// Initialize the span.
-	h.initSpan(s, typ, spanclass, base, npages)
+	h.initSpan(s, typ, spanclass, base, npages, scav)
 
 	if valgrindenabled {
 		valgrindMempoolMalloc(unsafe.Pointer(arenaBase(arenaIndex(base))), unsafe.Pointer(base), npages*pageSize)
@@ -1440,11 +1440,17 @@ HaveSpan:
 
 // initSpan initializes a blank span s which will represent the range
 // [base, base+npages*pageSize). typ is the type of span being allocated.
-func (h *mheap) initSpan(s *mspan, typ spanAllocType, spanclass spanClass, base, npages uintptr) {
+func (h *mheap) initSpan(s *mspan, typ spanAllocType, spanclass spanClass, base, npages, scav uintptr) {
 	// At this point, both s != nil and base != 0, and the heap
 	// lock is no longer held. Initialize the span.
 	s.init(base, npages)
-	if h.allocNeedsZero(base, npages) {
+	// Always call allocNeedsZero to update the arena's zeroedBase watermark
+	// and determine if the memory is considered dirty.
+	needZero := h.allocNeedsZero(base, npages)
+	// If these pages were scavenged (returned to the OS), the kernel guarantees
+	// they will be zero-filled on next use (fault-in), so we can treat them as
+	// already zeroed and skip explicit clearing.
+	if (needZeroAfterSysUnused() || scav != npages*pageSize) && needZero {
 		s.needzero = 1
 	}
 	nbytes := npages * pageSize