runtime: account for tiny allocs, for testing.AllocsPerRun

Fixes #8734.

LGTM=r, bradfitz, dvyukov
R=bradfitz, r, dvyukov
CC=golang-codereviews, iant, khr
https://golang.org/cl/143150043

src/runtime/malloc.c

@@ -79,6 +79,8 @@ runtime·purgecachedstats(MCache *c)
 	h = &runtime·mheap;
 	mstats.heap_alloc += c->local_cachealloc;
 	c->local_cachealloc = 0;
+	mstats.tinyallocs += c->local_tinyallocs;
+	c->local_tinyallocs = 0;
 	mstats.nlookup += c->local_nlookup;
 	c->local_nlookup = 0;
 	h->largefree += c->local_largefree;
@@ -92,9 +94,10 @@ runtime·purgecachedstats(MCache *c)
 }
 
 // Size of the trailing by_size array differs between Go and C,
+// and all data after by_size is local to C, not exported to Go.
 // NumSizeClasses was changed, but we can not change Go struct because of backward compatibility.
 // sizeof_C_MStats is what C thinks about size of Go struct.
-uintptr runtime·sizeof_C_MStats = sizeof(MStats) - (NumSizeClasses - 61) * sizeof(mstats.by_size[0]);
+uintptr runtime·sizeof_C_MStats = offsetof(MStats, by_size[61]);
 
 #define MaxArena32 (2U<<30)
 

src/runtime/malloc.go

@@ -103,7 +103,6 @@ func mallocgc(size uintptr, typ *_type, flags int) unsafe.Pointer {
 			// standalone escaping variables. On a json benchmark
 			// the allocator reduces number of allocations by ~12% and
 			// reduces heap size by ~20%.
-
 			tinysize := uintptr(c.tinysize)
 			if size <= tinysize {
 				tiny := unsafe.Pointer(c.tiny)
@@ -121,6 +120,7 @@ func mallocgc(size uintptr, typ *_type, flags int) unsafe.Pointer {
 					x = tiny
 					c.tiny = (*byte)(add(x, size))
 					c.tinysize -= uintptr(size1)
+					c.local_tinyallocs++
 					if debugMalloc {
 						mp := acquirem()
 						if mp.mallocing == 0 {

src/runtime/malloc.h

@@ -278,6 +278,8 @@ struct MStats
 		uint64 nmalloc;
 		uint64 nfree;
 	} by_size[NumSizeClasses];
+	
+	uint64	tinyallocs;	// number of tiny allocations that didn't cause actual allocation; not exported to Go directly
 };
 
 #define mstats runtime·memstats
@@ -331,6 +333,7 @@ struct MCache
 	// See "Tiny allocator" comment in malloc.goc.
 	byte*	tiny;
 	uintptr	tinysize;
+	uintptr	local_tinyallocs;	// number of tiny allocs not counted in other stats
 	// The rest is not accessed on every malloc.
 	MSpan*	alloc[NumSizeClasses];	// spans to allocate from
 

src/runtime/mem.go

@@ -64,7 +64,7 @@ func init() {
 	var memStats MemStats
 	if sizeof_C_MStats != unsafe.Sizeof(memStats) {
 		println(sizeof_C_MStats, unsafe.Sizeof(memStats))
-		panic("MStats vs MemStatsType size mismatch")
+		gothrow("MStats vs MemStatsType size mismatch")
 	}
 }
 

src/runtime/mgc0.c

@@ -1245,6 +1245,7 @@ runtime·updatememstats(GCStats *stats)
 		mstats.by_size[i].nmalloc += runtime·mheap.nsmallfree[i];
 		smallfree += runtime·mheap.nsmallfree[i] * runtime·class_to_size[i];
 	}
+	mstats.nfree += mstats.tinyallocs;
 	mstats.nmalloc += mstats.nfree;
 
 	// Calculate derived stats.

src/runtime/mheap.c

@@ -184,6 +184,8 @@ mheap_alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large)
 	// transfer stats from cache to global
 	mstats.heap_alloc += g->m->mcache->local_cachealloc;
 	g->m->mcache->local_cachealloc = 0;
+	mstats.tinyallocs += g->m->mcache->local_tinyallocs;
+	g->m->mcache->local_tinyallocs = 0;
 
 	s = MHeap_AllocSpanLocked(h, npage);
 	if(s != nil) {
@@ -465,6 +467,8 @@ mheap_free(MHeap *h, MSpan *s, int32 acct)
 	runtime·lock(&h->lock);
 	mstats.heap_alloc += g->m->mcache->local_cachealloc;
 	g->m->mcache->local_cachealloc = 0;
+	mstats.tinyallocs += g->m->mcache->local_tinyallocs;
+	g->m->mcache->local_tinyallocs = 0;
 	if(acct) {
 		mstats.heap_alloc -= s->npages<<PageShift;
 		mstats.heap_objects--;

src/testing/allocs_test.go

@@ -0,0 +1,29 @@
+// Copyright 2014 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package testing_test
+
+import "testing"
+
+var global interface{}
+
+var allocsPerRunTests = []struct {
+	name   string
+	fn     func()
+	allocs float64
+}{
+	{"alloc *byte", func() { global = new(*byte) }, 1},
+	{"alloc complex128", func() { global = new(complex128) }, 1},
+	{"alloc float64", func() { global = new(float64) }, 1},
+	{"alloc int32", func() { global = new(int32) }, 1},
+	{"alloc byte", func() { global = new(byte) }, 1},
+}
+
+func TestAllocsPerRun(t *testing.T) {
+	for _, tt := range allocsPerRunTests {
+		if allocs := testing.AllocsPerRun(100, tt.fn); allocs != tt.allocs {
+			t.Errorf("AllocsPerRun(100, %s) = %v, want %v", tt.name, allocs, tt.allocs)
+		}
+	}
+}