internal/runtime/maps: loop invariant code motion with h2(hash) by hand

Change-Id: I0cd9763aeedfe326bc566da39b8be0d0ebd113ec
GitHub-Last-Rev: 1ec8864414
GitHub-Pull-Request: golang/go#74952
Reviewed-on: https://go-review.googlesource.com/c/go/+/694016
Auto-Submit: Michael Pratt <mpratt@google.com>
Reviewed-by: Keith Randall <khr@google.com>
Reviewed-by: Dmitri Shuralyov <dmitshur@google.com>
Reviewed-by: Michael Pratt <mpratt@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Keith Randall <khr@golang.org>

src/internal/runtime/maps/runtime.go

@@ -94,10 +94,11 @@ func runtime_mapaccess1(typ *abi.MapType, m *Map, key unsafe.Pointer) unsafe.Poi
 
 	// Probe table.
 	seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+	h2Hash := h2(hash)
 	for ; ; seq = seq.next() {
 		g := t.groups.group(typ, seq.offset)
 
-		match := g.ctrls().matchH2(h2(hash))
+		match := g.ctrls().matchH2(h2Hash)
 
 		for match != 0 {
 			i := match.first()
@@ -168,10 +169,11 @@ func runtime_mapaccess2(typ *abi.MapType, m *Map, key unsafe.Pointer) (unsafe.Po
 
 	// Probe table.
 	seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+	h2Hash := h2(hash)
 	for ; ; seq = seq.next() {
 		g := t.groups.group(typ, seq.offset)
 
-		match := g.ctrls().matchH2(h2(hash))
+		match := g.ctrls().matchH2(h2Hash)
 
 		for match != 0 {
 			i := match.first()
@@ -262,9 +264,10 @@ outer:
 		var firstDeletedGroup groupReference
 		var firstDeletedSlot uintptr
 
+		h2Hash := h2(hash)
 		for ; ; seq = seq.next() {
 			g := t.groups.group(typ, seq.offset)
-			match := g.ctrls().matchH2(h2(hash))
+			match := g.ctrls().matchH2(h2Hash)
 
 			// Look for an existing slot containing this key.
 			for match != 0 {
@@ -329,7 +332,7 @@ outer:
 						slotElem = emem
 					}
 
-					g.ctrls().set(i, ctrl(h2(hash)))
+					g.ctrls().set(i, ctrl(h2Hash))
 					t.growthLeft--
 					t.used++
 					m.used++

src/internal/runtime/maps/runtime_fast32.go

@@ -55,10 +55,11 @@ func runtime_mapaccess1_fast32(typ *abi.MapType, m *Map, key uint32) unsafe.Poin
 
 	// Probe table.
 	seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+	h2Hash := h2(hash)
 	for ; ; seq = seq.next() {
 		g := t.groups.group(typ, seq.offset)
 
-		match := g.ctrls().matchH2(h2(hash))
+		match := g.ctrls().matchH2(h2Hash)
 
 		for match != 0 {
 			i := match.first()
@@ -124,10 +125,11 @@ func runtime_mapaccess2_fast32(typ *abi.MapType, m *Map, key uint32) (unsafe.Poi
 
 	// Probe table.
 	seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+	h2Hash := h2(hash)
 	for ; ; seq = seq.next() {
 		g := t.groups.group(typ, seq.offset)
 
-		match := g.ctrls().matchH2(h2(hash))
+		match := g.ctrls().matchH2(h2Hash)
 
 		for match != 0 {
 			i := match.first()
@@ -245,9 +247,10 @@ outer:
 		var firstDeletedGroup groupReference
 		var firstDeletedSlot uintptr
 
+		h2Hash := h2(hash)
 		for ; ; seq = seq.next() {
 			g := t.groups.group(typ, seq.offset)
-			match := g.ctrls().matchH2(h2(hash))
+			match := g.ctrls().matchH2(h2Hash)
 
 			// Look for an existing slot containing this key.
 			for match != 0 {
@@ -302,7 +305,7 @@ outer:
 
 				slotElem = g.elem(typ, i)
 
-				g.ctrls().set(i, ctrl(h2(hash)))
+				g.ctrls().set(i, ctrl(h2Hash))
 				t.growthLeft--
 				t.used++
 				m.used++
@@ -383,9 +386,10 @@ outer:
 		var firstDeletedGroup groupReference
 		var firstDeletedSlot uintptr
 
+		h2Hash := h2(hash)
 		for ; ; seq = seq.next() {
 			g := t.groups.group(typ, seq.offset)
-			match := g.ctrls().matchH2(h2(hash))
+			match := g.ctrls().matchH2(h2Hash)
 
 			// Look for an existing slot containing this key.
 			for match != 0 {
@@ -435,7 +439,7 @@ outer:
 
 				slotElem = g.elem(typ, i)
 
-				g.ctrls().set(i, ctrl(h2(hash)))
+				g.ctrls().set(i, ctrl(h2Hash))
 				t.growthLeft--
 				t.used++
 				m.used++

src/internal/runtime/maps/runtime_fast64.go

@@ -55,10 +55,11 @@ func runtime_mapaccess1_fast64(typ *abi.MapType, m *Map, key uint64) unsafe.Poin
 
 	// Probe table.
 	seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+	h2Hash := h2(hash)
 	for ; ; seq = seq.next() {
 		g := t.groups.group(typ, seq.offset)
 
-		match := g.ctrls().matchH2(h2(hash))
+		match := g.ctrls().matchH2(h2Hash)
 
 		for match != 0 {
 			i := match.first()
@@ -124,10 +125,12 @@ func runtime_mapaccess2_fast64(typ *abi.MapType, m *Map, key uint64) (unsafe.Poi
 
 	// Probe table.
 	seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+
+	h2Hash := h2(hash)
 	for ; ; seq = seq.next() {
 		g := t.groups.group(typ, seq.offset)
 
-		match := g.ctrls().matchH2(h2(hash))
+		match := g.ctrls().matchH2(h2Hash)
 
 		for match != 0 {
 			i := match.first()
@@ -245,9 +248,10 @@ outer:
 		var firstDeletedGroup groupReference
 		var firstDeletedSlot uintptr
 
+		h2Hash := h2(hash)
 		for ; ; seq = seq.next() {
 			g := t.groups.group(typ, seq.offset)
-			match := g.ctrls().matchH2(h2(hash))
+			match := g.ctrls().matchH2(h2Hash)
 
 			// Look for an existing slot containing this key.
 			for match != 0 {
@@ -302,7 +306,7 @@ outer:
 
 				slotElem = g.elem(typ, i)
 
-				g.ctrls().set(i, ctrl(h2(hash)))
+				g.ctrls().set(i, ctrl(h2Hash))
 				t.growthLeft--
 				t.used++
 				m.used++
@@ -422,9 +426,10 @@ outer:
 		var firstDeletedGroup groupReference
 		var firstDeletedSlot uintptr
 
+		h2Hash := h2(hash)
 		for ; ; seq = seq.next() {
 			g := t.groups.group(typ, seq.offset)
-			match := g.ctrls().matchH2(h2(hash))
+			match := g.ctrls().matchH2(h2Hash)
 
 			// Look for an existing slot containing this key.
 			for match != 0 {
@@ -474,7 +479,7 @@ outer:
 
 				slotElem = g.elem(typ, i)
 
-				g.ctrls().set(i, ctrl(h2(hash)))
+				g.ctrls().set(i, ctrl(h2Hash))
 				t.growthLeft--
 				t.used++
 				m.used++

src/internal/runtime/maps/runtime_faststr.go

@@ -131,10 +131,11 @@ func runtime_mapaccess1_faststr(typ *abi.MapType, m *Map, key string) unsafe.Poi
 
 	// Probe table.
 	seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+	h2Hash := h2(hash)
 	for ; ; seq = seq.next() {
 		g := t.groups.group(typ, seq.offset)
 
-		match := g.ctrls().matchH2(h2(hash))
+		match := g.ctrls().matchH2(h2Hash)
 
 		for match != 0 {
 			i := match.first()
@@ -190,10 +191,11 @@ func runtime_mapaccess2_faststr(typ *abi.MapType, m *Map, key string) (unsafe.Po
 
 	// Probe table.
 	seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+	h2Hash := h2(hash)
 	for ; ; seq = seq.next() {
 		g := t.groups.group(typ, seq.offset)
 
-		match := g.ctrls().matchH2(h2(hash))
+		match := g.ctrls().matchH2(h2Hash)
 
 		for match != 0 {
 			i := match.first()
@@ -313,9 +315,10 @@ outer:
 		var firstDeletedGroup groupReference
 		var firstDeletedSlot uintptr
 
+		h2Hash := h2(hash)
 		for ; ; seq = seq.next() {
 			g := t.groups.group(typ, seq.offset)
-			match := g.ctrls().matchH2(h2(hash))
+			match := g.ctrls().matchH2(h2Hash)
 
 			// Look for an existing slot containing this key.
 			for match != 0 {
@@ -373,7 +376,7 @@ outer:
 
 				slotElem = g.elem(typ, i)
 
-				g.ctrls().set(i, ctrl(h2(hash)))
+				g.ctrls().set(i, ctrl(h2Hash))
 				t.growthLeft--
 				t.used++
 				m.used++

src/internal/runtime/maps/table.go

@@ -192,10 +192,11 @@ func (t *table) getWithKey(typ *abi.MapType, hash uintptr, key unsafe.Pointer) (
 	// load factors, k is less than 32, meaning that the number of false
 	// positive comparisons we must perform is less than 1/8 per find.
 	seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+	h2Hash := h2(hash)
 	for ; ; seq = seq.next() {
 		g := t.groups.group(typ, seq.offset)
 
-		match := g.ctrls().matchH2(h2(hash))
+		match := g.ctrls().matchH2(h2Hash)
 
 		for match != 0 {
 			i := match.first()
@@ -225,10 +226,11 @@ func (t *table) getWithKey(typ *abi.MapType, hash uintptr, key unsafe.Pointer) (
 
 func (t *table) getWithoutKey(typ *abi.MapType, hash uintptr, key unsafe.Pointer) (unsafe.Pointer, bool) {
 	seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+	h2Hash := h2(hash)
 	for ; ; seq = seq.next() {
 		g := t.groups.group(typ, seq.offset)
 
-		match := g.ctrls().matchH2(h2(hash))
+		match := g.ctrls().matchH2(h2Hash)
 
 		for match != 0 {
 			i := match.first()
@@ -271,9 +273,10 @@ func (t *table) PutSlot(typ *abi.MapType, m *Map, hash uintptr, key unsafe.Point
 	var firstDeletedGroup groupReference
 	var firstDeletedSlot uintptr
 
+	h2Hash := h2(hash)
 	for ; ; seq = seq.next() {
 		g := t.groups.group(typ, seq.offset)
-		match := g.ctrls().matchH2(h2(hash))
+		match := g.ctrls().matchH2(h2Hash)
 
 		// Look for an existing slot containing this key.
 		for match != 0 {
@@ -348,7 +351,7 @@ func (t *table) PutSlot(typ *abi.MapType, m *Map, hash uintptr, key unsafe.Point
 				slotElem = emem
 			}
 
-			g.ctrls().set(i, ctrl(h2(hash)))
+			g.ctrls().set(i, ctrl(h2Hash))
 			t.growthLeft--
 			t.used++
 			m.used++
@@ -420,9 +423,10 @@ func (t *table) uncheckedPutSlot(typ *abi.MapType, hash uintptr, key, elem unsaf
 // Delete returns true if it put a tombstone in t.
 func (t *table) Delete(typ *abi.MapType, m *Map, hash uintptr, key unsafe.Pointer) bool {
 	seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
+	h2Hash := h2(hash)
 	for ; ; seq = seq.next() {
 		g := t.groups.group(typ, seq.offset)
-		match := g.ctrls().matchH2(h2(hash))
+		match := g.ctrls().matchH2(h2Hash)
 
 		for match != 0 {
 			i := match.first()