Stowage/go
2
0
Fork 0
mirror of https://github.com/golang/go.git synced 2026-06-27 03:11:23 +00:00
Code Issues Projects Releases Packages Wiki Activity

runtime/maps: only grow small full maps when inserting new keys

Browse source 
For small full maps, the existing implementation triggers the growth logic
unconditionally, regardless of whether the key already exists.

This change fixes this issue by only triggering growth when inserting a new
key, not when updating an existing key.

Fixes #78853

Change-Id: I087abf8f74b73dbc7498319dc000832a993ee0bd
GitHub-Last-Rev: ac99de3655
GitHub-Pull-Request: golang/go#78929
Reviewed-on: https://go-review.googlesource.com/c/go/+/770303
Reviewed-by: Keith Randall <khr@golang.org>
Reviewed-by: Cherry Mui <cherryyz@google.com>
Reviewed-by: Keith Randall <khr@google.com>
Auto-Submit: Keith Randall <khr@golang.org>
LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
This commit is contained in:
Hesen 2026-04-29 02:53:19 +00:00 • committed by Gopher Robot
parent 409f784bea
commit 8042aaf03c
7 changed files with 327 additions and 72 deletions
Download patch file Download diff file Expand all files Collapse all files

30
src/internal/runtime/maps/map.go
View file

@ -499,22 +499,23 @@ func (m *Map) PutSlot(typ *abi.MapType, key unsafe.Pointer) unsafe.Pointer {
}
if m.dirLen == 0 {
if m.used < abi.MapGroupSlots {
elem := m.putSlotSmall(typ, hash, key)
elem := m.putSlotSmall(typ, hash, key)
if elem == nil {
// Can't fit another entry, grow to full size map.
tab := m.growToTable(typ)
if m.writing == 0 {
fatal("concurrent map writes")
}
m.writing ^= 1
elem = tab.uncheckedPutSlotForAssign(typ, hash, key)
m.used++
return elem
tab.checkInvariants(typ, m)
}
// Can't fit another entry, grow to full size map.
//
// TODO(prattmic): If this is an update to an existing key then
// we actually don't need to grow.
m.growToTable(typ)
if m.writing == 0 {
fatal("concurrent map writes")
}
m.writing ^= 1
return elem
}
for {
@ -568,7 +569,7 @@ func (m *Map) putSlotSmall(typ *abi.MapType, hash uintptr, key unsafe.Pointer) u
// more efficient than matchEmpty.
match = g.ctrls().matchEmptyOrDeleted()
if match == 0 {
fatal("small map with no empty slot (concurrent map writes?)")
// No empty slot found. Need to grow the map.
return nil
}
@ -605,7 +606,7 @@ func (m *Map) growToSmall(typ *abi.MapType) {
g.ctrls().setEmpty()
}
func (m *Map) growToTable(typ *abi.MapType) {
func (m *Map) growToTable(typ *abi.MapType) *table {
tab := newTable(typ, 2*abi.MapGroupSlots, 0, 0)
g := groupReference{
@ -642,6 +643,7 @@ func (m *Map) growToTable(typ *abi.MapType) {
m.globalDepth = 0
m.globalShift = depthToShift(m.globalDepth)
return tab
}
func (m *Map) Delete(typ *abi.MapType, key unsafe.Pointer) {

50
src/internal/runtime/maps/map_test.go
View file

@ -57,6 +57,56 @@ func TestMapPut(t *testing.T) {
}
}
func TestSmallMapGrow(t *testing.T) {
m, typ := maps.NewTestMap[uint32, uint64](8)
key := uint32(0)
elem := uint64(256 + 0)
for i := 0; i < 8; i++ {
key += 1
elem += 1
m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem))
if maps.DebugLog {
fmt.Printf("After put %d: %v\n", key, m)
}
}
if m.TableCount() != 0 {
t.Errorf("TableCount() got %d want 0", m.TableCount())
}
key = uint32(0)
elem = uint64(256 + 10)
for i := 0; i < 8; i++ {
key += 1
elem += 1
m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem))
if maps.DebugLog {
fmt.Printf("After put %d: %v\n", key, m)
}
}
if m.TableCount() != 0 {
t.Errorf("TableCount() got %d want 0", m.TableCount())
}
key += 1
elem += 1
m.Put(typ, unsafe.Pointer(&key), unsafe.Pointer(&elem))
if maps.DebugLog {
fmt.Printf("After put %d: %v\n", key, m)
}
if m.TableCount() != 1 {
t.Errorf("TableCount() got %d want 1", m.TableCount())
}
}
// Grow enough to cause a table split.
func TestMapSplit(t *testing.T) {
m, typ := maps.NewTestMap[uint32, uint64](0)

22
src/internal/runtime/maps/runtime.go
View file

@ -171,19 +171,23 @@ func runtime_mapassign(typ *abi.MapType, m *Map, key unsafe.Pointer) unsafe.Poin
}
if m.dirLen == 0 {
if m.used < abi.MapGroupSlots {
elem := m.putSlotSmall(typ, hash, key)
elem := m.putSlotSmall(typ, hash, key)
if elem == nil {
// Can't fit another entry, grow to full size map.
tab := m.growToTable(typ)
if m.writing == 0 {
fatal("concurrent map writes")
}
m.writing ^= 1
elem = tab.uncheckedPutSlotForAssign(typ, hash, key)
m.used++
return elem
tab.checkInvariants(typ, m)
}
// Can't fit another entry, grow to full size map.
m.growToTable(typ)
if m.writing == 0 {
fatal("concurrent map writes")
}
m.writing ^= 1
return elem
}
var slotElem unsafe.Pointer

77
src/internal/runtime/maps/runtime_fast32.go
View file

@ -136,7 +136,8 @@ func (m *Map) putSlotSmallFast32(typ *abi.MapType, hash uintptr, key uint32) uns
// more efficient than matchEmpty.
match = g.ctrls().matchEmptyOrDeleted()
if match == 0 {
fatal("small map with no empty slot (concurrent map writes?)")
// No empty slot found. Need to grow the map.
return nil
}
i := match.first()
@ -152,6 +153,36 @@ func (m *Map) putSlotSmallFast32(typ *abi.MapType, hash uintptr, key uint32) uns
return slotElem
}
func (t *table) uncheckedPutSlotForAssignFast32(typ *abi.MapType, hash uintptr, key uint32) unsafe.Pointer {
if t.growthLeft == 0 {
panic("invariant failed: growthLeft is unexpectedly 0")
}
// Given key and its hash hash(key), to insert it, we construct a
// probeSeq, and use it to find the first group with an unoccupied (empty
// or deleted) slot. We place the key/value into the first such slot in
// the group and mark it as full with key's H2.
seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
for ; ; seq = seq.next() {
g := t.groups.group(typ, seq.offset)
match := g.ctrls().matchEmptyOrDeleted()
if match != 0 {
i := match.first()
slotKey := g.key(typ, i)
*(*uint32)(slotKey) = key
slotElem := g.elem(typ, i)
t.growthLeft--
t.used++
g.ctrls().set(i, ctrl(h2(hash)))
return slotElem
}
}
}
//go:linkname runtime_mapassign_fast32 runtime.mapassign_fast32
func runtime_mapassign_fast32(typ *abi.MapType, m *Map, key uint32) unsafe.Pointer {
if m == nil {
@ -180,19 +211,23 @@ func runtime_mapassign_fast32(typ *abi.MapType, m *Map, key uint32) unsafe.Point
}
if m.dirLen == 0 {
if m.used < abi.MapGroupSlots {
elem := m.putSlotSmallFast32(typ, hash, key)
elem := m.putSlotSmallFast32(typ, hash, key)
if elem == nil {
// Can't fit another entry, grow to full size map.
tab := m.growToTable(typ)
if m.writing == 0 {
fatal("concurrent map writes")
}
m.writing ^= 1
elem = tab.uncheckedPutSlotForAssignFast32(typ, hash, key)
m.used++
return elem
tab.checkInvariants(typ, m)
}
// Can't fit another entry, grow to full size map.
m.growToTable(typ)
if m.writing == 0 {
fatal("concurrent map writes")
}
m.writing ^= 1
return elem
}
var slotElem unsafe.Pointer
@ -322,19 +357,23 @@ func runtime_mapassign_fast32ptr(typ *abi.MapType, m *Map, key unsafe.Pointer) u
}
if m.dirLen == 0 {
if m.used < abi.MapGroupSlots {
elem := m.putSlotSmallFastPtr(typ, hash, key)
elem := m.putSlotSmallFastPtr(typ, hash, key)
if elem == nil {
// Can't fit another entry, grow to full size map.
tab := m.growToTable(typ)
if m.writing == 0 {
fatal("concurrent map writes")
}
m.writing ^= 1
elem = tab.uncheckedPutSlotForAssignFastPtr(typ, hash, key)
m.used++
return elem
tab.checkInvariants(typ, m)
}
// Can't fit another entry, grow to full size map.
m.growToTable(typ)
if m.writing == 0 {
fatal("concurrent map writes")
}
m.writing ^= 1
return elem
}
var slotElem unsafe.Pointer

110
src/internal/runtime/maps/runtime_fast64.go
View file

@ -128,7 +128,8 @@ func (m *Map) putSlotSmallFast64(typ *abi.MapType, hash uintptr, key uint64) uns
// more efficient than matchEmpty.
match = g.ctrls().matchEmptyOrDeleted()
if match == 0 {
fatal("small map with no empty slot (concurrent map writes?)")
// No empty slot found. Need to grow the map.
return nil
}
i := match.first()
@ -144,6 +145,36 @@ func (m *Map) putSlotSmallFast64(typ *abi.MapType, hash uintptr, key uint64) uns
return slotElem
}
func (t *table) uncheckedPutSlotForAssignFast64(typ *abi.MapType, hash uintptr, key uint64) unsafe.Pointer {
if t.growthLeft == 0 {
panic("invariant failed: growthLeft is unexpectedly 0")
}
// Given key and its hash hash(key), to insert it, we construct a
// probeSeq, and use it to find the first group with an unoccupied (empty
// or deleted) slot. We place the key/value into the first such slot in
// the group and mark it as full with key's H2.
seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
for ; ; seq = seq.next() {
g := t.groups.group(typ, seq.offset)
match := g.ctrls().matchEmptyOrDeleted()
if match != 0 {
i := match.first()
slotKey := g.key(typ, i)
*(*uint64)(slotKey) = key
slotElem := g.elem(typ, i)
t.growthLeft--
t.used++
g.ctrls().set(i, ctrl(h2(hash)))
return slotElem
}
}
}
//go:linkname runtime_mapassign_fast64 runtime.mapassign_fast64
func runtime_mapassign_fast64(typ *abi.MapType, m *Map, key uint64) unsafe.Pointer {
if m == nil {
@ -172,19 +203,23 @@ func runtime_mapassign_fast64(typ *abi.MapType, m *Map, key uint64) unsafe.Point
}
if m.dirLen == 0 {
if m.used < abi.MapGroupSlots {
elem := m.putSlotSmallFast64(typ, hash, key)
elem := m.putSlotSmallFast64(typ, hash, key)
if elem == nil {
// Can't fit another entry, grow to full size map.
tab := m.growToTable(typ)
if m.writing == 0 {
fatal("concurrent map writes")
}
m.writing ^= 1
elem = tab.uncheckedPutSlotForAssignFast64(typ, hash, key)
m.used++
return elem
tab.checkInvariants(typ, m)
}
// Can't fit another entry, grow to full size map.
m.growToTable(typ)
if m.writing == 0 {
fatal("concurrent map writes")
}
m.writing ^= 1
return elem
}
var slotElem unsafe.Pointer
@ -306,7 +341,8 @@ func (m *Map) putSlotSmallFastPtr(typ *abi.MapType, hash uintptr, key unsafe.Poi
// more efficient than matchEmpty.
match = g.ctrls().matchEmptyOrDeleted()
if match == 0 {
fatal("small map with no empty slot (concurrent map writes?)")
// No empty slot found. Need to grow the map.
return nil
}
i := match.first()
@ -322,6 +358,36 @@ func (m *Map) putSlotSmallFastPtr(typ *abi.MapType, hash uintptr, key unsafe.Poi
return slotElem
}
func (t *table) uncheckedPutSlotForAssignFastPtr(typ *abi.MapType, hash uintptr, key unsafe.Pointer) unsafe.Pointer {
if t.growthLeft == 0 {
panic("invariant failed: growthLeft is unexpectedly 0")
}
// Given key and its hash hash(key), to insert it, we construct a
// probeSeq, and use it to find the first group with an unoccupied (empty
// or deleted) slot. We place the key/value into the first such slot in
// the group and mark it as full with key's H2.
seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
for ; ; seq = seq.next() {
g := t.groups.group(typ, seq.offset)
match := g.ctrls().matchEmptyOrDeleted()
if match != 0 {
i := match.first()
slotKey := g.key(typ, i)
*(*unsafe.Pointer)(slotKey) = key
slotElem := g.elem(typ, i)
t.growthLeft--
t.used++
g.ctrls().set(i, ctrl(h2(hash)))
return slotElem
}
}
}
// Key is a 64-bit pointer (only called on 64-bit GOARCH).
//
//go:linkname runtime_mapassign_fast64ptr runtime.mapassign_fast64ptr
@ -352,19 +418,23 @@ func runtime_mapassign_fast64ptr(typ *abi.MapType, m *Map, key unsafe.Pointer) u
}
if m.dirLen == 0 {
if m.used < abi.MapGroupSlots {
elem := m.putSlotSmallFastPtr(typ, hash, key)
elem := m.putSlotSmallFastPtr(typ, hash, key)
if elem == nil {
// Can't fit another entry, grow to full size map.
tab := m.growToTable(typ)
if m.writing == 0 {
fatal("concurrent map writes")
}
m.writing ^= 1
elem = tab.uncheckedPutSlotForAssignFastPtr(typ, hash, key)
m.used++
return elem
tab.checkInvariants(typ, m)
}
// Can't fit another entry, grow to full size map.
m.growToTable(typ)
if m.writing == 0 {
fatal("concurrent map writes")
}
m.writing ^= 1
return elem
}
var slotElem unsafe.Pointer

55
src/internal/runtime/maps/runtime_faststr.go
View file

@ -212,7 +212,8 @@ func (m *Map) putSlotSmallFastStr(typ *abi.MapType, hash uintptr, key string) un
// more efficient than matchEmpty.
match = g.ctrls().matchEmptyOrDeleted()
if match == 0 {
fatal("small map with no empty slot (concurrent map writes?)")
// No empty slot found. Need to grow the map.
return nil
}
i := match.first()
@ -228,6 +229,36 @@ func (m *Map) putSlotSmallFastStr(typ *abi.MapType, hash uintptr, key string) un
return slotElem
}
func (t *table) uncheckedPutSlotForAssignFastStr(typ *abi.MapType, hash uintptr, key string) unsafe.Pointer {
if t.growthLeft == 0 {
panic("invariant failed: growthLeft is unexpectedly 0")
}
// Given key and its hash hash(key), to insert it, we construct a
// probeSeq, and use it to find the first group with an unoccupied (empty
// or deleted) slot. We place the key/value into the first such slot in
// the group and mark it as full with key's H2.
seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
for ; ; seq = seq.next() {
g := t.groups.group(typ, seq.offset)
match := g.ctrls().matchEmptyOrDeleted()
if match != 0 {
i := match.first()
slotKey := g.key(typ, i)
*(*string)(slotKey) = key
slotElem := g.elem(typ, i)
t.growthLeft--
t.used++
g.ctrls().set(i, ctrl(h2(hash)))
return slotElem
}
}
}
//go:linkname runtime_mapassign_faststr runtime.mapassign_faststr
func runtime_mapassign_faststr(typ *abi.MapType, m *Map, key string) unsafe.Pointer {
if m == nil {
@ -256,19 +287,23 @@ func runtime_mapassign_faststr(typ *abi.MapType, m *Map, key string) unsafe.Poin
}
if m.dirLen == 0 {
if m.used < abi.MapGroupSlots {
elem := m.putSlotSmallFastStr(typ, hash, key)
elem := m.putSlotSmallFastStr(typ, hash, key)
if elem == nil {
// Can't fit another entry, grow to full size map.
tab := m.growToTable(typ)
if m.writing == 0 {
fatal("concurrent map writes")
}
m.writing ^= 1
elem = tab.uncheckedPutSlotForAssignFastStr(typ, hash, key)
m.used++
return elem
tab.checkInvariants(typ, m)
}
// Can't fit another entry, grow to full size map.
m.growToTable(typ)
if m.writing == 0 {
fatal("concurrent map writes")
}
m.writing ^= 1
return elem
}
var slotElem unsafe.Pointer

55
src/internal/runtime/maps/table.go
View file

@ -419,6 +419,61 @@ func (t *table) uncheckedPutSlot(typ *abi.MapType, hash uintptr, key, elem unsaf
}
}
// uncheckedPutSlotForAssign inserts a new key into the table for map
// assignment and returns the element slot.
//
// Decrements growthLeft and increments used.
//
// Requires that the entry does not exist in the table, and that the table has
// room for another element without rehashing.
//
// Before calling this method, you must ensure that the necessary check has
// been performed in advance.
//
// For indirect keys, memory is allocated and the key is copied into it.
// For indirect elements, memory is pre-allocated and the slot is returned
// to the caller, who must write the element value into it.
func (t *table) uncheckedPutSlotForAssign(typ *abi.MapType, hash uintptr, key unsafe.Pointer) unsafe.Pointer {
if t.growthLeft == 0 {
panic("invariant failed: growthLeft is unexpectedly 0")
}
// Given key and its hash hash(key), to insert it, we construct a
// probeSeq, and use it to find the first group with an unoccupied (empty
// or deleted) slot. We place the key into the first such slot in the
// group and mark it as full with key's H2. For indirect elements, we
// pre-allocate the element slot and return it to the caller.
seq := makeProbeSeq(h1(hash), t.groups.lengthMask)
for ; ; seq = seq.next() {
g := t.groups.group(typ, seq.offset)
match := g.ctrls().matchEmptyOrDeleted()
if match != 0 {
i := match.first()
slotKey := g.key(typ, i)
if typ.IndirectKey() {
kmem := newobject(typ.Key)
*(*unsafe.Pointer)(slotKey) = kmem
slotKey = kmem
}
typedmemmove(typ.Key, slotKey, key)
slotElem := g.elem(typ, i)
if typ.IndirectElem() {
emem := newobject(typ.Elem)
*(*unsafe.Pointer)(slotElem) = emem
slotElem = emem
}
t.growthLeft--
t.used++
g.ctrls().set(i, ctrl(h2(hash)))
return slotElem
}
}
}
// Delete returns true if it put a tombstone in t.
func (t *table) Delete(typ *abi.MapType, m *Map, hash uintptr, key unsafe.Pointer) bool {
seq := makeProbeSeq(h1(hash), t.groups.lengthMask)