mirror of
https://github.com/golang/go.git
synced 2026-06-27 19:30:52 +00:00
runtime: move post allocation work into postMallocgc
This makes it a little clearer that the work that happens after the allocation is duplicated in some cases. While it does produce some changes in the generated code, it should not affect the behavior or performance of the generated code. We should be able to remove the logic that checks for a terminating and non terminating return also after this. Change-Id: If01d45c08c88f9a8f1a9b0baa8c447ac6a6a6964 Reviewed-on: https://go-review.googlesource.com/c/go/+/779020 Reviewed-by: Michael Matloob <matloob@google.com> Reviewed-by: Michael Pratt <mpratt@google.com> Auto-Submit: Michael Matloob <matloob@golang.org> LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
This commit is contained in:
Michael Matloob
Gopher Robot
parent
c7a107bfbf
commit
15fd4ff942
3 changed files with 225 additions and 193 deletions
|
|
@ -167,6 +167,7 @@ func specializedMallocConfig(classes []class, sizeToSizeClass []uint8) generator
|
|||
name: name,
|
||||
ops: []op{
|
||||
{inlineFunc, "inlinedMalloc", "smallStub"},
|
||||
{inlineFunc, "postMallocgc", "postMallocgc"},
|
||||
{foldCondition, "isNoScan_", str(false)},
|
||||
{inlineFunc, "heapSetTypeNoHeaderStub", "heapSetTypeNoHeaderStub"},
|
||||
{inlineFunc, "nextFreeFastStub", "nextFreeFastStub"},
|
||||
|
|
@ -195,6 +196,8 @@ func specializedMallocConfig(classes []class, sizeToSizeClass []uint8) generator
|
|||
ops: []op{
|
||||
{inlineFunc, "inlinedMalloc", "tinyStub"},
|
||||
{inlineFunc, "nextFreeFastTiny", "nextFreeFastTiny"},
|
||||
{inlineFunc, "postMallocgc", "postMallocgc"},
|
||||
{inlineFunc, "nextFreeFastStub", "nextFreeFastStub"},
|
||||
{inlineFunc, "deductAssistCredit", "deductAssistCredit"},
|
||||
{subBasicLit, "elemsize_", str(elemsize)},
|
||||
{subBasicLit, "sizeclass_", str(tinySizeClass)},
|
||||
|
|
@ -213,6 +216,7 @@ func specializedMallocConfig(classes []class, sizeToSizeClass []uint8) generator
|
|||
name: name,
|
||||
ops: []op{
|
||||
{inlineFunc, "inlinedMalloc", "smallStub"},
|
||||
{inlineFunc, "postMallocgc", "postMallocgc"},
|
||||
{foldCondition, "isNoScan_", str(true)},
|
||||
{inlineFunc, "nextFreeFastStub", "nextFreeFastStub"},
|
||||
{subBasicLit, "elemsize_", str(elemsize)},
|
||||
|
|
@ -389,6 +393,23 @@ func inlineFunction(node ast.Node, from string, toDecl *ast.FuncDecl) ast.Node {
|
|||
replaceCallExprStmt(cursor, toDecl)
|
||||
}
|
||||
return false
|
||||
case *ast.ReturnStmt:
|
||||
if len(node.Results) == 1 && isCallTo(node.Results[0], from) {
|
||||
args := node.Results[0].(*ast.CallExpr).Args
|
||||
if !argsMatchParameters(args, toDecl.Type.Params) {
|
||||
log.Fatalf("applying op: arguments to %v don't match parameter names of %v: %v", from, toDecl.Name, debugPrint(args...))
|
||||
}
|
||||
replaceTailCall(cursor, toDecl)
|
||||
}
|
||||
return false
|
||||
case *ast.CallExpr:
|
||||
if isCallTo(node, from) {
|
||||
switch cursor.Parent().(type) {
|
||||
case *ast.AssignStmt, *ast.ExprStmt:
|
||||
default:
|
||||
log.Fatalf("applying op: all calls to function %q being replaced must appear in an assignment or expression statement, appears in %T", from, cursor.Parent())
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}, nil)
|
||||
|
|
@ -443,6 +464,27 @@ func replaceCallExprStmt(cursor *astutil.Cursor, funcdecl *ast.FuncDecl) {
|
|||
cursor.Delete()
|
||||
}
|
||||
|
||||
func replaceTailCall(cursor *astutil.Cursor, funcdecl *ast.FuncDecl) {
|
||||
if !hasTerminatingReturn(funcdecl.Body) {
|
||||
log.Fatal("function being inlined must have a return at the end")
|
||||
}
|
||||
|
||||
body := internalastutil.CloneNode(funcdecl.Body)
|
||||
if len(body.List) < 1 {
|
||||
log.Fatal("replacing with empty bodied function")
|
||||
}
|
||||
|
||||
// The op happens in two steps: first we insert the body of the function being inlined (except for
|
||||
// the final return) before the assignment, and then we change the assignment statement to replace the function call
|
||||
// with the expressions being returned.
|
||||
|
||||
// Insert the body up to the final return.
|
||||
for _, stmt := range body.List {
|
||||
cursor.InsertBefore(stmt)
|
||||
}
|
||||
cursor.Delete()
|
||||
}
|
||||
|
||||
// replaceAssignment replaces an assignment statement where the right hand side is a function call
|
||||
// whose arguments have the same names as the parameters to funcdecl with the body of funcdecl.
|
||||
// It sets the left hand side of the assignment to the return values of the function.
|
||||
|
|
|
|||
|
|
@ -171,6 +171,7 @@ func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsaf
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -335,6 +336,7 @@ func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsaf
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -499,6 +501,7 @@ func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsaf
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -663,6 +666,7 @@ func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsaf
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -827,6 +831,7 @@ func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsaf
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -991,6 +996,7 @@ func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsaf
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -1155,6 +1161,7 @@ func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsaf
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -1319,6 +1326,7 @@ func mallocgcSmallScanNoHeaderSC8(size uintptr, typ *_type, needzero bool) unsaf
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -1483,6 +1491,7 @@ func mallocgcSmallScanNoHeaderSC9(size uintptr, typ *_type, needzero bool) unsaf
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -1647,6 +1656,7 @@ func mallocgcSmallScanNoHeaderSC10(size uintptr, typ *_type, needzero bool) unsa
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -1712,20 +1722,17 @@ func mallocgcTinySC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
|
|||
mp.mallocing = 0
|
||||
releasem(mp)
|
||||
const elemsize = 0
|
||||
{
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
checkGCTrigger := false
|
||||
|
|
@ -1792,6 +1799,7 @@ func mallocgcTinySC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -1833,30 +1841,26 @@ func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
|
||||
if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
|
||||
|
||||
v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
mp.mallocing = 0
|
||||
releasem(mp)
|
||||
x := v
|
||||
{
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
return x
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
var nextFreeFastResult gclinkptr
|
||||
|
|
@ -1920,6 +1924,7 @@ func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -1961,30 +1966,26 @@ func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
|
||||
if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
|
||||
|
||||
v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
mp.mallocing = 0
|
||||
releasem(mp)
|
||||
x := v
|
||||
{
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
return x
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
var nextFreeFastResult gclinkptr
|
||||
|
|
@ -2048,6 +2049,7 @@ func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -2089,30 +2091,26 @@ func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
|
||||
if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
|
||||
|
||||
v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
mp.mallocing = 0
|
||||
releasem(mp)
|
||||
x := v
|
||||
{
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
return x
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
var nextFreeFastResult gclinkptr
|
||||
|
|
@ -2176,6 +2174,7 @@ func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -2217,30 +2216,26 @@ func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
|
||||
if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
|
||||
|
||||
v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
mp.mallocing = 0
|
||||
releasem(mp)
|
||||
x := v
|
||||
{
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
return x
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
var nextFreeFastResult gclinkptr
|
||||
|
|
@ -2304,6 +2299,7 @@ func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -2345,30 +2341,26 @@ func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
|
||||
if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
|
||||
|
||||
v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
mp.mallocing = 0
|
||||
releasem(mp)
|
||||
x := v
|
||||
{
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
return x
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
var nextFreeFastResult gclinkptr
|
||||
|
|
@ -2432,6 +2424,7 @@ func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -2473,30 +2466,26 @@ func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
|
||||
if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
|
||||
|
||||
v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
mp.mallocing = 0
|
||||
releasem(mp)
|
||||
x := v
|
||||
{
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
return x
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
var nextFreeFastResult gclinkptr
|
||||
|
|
@ -2560,6 +2549,7 @@ func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -2601,30 +2591,26 @@ func mallocgcSmallNoScanSC8(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
|
||||
if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
|
||||
|
||||
v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
mp.mallocing = 0
|
||||
releasem(mp)
|
||||
x := v
|
||||
{
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
return x
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
var nextFreeFastResult gclinkptr
|
||||
|
|
@ -2688,6 +2674,7 @@ func mallocgcSmallNoScanSC8(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -2729,30 +2716,26 @@ func mallocgcSmallNoScanSC9(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
|
||||
if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
|
||||
|
||||
v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
mp.mallocing = 0
|
||||
releasem(mp)
|
||||
x := v
|
||||
{
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
return x
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
var nextFreeFastResult gclinkptr
|
||||
|
|
@ -2816,6 +2799,7 @@ func mallocgcSmallNoScanSC9(size uintptr, typ *_type, needzero bool) unsafe.Poin
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
|
|
@ -2857,30 +2841,26 @@ func mallocgcSmallNoScanSC10(size uintptr, typ *_type, needzero bool) unsafe.Poi
|
|||
|
||||
if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
|
||||
|
||||
v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
mp.mallocing = 0
|
||||
releasem(mp)
|
||||
x := v
|
||||
{
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
return x
|
||||
addSecret(x, size)
|
||||
}
|
||||
|
||||
if gcBlackenEnabled != 0 && elemsize != 0 {
|
||||
if assistG := getg().m.curg; assistG != nil {
|
||||
assistG.gcAssistBytes -= int64(elemsize - size)
|
||||
}
|
||||
}
|
||||
|
||||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
var nextFreeFastResult gclinkptr
|
||||
|
|
@ -2944,5 +2924,6 @@ func mallocgcSmallNoScanSC10(size uintptr, typ *_type, needzero bool) unsafe.Poi
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
|
||||
return x
|
||||
}
|
||||
|
|
|
|||
|
|
@ -95,8 +95,10 @@ func mallocStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
|
|||
}
|
||||
|
||||
// Actually do the allocation.
|
||||
x, elemsize := inlinedMalloc(size, typ, needzero)
|
||||
return inlinedMalloc(size, typ, needzero)
|
||||
}
|
||||
|
||||
func postMallocgc(x unsafe.Pointer, typ *_type, size uintptr, elemsize uintptr) {
|
||||
if !isTiny_ {
|
||||
gp := getg()
|
||||
if goexperiment.RuntimeSecret && gp.secret > 0 {
|
||||
|
|
@ -117,7 +119,6 @@ func mallocStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
|
|||
if debug.malloc {
|
||||
postMallocgcDebug(x, elemsize, typ)
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
// deductAssistCredit reduces the current G's GC assist credit
|
||||
|
|
@ -143,8 +144,8 @@ func deductAssistCredit(size uintptr) {
|
|||
// will be replaced with the inlined body of smallStub or tinyStub when generating the
|
||||
// size-specialized malloc function. See the comment at the top of this file for more
|
||||
// information.
|
||||
func inlinedMalloc(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) {
|
||||
return unsafe.Pointer(uintptr(0)), 0
|
||||
func inlinedMalloc(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
|
||||
return unsafe.Pointer(uintptr(0))
|
||||
}
|
||||
|
||||
func doubleCheckSmallScanNoHeader(size uintptr, typ *_type, mp *m) {
|
||||
|
|
@ -162,7 +163,7 @@ func doubleCheckSmallScanNoHeader(size uintptr, typ *_type, mp *m) {
|
|||
}
|
||||
}
|
||||
|
||||
func smallStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) {
|
||||
func smallStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
|
||||
const sizeclass = sizeclass_
|
||||
const elemsize = elemsize_
|
||||
|
||||
|
|
@ -187,7 +188,7 @@ func smallStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr
|
|||
// First, check for a reusable object.
|
||||
if runtimeFreegcEnabled && c.hasReusableNoscan(spc) {
|
||||
// We have a reusable object, use it.
|
||||
v := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
x := mallocgcSmallNoscanReuse(c, span, spc, elemsize, needzero)
|
||||
mp.mallocing = 0
|
||||
releasem(mp)
|
||||
|
||||
|
|
@ -197,7 +198,9 @@ func smallStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr
|
|||
// mallocgcSmallNoScanSC2 function. One set of those correspond to this
|
||||
// return here. We might be able to de-duplicate the generated return path
|
||||
// by updating the generator, perhaps by jumping to a shared return or similar.
|
||||
return v, elemsize
|
||||
postMallocgc(x, typ, size, elemsize)
|
||||
|
||||
return x
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -275,7 +278,9 @@ func smallStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr
|
|||
}
|
||||
}
|
||||
|
||||
return x, elemsize
|
||||
postMallocgc(x, typ, size, elemsize)
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
func doubleCheckSmallNoScan(typ *_type, mp *m) {
|
||||
|
|
@ -302,7 +307,7 @@ func doubleCheckTiny(size uintptr, typ *_type, mp *m) {
|
|||
}
|
||||
}
|
||||
|
||||
func tinyStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr) {
|
||||
func tinyStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer {
|
||||
const elemsize = elemsize_
|
||||
|
||||
// Set mp.mallocing to keep from being preempted by GC.
|
||||
|
|
@ -366,7 +371,9 @@ func tinyStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr)
|
|||
c.tinyAllocs++
|
||||
mp.mallocing = 0
|
||||
releasem(mp)
|
||||
return x, 0
|
||||
const elemsize = 0
|
||||
postMallocgc(x, typ, size, elemsize)
|
||||
return x
|
||||
}
|
||||
// Allocate a new maxTinySize block.
|
||||
checkGCTrigger := false
|
||||
|
|
@ -434,7 +441,9 @@ func tinyStub(size uintptr, typ *_type, needzero bool) (unsafe.Pointer, uintptr)
|
|||
}
|
||||
}
|
||||
|
||||
return x, elemsize
|
||||
postMallocgc(x, typ, size, elemsize)
|
||||
|
||||
return x
|
||||
}
|
||||
|
||||
// TODO(matloob): Should we let the go compiler inline this instead of using mkmalloc?
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue