diff --git a/src/runtime/_mkmalloc/mkmalloc.go b/src/runtime/_mkmalloc/mkmalloc.go index 46c50d66611..8032983da8c 100644 --- a/src/runtime/_mkmalloc/mkmalloc.go +++ b/src/runtime/_mkmalloc/mkmalloc.go @@ -107,6 +107,7 @@ type replacementKind int const ( inlineFunc = replacementKind(iota) subBasicLit + foldCondition ) // op is a single inlining operation for the inliner. Any calls to the function @@ -171,7 +172,7 @@ func specializedMallocConfig(classes []class, sizeToSizeClass []uint8) generator {subBasicLit, "elemsize_", str(elemsize)}, {subBasicLit, "sizeclass_", str(sc)}, {subBasicLit, "noscanint_", str(noscan)}, - {subBasicLit, "isTiny_", str(0)}, + {foldCondition, "isTiny_", str(false)}, }, }) } @@ -199,7 +200,7 @@ func specializedMallocConfig(classes []class, sizeToSizeClass []uint8) generator {subBasicLit, "sizeclass_", str(tinySizeClass)}, {subBasicLit, "size_", str(s)}, {subBasicLit, "noscanint_", str(noscan)}, - {subBasicLit, "isTiny_", str(1)}, + {foldCondition, "isTiny_", str(true)}, }, }) } @@ -217,7 +218,7 @@ func specializedMallocConfig(classes []class, sizeToSizeClass []uint8) generator {subBasicLit, "elemsize_", str(elemsize)}, {subBasicLit, "sizeclass_", str(sc)}, {subBasicLit, "noscanint_", str(noscan)}, - {subBasicLit, "isTiny_", str(0)}, + {foldCondition, "isTiny_", str(false)}, }, }) } @@ -277,10 +278,17 @@ func inline(config generatorConfig) []byte { // Apply each of the ops given by the specs stamped := ast.Node(containingFuncCopy) for _, repl := range spec.ops { - if toDecl, ok := funcDecls[repl.to]; ok { - stamped = inlineFunction(stamped, repl.from, toDecl) - } else { + switch repl.kind { + case inlineFunc: + if toDecl, ok := funcDecls[repl.to]; ok { + stamped = inlineFunction(stamped, repl.from, toDecl) + } + case subBasicLit: stamped = substituteWithBasicLit(stamped, repl.from, repl.to) + case foldCondition: + stamped = foldIfCondition(stamped, repl.from, repl.to) + default: + log.Fatal("unknown op kind %v", repl.kind) } } @@ -310,6 +318,43 @@ func substituteWithBasicLit(node ast.Node, from, to string) ast.Node { }, nil) } +// foldIfCondition looks for if statements with a single boolean variable from, or +// the negation of from and either replaces it with its body or nothing, +// depending on whether the to value is true or false. +func foldIfCondition(node ast.Node, from, to string) ast.Node { + var isTrue bool + switch to { + case "true": + isTrue = true + case "false": + isTrue = false + default: + log.Fatalf("op 'to' expr %q is not true or false", to) + } + return astutil.Apply(node, func(cursor *astutil.Cursor) bool { + var foldIfTrue bool + ifexpr, ok := cursor.Node().(*ast.IfStmt) + if !ok { + return true + } + if isIdentWithName(ifexpr.Cond, from) { + foldIfTrue = true + } else if unaryexpr, ok := ifexpr.Cond.(*ast.UnaryExpr); ok && unaryexpr.Op == token.NOT && isIdentWithName(unaryexpr.X, from) { + foldIfTrue = false + } else { + // not an if with from or !from. + return true + } + if foldIfTrue == isTrue { + for _, stmt := range ifexpr.Body.List { + cursor.InsertBefore(stmt) + } + } + cursor.Delete() + return true + }, nil) +} + // inlineFunction recursively replaces calls to the function 'from' with the body of the function // 'toDecl'. All calls to 'from' must appear in assignment statements. // The replacement is very simple: it doesn't substitute the arguments for the parameters, so the diff --git a/src/runtime/malloc_generated.go b/src/runtime/malloc_generated.go index 6864ca05d31..cf329d26969 100644 --- a/src/runtime/malloc_generated.go +++ b/src/runtime/malloc_generated.go @@ -12,12 +12,6 @@ import ( func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -159,6 +153,7 @@ func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsaf gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -182,12 +177,6 @@ func mallocgcSmallScanNoHeaderSC1(size uintptr, typ *_type, needzero bool) unsaf func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -329,6 +318,7 @@ func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsaf gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -352,12 +342,6 @@ func mallocgcSmallScanNoHeaderSC2(size uintptr, typ *_type, needzero bool) unsaf func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -499,6 +483,7 @@ func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsaf gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -522,12 +507,6 @@ func mallocgcSmallScanNoHeaderSC3(size uintptr, typ *_type, needzero bool) unsaf func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -669,6 +648,7 @@ func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsaf gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -692,12 +672,6 @@ func mallocgcSmallScanNoHeaderSC4(size uintptr, typ *_type, needzero bool) unsaf func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -839,6 +813,7 @@ func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsaf gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -862,12 +837,6 @@ func mallocgcSmallScanNoHeaderSC5(size uintptr, typ *_type, needzero bool) unsaf func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -1009,6 +978,7 @@ func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsaf gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -1032,12 +1002,6 @@ func mallocgcSmallScanNoHeaderSC6(size uintptr, typ *_type, needzero bool) unsaf func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -1179,6 +1143,7 @@ func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsaf gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -1202,12 +1167,6 @@ func mallocgcSmallScanNoHeaderSC7(size uintptr, typ *_type, needzero bool) unsaf func mallocgcSmallScanNoHeaderSC8(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -1349,6 +1308,7 @@ func mallocgcSmallScanNoHeaderSC8(size uintptr, typ *_type, needzero bool) unsaf gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -1372,12 +1332,6 @@ func mallocgcSmallScanNoHeaderSC8(size uintptr, typ *_type, needzero bool) unsaf func mallocgcSmallScanNoHeaderSC9(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -1519,6 +1473,7 @@ func mallocgcSmallScanNoHeaderSC9(size uintptr, typ *_type, needzero bool) unsaf gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -1542,12 +1497,6 @@ func mallocgcSmallScanNoHeaderSC9(size uintptr, typ *_type, needzero bool) unsaf func mallocgcSmallScanNoHeaderSC10(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -1689,6 +1638,7 @@ func mallocgcSmallScanNoHeaderSC10(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -1712,12 +1662,6 @@ func mallocgcSmallScanNoHeaderSC10(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC11(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -1859,6 +1803,7 @@ func mallocgcSmallScanNoHeaderSC11(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -1882,12 +1827,6 @@ func mallocgcSmallScanNoHeaderSC11(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC12(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -2029,6 +1968,7 @@ func mallocgcSmallScanNoHeaderSC12(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -2052,12 +1992,6 @@ func mallocgcSmallScanNoHeaderSC12(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC13(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -2199,6 +2133,7 @@ func mallocgcSmallScanNoHeaderSC13(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -2222,12 +2157,6 @@ func mallocgcSmallScanNoHeaderSC13(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC14(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -2369,6 +2298,7 @@ func mallocgcSmallScanNoHeaderSC14(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -2392,12 +2322,6 @@ func mallocgcSmallScanNoHeaderSC14(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC15(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -2539,6 +2463,7 @@ func mallocgcSmallScanNoHeaderSC15(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -2562,12 +2487,6 @@ func mallocgcSmallScanNoHeaderSC15(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC16(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -2709,6 +2628,7 @@ func mallocgcSmallScanNoHeaderSC16(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -2732,12 +2652,6 @@ func mallocgcSmallScanNoHeaderSC16(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC17(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -2879,6 +2793,7 @@ func mallocgcSmallScanNoHeaderSC17(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -2902,12 +2817,6 @@ func mallocgcSmallScanNoHeaderSC17(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC18(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -3049,6 +2958,7 @@ func mallocgcSmallScanNoHeaderSC18(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -3072,12 +2982,6 @@ func mallocgcSmallScanNoHeaderSC18(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC19(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -3219,6 +3123,7 @@ func mallocgcSmallScanNoHeaderSC19(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -3242,12 +3147,6 @@ func mallocgcSmallScanNoHeaderSC19(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC20(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -3389,6 +3288,7 @@ func mallocgcSmallScanNoHeaderSC20(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -3412,12 +3312,6 @@ func mallocgcSmallScanNoHeaderSC20(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC21(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -3559,6 +3453,7 @@ func mallocgcSmallScanNoHeaderSC21(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -3582,12 +3477,6 @@ func mallocgcSmallScanNoHeaderSC21(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC22(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -3729,6 +3618,7 @@ func mallocgcSmallScanNoHeaderSC22(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -3752,12 +3642,6 @@ func mallocgcSmallScanNoHeaderSC22(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC23(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -3899,6 +3783,7 @@ func mallocgcSmallScanNoHeaderSC23(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -3922,12 +3807,6 @@ func mallocgcSmallScanNoHeaderSC23(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC24(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -4069,6 +3948,7 @@ func mallocgcSmallScanNoHeaderSC24(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -4092,12 +3972,6 @@ func mallocgcSmallScanNoHeaderSC24(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC25(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -4239,6 +4113,7 @@ func mallocgcSmallScanNoHeaderSC25(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -4262,12 +4137,6 @@ func mallocgcSmallScanNoHeaderSC25(size uintptr, typ *_type, needzero bool) unsa func mallocgcSmallScanNoHeaderSC26(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -4409,6 +4278,7 @@ func mallocgcSmallScanNoHeaderSC26(size uintptr, typ *_type, needzero bool) unsa gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -4432,12 +4302,11 @@ func mallocgcSmallScanNoHeaderSC26(size uintptr, typ *_type, needzero bool) unsa func mallocTiny1(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -4489,11 +4358,6 @@ func mallocTiny1(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -4576,11 +4440,6 @@ func mallocTiny1(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -4599,12 +4458,11 @@ func mallocTiny1(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocTiny2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -4656,11 +4514,6 @@ func mallocTiny2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -4743,11 +4596,6 @@ func mallocTiny2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -4766,12 +4614,11 @@ func mallocTiny2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocTiny3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -4823,11 +4670,6 @@ func mallocTiny3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -4910,11 +4752,6 @@ func mallocTiny3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -4933,12 +4770,11 @@ func mallocTiny3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocTiny4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -4990,11 +4826,6 @@ func mallocTiny4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -5077,11 +4908,6 @@ func mallocTiny4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -5100,12 +4926,11 @@ func mallocTiny4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocTiny5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -5157,11 +4982,6 @@ func mallocTiny5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -5244,11 +5064,6 @@ func mallocTiny5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -5267,12 +5082,11 @@ func mallocTiny5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocTiny6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -5324,11 +5138,6 @@ func mallocTiny6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -5411,11 +5220,6 @@ func mallocTiny6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -5434,12 +5238,11 @@ func mallocTiny6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocTiny7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -5491,11 +5294,6 @@ func mallocTiny7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -5578,11 +5376,6 @@ func mallocTiny7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -5601,12 +5394,11 @@ func mallocTiny7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocTiny8(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -5658,11 +5450,6 @@ func mallocTiny8(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -5745,11 +5532,6 @@ func mallocTiny8(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -5768,12 +5550,11 @@ func mallocTiny8(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocTiny9(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -5825,11 +5606,6 @@ func mallocTiny9(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -5912,11 +5688,6 @@ func mallocTiny9(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -5935,12 +5706,11 @@ func mallocTiny9(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocTiny10(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -5992,11 +5762,6 @@ func mallocTiny10(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -6079,11 +5844,6 @@ func mallocTiny10(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -6102,12 +5862,11 @@ func mallocTiny10(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocTiny11(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -6159,11 +5918,6 @@ func mallocTiny11(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -6246,11 +6000,6 @@ func mallocTiny11(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -6269,12 +6018,11 @@ func mallocTiny11(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocTiny12(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -6326,11 +6074,6 @@ func mallocTiny12(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -6413,11 +6156,6 @@ func mallocTiny12(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -6436,12 +6174,11 @@ func mallocTiny12(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocTiny13(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -6493,11 +6230,6 @@ func mallocTiny13(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -6580,11 +6312,6 @@ func mallocTiny13(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -6603,12 +6330,11 @@ func mallocTiny13(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocTiny14(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -6660,11 +6386,6 @@ func mallocTiny14(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -6747,11 +6468,6 @@ func mallocTiny14(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -6770,12 +6486,11 @@ func mallocTiny14(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocTiny15(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 1 == - 1 gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { + if goexperiment.RuntimeSecret && gp.secret > 0 { return mallocgcSmallNoScanSC2(size, typ, needzero) } + if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -6827,11 +6542,6 @@ func mallocTiny15(size uintptr, typ *_type, needzero bool) unsafe.Pointer { const elemsize = 0 { - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -6914,11 +6624,6 @@ func mallocTiny15(size uintptr, typ *_type, needzero bool) unsafe.Pointer { x = add(x, elemsize-constsize) } - if goexperiment.RuntimeSecret && gp.secret > 0 { - - addSecret(x) - } - if valgrindenabled { valgrindMalloc(x, size) } @@ -6937,12 +6642,6 @@ func mallocTiny15(size uintptr, typ *_type, needzero bool) unsafe.Pointer { func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -6984,6 +6683,7 @@ func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Poin x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -7054,6 +6754,7 @@ func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Poin gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -7077,12 +6778,6 @@ func mallocgcSmallNoScanSC2(size uintptr, typ *_type, needzero bool) unsafe.Poin func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -7124,6 +6819,7 @@ func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Poin x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -7194,6 +6890,7 @@ func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Poin gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -7217,12 +6914,6 @@ func mallocgcSmallNoScanSC3(size uintptr, typ *_type, needzero bool) unsafe.Poin func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -7264,6 +6955,7 @@ func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Poin x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -7334,6 +7026,7 @@ func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Poin gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -7357,12 +7050,6 @@ func mallocgcSmallNoScanSC4(size uintptr, typ *_type, needzero bool) unsafe.Poin func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -7404,6 +7091,7 @@ func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Poin x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -7474,6 +7162,7 @@ func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Poin gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -7497,12 +7186,6 @@ func mallocgcSmallNoScanSC5(size uintptr, typ *_type, needzero bool) unsafe.Poin func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -7544,6 +7227,7 @@ func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Poin x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -7614,6 +7298,7 @@ func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Poin gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -7637,12 +7322,6 @@ func mallocgcSmallNoScanSC6(size uintptr, typ *_type, needzero bool) unsafe.Poin func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -7684,6 +7363,7 @@ func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Poin x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -7754,6 +7434,7 @@ func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Poin gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -7777,12 +7458,6 @@ func mallocgcSmallNoScanSC7(size uintptr, typ *_type, needzero bool) unsafe.Poin func mallocgcSmallNoScanSC8(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -7824,6 +7499,7 @@ func mallocgcSmallNoScanSC8(size uintptr, typ *_type, needzero bool) unsafe.Poin x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -7894,6 +7570,7 @@ func mallocgcSmallNoScanSC8(size uintptr, typ *_type, needzero bool) unsafe.Poin gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -7917,12 +7594,6 @@ func mallocgcSmallNoScanSC8(size uintptr, typ *_type, needzero bool) unsafe.Poin func mallocgcSmallNoScanSC9(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -7964,6 +7635,7 @@ func mallocgcSmallNoScanSC9(size uintptr, typ *_type, needzero bool) unsafe.Poin x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -8034,6 +7706,7 @@ func mallocgcSmallNoScanSC9(size uintptr, typ *_type, needzero bool) unsafe.Poin gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -8057,12 +7730,6 @@ func mallocgcSmallNoScanSC9(size uintptr, typ *_type, needzero bool) unsafe.Poin func mallocgcSmallNoScanSC10(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -8104,6 +7771,7 @@ func mallocgcSmallNoScanSC10(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -8174,6 +7842,7 @@ func mallocgcSmallNoScanSC10(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -8197,12 +7866,6 @@ func mallocgcSmallNoScanSC10(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC11(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -8244,6 +7907,7 @@ func mallocgcSmallNoScanSC11(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -8314,6 +7978,7 @@ func mallocgcSmallNoScanSC11(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -8337,12 +8002,6 @@ func mallocgcSmallNoScanSC11(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC12(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -8384,6 +8043,7 @@ func mallocgcSmallNoScanSC12(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -8454,6 +8114,7 @@ func mallocgcSmallNoScanSC12(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -8477,12 +8138,6 @@ func mallocgcSmallNoScanSC12(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC13(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -8524,6 +8179,7 @@ func mallocgcSmallNoScanSC13(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -8594,6 +8250,7 @@ func mallocgcSmallNoScanSC13(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -8617,12 +8274,6 @@ func mallocgcSmallNoScanSC13(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC14(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -8664,6 +8315,7 @@ func mallocgcSmallNoScanSC14(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -8734,6 +8386,7 @@ func mallocgcSmallNoScanSC14(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -8757,12 +8410,6 @@ func mallocgcSmallNoScanSC14(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC15(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -8804,6 +8451,7 @@ func mallocgcSmallNoScanSC15(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -8874,6 +8522,7 @@ func mallocgcSmallNoScanSC15(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -8897,12 +8546,6 @@ func mallocgcSmallNoScanSC15(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC16(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -8944,6 +8587,7 @@ func mallocgcSmallNoScanSC16(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9014,6 +8658,7 @@ func mallocgcSmallNoScanSC16(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9037,12 +8682,6 @@ func mallocgcSmallNoScanSC16(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC17(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -9084,6 +8723,7 @@ func mallocgcSmallNoScanSC17(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9154,6 +8794,7 @@ func mallocgcSmallNoScanSC17(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9177,12 +8818,6 @@ func mallocgcSmallNoScanSC17(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC18(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -9224,6 +8859,7 @@ func mallocgcSmallNoScanSC18(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9294,6 +8930,7 @@ func mallocgcSmallNoScanSC18(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9317,12 +8954,6 @@ func mallocgcSmallNoScanSC18(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC19(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -9364,6 +8995,7 @@ func mallocgcSmallNoScanSC19(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9434,6 +9066,7 @@ func mallocgcSmallNoScanSC19(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9457,12 +9090,6 @@ func mallocgcSmallNoScanSC19(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC20(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -9504,6 +9131,7 @@ func mallocgcSmallNoScanSC20(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9574,6 +9202,7 @@ func mallocgcSmallNoScanSC20(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9597,12 +9226,6 @@ func mallocgcSmallNoScanSC20(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC21(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -9644,6 +9267,7 @@ func mallocgcSmallNoScanSC21(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9714,6 +9338,7 @@ func mallocgcSmallNoScanSC21(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9737,12 +9362,6 @@ func mallocgcSmallNoScanSC21(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC22(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -9784,6 +9403,7 @@ func mallocgcSmallNoScanSC22(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9854,6 +9474,7 @@ func mallocgcSmallNoScanSC22(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9877,12 +9498,6 @@ func mallocgcSmallNoScanSC22(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC23(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -9924,6 +9539,7 @@ func mallocgcSmallNoScanSC23(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -9994,6 +9610,7 @@ func mallocgcSmallNoScanSC23(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -10017,12 +9634,6 @@ func mallocgcSmallNoScanSC23(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC24(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -10064,6 +9675,7 @@ func mallocgcSmallNoScanSC24(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -10134,6 +9746,7 @@ func mallocgcSmallNoScanSC24(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -10157,12 +9770,6 @@ func mallocgcSmallNoScanSC24(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC25(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -10204,6 +9811,7 @@ func mallocgcSmallNoScanSC25(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -10274,6 +9882,7 @@ func mallocgcSmallNoScanSC25(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -10297,12 +9906,6 @@ func mallocgcSmallNoScanSC25(size uintptr, typ *_type, needzero bool) unsafe.Poi func mallocgcSmallNoScanSC26(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - const isTiny = 0 == - 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) - } if doubleCheckMalloc { if gcphase == _GCmarktermination { throw("mallocgc called with gcphase == _GCmarktermination") @@ -10344,6 +9947,7 @@ func mallocgcSmallNoScanSC26(size uintptr, typ *_type, needzero bool) unsafe.Poi x := v { + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) @@ -10414,6 +10018,7 @@ func mallocgcSmallNoScanSC26(size uintptr, typ *_type, needzero bool) unsafe.Poi gcStart(t) } } + gp := getg() if goexperiment.RuntimeSecret && gp.secret > 0 { addSecret(x) diff --git a/src/runtime/malloc_stubs.go b/src/runtime/malloc_stubs.go index 58ca1d5f79f..8c424935bf2 100644 --- a/src/runtime/malloc_stubs.go +++ b/src/runtime/malloc_stubs.go @@ -37,7 +37,7 @@ const elemsize_ = 8 const sizeclass_ = 0 const noscanint_ = 0 const size_ = 0 -const isTiny_ = 0 +const isTiny_ = false func malloc0(size uintptr, typ *_type, needzero bool) unsafe.Pointer { if doubleCheckMalloc { @@ -58,15 +58,16 @@ func mallocPanic(size uintptr, typ *_type, needzero bool) unsafe.Pointer { // to steer out of this codepath early if sanitizers are enabled. func mallocStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer { - // secret code, need to avoid the tiny allocator since it might keep - // co-located values alive longer and prevent timely zero-ing - // - // Call directly into the NoScan allocator. - // See go.dev/issue/76356 - const isTiny = isTiny_ == 1 - gp := getg() - if goexperiment.RuntimeSecret && isTiny && gp.secret > 0 { - return mallocgcSmallNoScanSC2(size, typ, needzero) + if isTiny_ { + // secret code, need to avoid the tiny allocator since it might keep + // co-located values alive longer and prevent timely zero-ing + // + // Call directly into the NoScan allocator. + // See go.dev/issue/76356 + gp := getg() + if goexperiment.RuntimeSecret && gp.secret > 0 { + return mallocgcSmallNoScanSC2(size, typ, needzero) + } } if doubleCheckMalloc { if gcphase == _GCmarktermination { @@ -95,10 +96,13 @@ func mallocStub(size uintptr, typ *_type, needzero bool) unsafe.Pointer { // Actually do the allocation. x, elemsize := inlinedMalloc(size, typ, needzero) - if goexperiment.RuntimeSecret && gp.secret > 0 { - // Mark any object allocated while in secret mode as secret. - // This ensures we zero it immediately when freeing it. - addSecret(x) + if !isTiny_ { + gp := getg() + if goexperiment.RuntimeSecret && gp.secret > 0 { + // Mark any object allocated while in secret mode as secret. + // This ensures we zero it immediately when freeing it. + addSecret(x) + } } // Notify valgrind, if enabled.