2016-04-11 17:35:55 +03:00
|
|
|
// Copyright 2013 The Go Authors. All rights reserved.
|
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
|
|
package obj
|
|
|
|
|
|
|
|
|
|
import (
|
2017-04-18 12:53:25 -07:00
|
|
|
"cmd/internal/objabi"
|
2016-04-11 17:35:55 +03:00
|
|
|
"fmt"
|
|
|
|
|
"strings"
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
type Plist struct {
|
|
|
|
|
Firstpc *Prog
|
2017-03-23 16:39:04 -07:00
|
|
|
Curfn interface{} // holds a *gc.Node, if non-nil
|
2016-04-11 17:35:55 +03:00
|
|
|
}
|
|
|
|
|
|
2017-04-04 14:31:55 -07:00
|
|
|
// ProgAlloc is a function that allocates Progs.
|
|
|
|
|
// It is used to provide access to cached/bulk-allocated Progs to the assemblers.
|
|
|
|
|
type ProgAlloc func() *Prog
|
|
|
|
|
|
2017-10-06 11:32:28 -04:00
|
|
|
func Flushplist(ctxt *Link, plist *Plist, newprog ProgAlloc, myimportpath string) {
|
2016-04-11 17:35:55 +03:00
|
|
|
// Build list of symbols, and assign instructions to lists.
|
|
|
|
|
var curtext *LSym
|
|
|
|
|
var etext *Prog
|
|
|
|
|
var text []*LSym
|
|
|
|
|
|
2017-02-17 16:52:16 -05:00
|
|
|
var plink *Prog
|
|
|
|
|
for p := plist.Firstpc; p != nil; p = plink {
|
2018-12-07 10:00:36 -08:00
|
|
|
if ctxt.Debugasm > 0 && ctxt.Debugvlog {
|
2017-02-17 16:52:16 -05:00
|
|
|
fmt.Printf("obj: %v\n", p)
|
|
|
|
|
}
|
|
|
|
|
plink = p.Link
|
|
|
|
|
p.Link = nil
|
2016-04-11 17:35:55 +03:00
|
|
|
|
2017-02-17 16:52:16 -05:00
|
|
|
switch p.As {
|
|
|
|
|
case AEND:
|
|
|
|
|
continue
|
2016-04-11 17:35:55 +03:00
|
|
|
|
2017-02-17 16:52:16 -05:00
|
|
|
case ATEXT:
|
|
|
|
|
s := p.From.Sym
|
|
|
|
|
if s == nil {
|
|
|
|
|
// func _() { }
|
|
|
|
|
curtext = nil
|
2016-04-11 17:35:55 +03:00
|
|
|
continue
|
2017-02-17 16:52:16 -05:00
|
|
|
}
|
|
|
|
|
text = append(text, s)
|
|
|
|
|
etext = p
|
|
|
|
|
curtext = s
|
|
|
|
|
continue
|
2016-04-11 17:35:55 +03:00
|
|
|
|
2017-02-17 16:52:16 -05:00
|
|
|
case AFUNCDATA:
|
|
|
|
|
// Rewrite reference to go_args_stackmap(SB) to the Go-provided declaration information.
|
|
|
|
|
if curtext == nil { // func _() {}
|
2016-04-11 17:35:55 +03:00
|
|
|
continue
|
|
|
|
|
}
|
2017-02-17 16:52:16 -05:00
|
|
|
if p.To.Sym.Name == "go_args_stackmap" {
|
2017-04-18 12:53:25 -07:00
|
|
|
if p.From.Type != TYPE_CONST || p.From.Offset != objabi.FUNCDATA_ArgsPointerMaps {
|
2017-02-17 16:52:16 -05:00
|
|
|
ctxt.Diag("FUNCDATA use of go_args_stackmap(SB) without FUNCDATA_ArgsPointerMaps")
|
|
|
|
|
}
|
2017-04-20 07:13:02 -07:00
|
|
|
p.To.Sym = ctxt.LookupDerived(curtext, curtext.Name+".args_stackmap")
|
2017-02-17 16:52:16 -05:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if curtext == nil {
|
|
|
|
|
etext = nil
|
|
|
|
|
continue
|
2016-04-11 17:35:55 +03:00
|
|
|
}
|
2017-02-17 16:52:16 -05:00
|
|
|
etext.Link = p
|
|
|
|
|
etext = p
|
2016-04-11 17:35:55 +03:00
|
|
|
}
|
|
|
|
|
|
2017-04-05 07:05:35 -07:00
|
|
|
if newprog == nil {
|
|
|
|
|
newprog = ctxt.NewProg
|
|
|
|
|
}
|
2017-04-04 14:31:55 -07:00
|
|
|
|
2021-04-30 19:45:11 -04:00
|
|
|
// Add reference to Go arguments for assembly functions without them.
|
|
|
|
|
if ctxt.IsAsm {
|
|
|
|
|
for _, s := range text {
|
|
|
|
|
if !strings.HasPrefix(s.Name, "\"\".") {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
// The current args_stackmap generation in the compiler assumes
|
|
|
|
|
// that the function in question is ABI0, so avoid introducing
|
|
|
|
|
// an args_stackmap reference if the func is not ABI0 (better to
|
|
|
|
|
// have no stackmap than an incorrect/lying stackmap).
|
|
|
|
|
if s.ABI() != ABI0 {
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
foundArgMap, foundArgInfo := false, false
|
|
|
|
|
for p := s.Func().Text; p != nil; p = p.Link {
|
|
|
|
|
if p.As == AFUNCDATA && p.From.Type == TYPE_CONST {
|
|
|
|
|
if p.From.Offset == objabi.FUNCDATA_ArgsPointerMaps {
|
|
|
|
|
foundArgMap = true
|
|
|
|
|
}
|
|
|
|
|
if p.From.Offset == objabi.FUNCDATA_ArgInfo {
|
|
|
|
|
foundArgInfo = true
|
|
|
|
|
}
|
|
|
|
|
if foundArgMap && foundArgInfo {
|
|
|
|
|
break
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if !foundArgMap {
|
|
|
|
|
p := Appendp(s.Func().Text, newprog)
|
|
|
|
|
p.As = AFUNCDATA
|
|
|
|
|
p.From.Type = TYPE_CONST
|
|
|
|
|
p.From.Offset = objabi.FUNCDATA_ArgsPointerMaps
|
|
|
|
|
p.To.Type = TYPE_MEM
|
|
|
|
|
p.To.Name = NAME_EXTERN
|
|
|
|
|
p.To.Sym = ctxt.LookupDerived(s, s.Name+".args_stackmap")
|
|
|
|
|
}
|
|
|
|
|
if !foundArgInfo {
|
|
|
|
|
p := Appendp(s.Func().Text, newprog)
|
|
|
|
|
p.As = AFUNCDATA
|
|
|
|
|
p.From.Type = TYPE_CONST
|
|
|
|
|
p.From.Offset = objabi.FUNCDATA_ArgInfo
|
|
|
|
|
p.To.Type = TYPE_MEM
|
|
|
|
|
p.To.Name = NAME_EXTERN
|
|
|
|
|
p.To.Sym = ctxt.LookupDerived(s, fmt.Sprintf("%s.arginfo%d", s.Name, s.ABI()))
|
2016-04-11 17:35:55 +03:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Turn functions into machine code images.
|
|
|
|
|
for _, s := range text {
|
|
|
|
|
mkfwd(s)
|
2021-01-13 15:45:28 -08:00
|
|
|
if ctxt.Arch.ErrorCheck != nil {
|
|
|
|
|
ctxt.Arch.ErrorCheck(ctxt, s)
|
|
|
|
|
}
|
2017-04-04 14:31:55 -07:00
|
|
|
linkpatch(ctxt, s, newprog)
|
|
|
|
|
ctxt.Arch.Preprocess(ctxt, s, newprog)
|
|
|
|
|
ctxt.Arch.Assemble(ctxt, s, newprog)
|
2019-04-12 12:02:49 -07:00
|
|
|
if ctxt.Errors > 0 {
|
|
|
|
|
continue
|
|
|
|
|
}
|
2016-04-11 17:35:55 +03:00
|
|
|
linkpcln(ctxt, s)
|
2020-03-06 12:43:59 -05:00
|
|
|
if myimportpath != "" {
|
|
|
|
|
ctxt.populateDWARF(plist.Curfn, s, myimportpath)
|
|
|
|
|
}
|
2016-04-11 17:35:55 +03:00
|
|
|
}
|
|
|
|
|
}
|
2016-09-16 15:31:04 -07:00
|
|
|
|
2017-04-12 13:23:07 -07:00
|
|
|
func (ctxt *Link) InitTextSym(s *LSym, flag int) {
|
2017-04-10 16:57:06 -07:00
|
|
|
if s == nil {
|
|
|
|
|
// func _() { }
|
|
|
|
|
return
|
|
|
|
|
}
|
2020-07-19 00:30:12 -04:00
|
|
|
if s.Func() != nil {
|
2017-04-10 16:57:06 -07:00
|
|
|
ctxt.Diag("InitTextSym double init for %s", s.Name)
|
|
|
|
|
}
|
2020-07-19 00:30:12 -04:00
|
|
|
s.NewFuncInfo()
|
2017-04-10 16:57:06 -07:00
|
|
|
if s.OnList() {
|
|
|
|
|
ctxt.Diag("symbol %s listed multiple times", s.Name)
|
|
|
|
|
}
|
2020-07-21 15:53:30 -04:00
|
|
|
name := strings.Replace(s.Name, "\"\"", ctxt.Pkgpath, -1)
|
2021-04-04 12:46:21 -04:00
|
|
|
s.Func().FuncID = objabi.GetFuncID(name, flag&WRAPPER != 0 || flag&ABIWRAPPER != 0)
|
cmd/asm, cmd/link, runtime: introduce FuncInfo flag bits
The runtime traceback code has its own definition of which functions
mark the top frame of a stack, separate from the TOPFRAME bits that
exist in the assembly and are passed along in DWARF information.
It's error-prone and redundant to have two different sources of truth.
This CL provides the actual TOPFRAME bits to the runtime, so that
the runtime can use those bits instead of reinventing its own category.
This CL also adds a new bit, SPWRITE, which marks functions that
write directly to SP (anything but adding and subtracting constants).
Such functions must stop a traceback, because the traceback has no
way to rederive the SP on entry. Again, the runtime has its own definition
which is mostly correct, but also missing some functions. During ordinary
goroutine context switches, such functions do not appear on the stack,
so the incompleteness in the runtime usually doesn't matter.
But profiling signals can arrive at any moment, and the runtime may
crash during traceback if it attempts to unwind an SP-writing frame
and gets out-of-sync with the actual stack. The runtime contains code
to try to detect likely candidates but again it is incomplete.
Deriving the SPWRITE bit automatically from the actual assembly code
provides the complete truth, and passing it to the runtime lets the
runtime use it.
This CL is part of a stack adding windows/arm64
support (#36439), intended to land in the Go 1.17 cycle.
This CL is, however, not windows/arm64-specific.
It is cleanup meant to make the port (and future ports) easier.
Change-Id: I227f53b23ac5b3dabfcc5e8ee3f00df4e113cf58
Reviewed-on: https://go-review.googlesource.com/c/go/+/288800
Trust: Russ Cox <rsc@golang.org>
Trust: Jason A. Donenfeld <Jason@zx2c4.com>
Reviewed-by: Cherry Zhang <cherryyz@google.com>
Reviewed-by: Jason A. Donenfeld <Jason@zx2c4.com>
2021-01-28 15:21:33 -05:00
|
|
|
s.Func().FuncFlag = toFuncFlag(flag)
|
2017-04-10 16:57:06 -07:00
|
|
|
s.Set(AttrOnList, true)
|
cmd/internal/obj: stop storing Text flags in From3
Prior to this CL, flags such as NOSPLIT
on ATEXT Progs were stored in From3.Offset.
Some but not all of those flags were also
duplicated into From.Sym.Attribute.
This CL migrates all of those flags into
From.Sym.Attribute and stops creating a From3.
A side-effect of this is that printing an
ATEXT Prog can no longer simply dump From3.Offset.
That's kind of good, since the raw flag value
wasn't very informative anyway, but it did
necessitate a bunch of updates to the cmd/asm tests.
The reason I'm doing this work now is that
avoiding storing flags in both From.Sym and From3.Offset
simplifies some other changes to fix the data
race first described in CL 40254.
This CL almost passes toolstash-check -all.
The only changes are in cases where the assembler
has decided that a function's flags may be altered,
e.g. to make a function with no calls in it NOSPLIT.
Prior to this CL, that information was not printed.
Sample before:
"".Ctz64 t=1 size=63 args=0x10 locals=0x0
0x0000 00000 (/Users/josh/go/tip/src/runtime/internal/sys/intrinsics.go:35) TEXT "".Ctz64(SB), $0-16
0x0000 00000 (/Users/josh/go/tip/src/runtime/internal/sys/intrinsics.go:35) FUNCDATA $0, gclocals·f207267fbf96a0178e8758c6e3e0ce28(SB)
Sample after:
"".Ctz64 t=1 nosplit size=63 args=0x10 locals=0x0
0x0000 00000 (/Users/josh/go/tip/src/runtime/internal/sys/intrinsics.go:35) TEXT "".Ctz64(SB), NOSPLIT, $0-16
0x0000 00000 (/Users/josh/go/tip/src/runtime/internal/sys/intrinsics.go:35) FUNCDATA $0, gclocals·f207267fbf96a0178e8758c6e3e0ce28(SB)
Observe the additional "nosplit" in the first line
and the additional "NOSPLIT" in the second line.
Updates #15756
Change-Id: I5c59bd8f3bdc7c780361f801d94a261f0aef3d13
Reviewed-on: https://go-review.googlesource.com/40495
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
2017-04-11 15:15:04 -07:00
|
|
|
s.Set(AttrDuplicateOK, flag&DUPOK != 0)
|
|
|
|
|
s.Set(AttrNoSplit, flag&NOSPLIT != 0)
|
|
|
|
|
s.Set(AttrReflectMethod, flag&REFLECTMETHOD != 0)
|
|
|
|
|
s.Set(AttrWrapper, flag&WRAPPER != 0)
|
2020-09-24 13:14:46 -04:00
|
|
|
s.Set(AttrABIWrapper, flag&ABIWRAPPER != 0)
|
cmd/internal/obj: stop storing Text flags in From3
Prior to this CL, flags such as NOSPLIT
on ATEXT Progs were stored in From3.Offset.
Some but not all of those flags were also
duplicated into From.Sym.Attribute.
This CL migrates all of those flags into
From.Sym.Attribute and stops creating a From3.
A side-effect of this is that printing an
ATEXT Prog can no longer simply dump From3.Offset.
That's kind of good, since the raw flag value
wasn't very informative anyway, but it did
necessitate a bunch of updates to the cmd/asm tests.
The reason I'm doing this work now is that
avoiding storing flags in both From.Sym and From3.Offset
simplifies some other changes to fix the data
race first described in CL 40254.
This CL almost passes toolstash-check -all.
The only changes are in cases where the assembler
has decided that a function's flags may be altered,
e.g. to make a function with no calls in it NOSPLIT.
Prior to this CL, that information was not printed.
Sample before:
"".Ctz64 t=1 size=63 args=0x10 locals=0x0
0x0000 00000 (/Users/josh/go/tip/src/runtime/internal/sys/intrinsics.go:35) TEXT "".Ctz64(SB), $0-16
0x0000 00000 (/Users/josh/go/tip/src/runtime/internal/sys/intrinsics.go:35) FUNCDATA $0, gclocals·f207267fbf96a0178e8758c6e3e0ce28(SB)
Sample after:
"".Ctz64 t=1 nosplit size=63 args=0x10 locals=0x0
0x0000 00000 (/Users/josh/go/tip/src/runtime/internal/sys/intrinsics.go:35) TEXT "".Ctz64(SB), NOSPLIT, $0-16
0x0000 00000 (/Users/josh/go/tip/src/runtime/internal/sys/intrinsics.go:35) FUNCDATA $0, gclocals·f207267fbf96a0178e8758c6e3e0ce28(SB)
Observe the additional "nosplit" in the first line
and the additional "NOSPLIT" in the second line.
Updates #15756
Change-Id: I5c59bd8f3bdc7c780361f801d94a261f0aef3d13
Reviewed-on: https://go-review.googlesource.com/40495
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
2017-04-11 15:15:04 -07:00
|
|
|
s.Set(AttrNeedCtxt, flag&NEEDCTXT != 0)
|
|
|
|
|
s.Set(AttrNoFrame, flag&NOFRAME != 0)
|
2017-04-18 12:53:25 -07:00
|
|
|
s.Type = objabi.STEXT
|
2017-04-12 17:43:30 -07:00
|
|
|
ctxt.Text = append(ctxt.Text, s)
|
2017-04-13 05:57:59 -07:00
|
|
|
|
2020-03-20 16:05:24 -04:00
|
|
|
// Set up DWARF entries for s
|
2020-05-01 19:13:30 -04:00
|
|
|
ctxt.dwarfSym(s)
|
2017-04-10 16:57:06 -07:00
|
|
|
}
|
|
|
|
|
|
cmd/asm, cmd/link, runtime: introduce FuncInfo flag bits
The runtime traceback code has its own definition of which functions
mark the top frame of a stack, separate from the TOPFRAME bits that
exist in the assembly and are passed along in DWARF information.
It's error-prone and redundant to have two different sources of truth.
This CL provides the actual TOPFRAME bits to the runtime, so that
the runtime can use those bits instead of reinventing its own category.
This CL also adds a new bit, SPWRITE, which marks functions that
write directly to SP (anything but adding and subtracting constants).
Such functions must stop a traceback, because the traceback has no
way to rederive the SP on entry. Again, the runtime has its own definition
which is mostly correct, but also missing some functions. During ordinary
goroutine context switches, such functions do not appear on the stack,
so the incompleteness in the runtime usually doesn't matter.
But profiling signals can arrive at any moment, and the runtime may
crash during traceback if it attempts to unwind an SP-writing frame
and gets out-of-sync with the actual stack. The runtime contains code
to try to detect likely candidates but again it is incomplete.
Deriving the SPWRITE bit automatically from the actual assembly code
provides the complete truth, and passing it to the runtime lets the
runtime use it.
This CL is part of a stack adding windows/arm64
support (#36439), intended to land in the Go 1.17 cycle.
This CL is, however, not windows/arm64-specific.
It is cleanup meant to make the port (and future ports) easier.
Change-Id: I227f53b23ac5b3dabfcc5e8ee3f00df4e113cf58
Reviewed-on: https://go-review.googlesource.com/c/go/+/288800
Trust: Russ Cox <rsc@golang.org>
Trust: Jason A. Donenfeld <Jason@zx2c4.com>
Reviewed-by: Cherry Zhang <cherryyz@google.com>
Reviewed-by: Jason A. Donenfeld <Jason@zx2c4.com>
2021-01-28 15:21:33 -05:00
|
|
|
func toFuncFlag(flag int) objabi.FuncFlag {
|
|
|
|
|
var out objabi.FuncFlag
|
|
|
|
|
if flag&TOPFRAME != 0 {
|
|
|
|
|
out |= objabi.FuncFlag_TOPFRAME
|
|
|
|
|
}
|
|
|
|
|
return out
|
|
|
|
|
}
|
|
|
|
|
|
2016-09-16 15:31:04 -07:00
|
|
|
func (ctxt *Link) Globl(s *LSym, size int64, flag int) {
|
2016-10-24 23:15:41 +03:00
|
|
|
if s.OnList() {
|
2017-04-10 16:57:06 -07:00
|
|
|
ctxt.Diag("symbol %s listed multiple times", s.Name)
|
2016-09-16 15:31:04 -07:00
|
|
|
}
|
2016-10-24 23:15:41 +03:00
|
|
|
s.Set(AttrOnList, true)
|
2016-09-16 15:31:04 -07:00
|
|
|
ctxt.Data = append(ctxt.Data, s)
|
|
|
|
|
s.Size = size
|
2017-04-19 15:30:08 +12:00
|
|
|
if s.Type == 0 {
|
2017-04-18 12:53:25 -07:00
|
|
|
s.Type = objabi.SBSS
|
2016-09-16 15:31:04 -07:00
|
|
|
}
|
|
|
|
|
if flag&DUPOK != 0 {
|
2016-10-24 23:15:41 +03:00
|
|
|
s.Set(AttrDuplicateOK, true)
|
2016-09-16 15:31:04 -07:00
|
|
|
}
|
|
|
|
|
if flag&RODATA != 0 {
|
2017-04-18 12:53:25 -07:00
|
|
|
s.Type = objabi.SRODATA
|
2016-09-16 15:31:04 -07:00
|
|
|
} else if flag&NOPTR != 0 {
|
cmd/internal/obj: fix LSym.Type during compilation, not linking
Prior to this CL, the compiler and assembler
were sloppy about the LSym.Type for LSyms
containing static data.
The linker then fixed this up, converting
Sxxx and SBSS to SDATA, and SNOPTRBSS to SNOPTRDATA
if it noticed that the symbol had associated data.
It is preferable to just get this right in cmd/compile
and cmd/asm, because it removes an unnecessary traversal
of the symbol table from the linker (see #14624).
Do this by touching up the LSym.Type fixes in
LSym.prepwrite and Link.Globl.
I have confirmed by instrumenting the linker
that the now-eliminated code paths were unreached.
And an additional check in the object file writing code
will help preserve that invariant.
There was a case in the Windows linker,
with internal linking and cgo,
where we were generating SNOPTRBSS symbols with data.
For now, convert those at the site at which they occur
into SNOPTRDATA, just like they were.
Does not pass toolstash-check,
but does generate identical linked binaries.
No compiler performance changes.
Change-Id: I77b071ab103685ff8e042cee9abb864385488872
Reviewed-on: https://go-review.googlesource.com/40864
Run-TryBot: Josh Bleecher Snyder <josharian@gmail.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Alex Brainman <alex.brainman@gmail.com>
Reviewed-by: Michael Hudson-Doyle <michael.hudson@canonical.com>
2017-04-16 08:22:44 -07:00
|
|
|
if s.Type == objabi.SDATA {
|
|
|
|
|
s.Type = objabi.SNOPTRDATA
|
|
|
|
|
} else {
|
|
|
|
|
s.Type = objabi.SNOPTRBSS
|
|
|
|
|
}
|
2016-09-16 15:31:04 -07:00
|
|
|
} else if flag&TLSBSS != 0 {
|
2017-04-18 12:53:25 -07:00
|
|
|
s.Type = objabi.STLSBSS
|
2016-09-16 15:31:04 -07:00
|
|
|
}
|
2020-06-30 07:55:16 -04:00
|
|
|
if strings.HasPrefix(s.Name, "\"\"."+StaticNamePref) {
|
|
|
|
|
s.Set(AttrStatic, true)
|
|
|
|
|
}
|
2016-09-16 15:31:04 -07:00
|
|
|
}
|
2018-04-21 21:13:56 -04:00
|
|
|
|
|
|
|
|
// EmitEntryLiveness generates PCDATA Progs after p to switch to the
|
|
|
|
|
// liveness map active at the entry of function s. It returns the last
|
|
|
|
|
// Prog generated.
|
|
|
|
|
func (ctxt *Link) EmitEntryLiveness(s *LSym, p *Prog, newprog ProgAlloc) *Prog {
|
cmd/internal/obj: mark split-stack prologue nonpreemptible
When there are both a synchronous preemption request (by
clobbering the stack guard) and an asynchronous one (by signal),
the running goroutine may observe the synchronous request first
in stack bounds check, and go to the path of calling morestack.
If the preemption signal arrives at this point before the call to
morestack, the goroutine will be asynchronously preempted,
entering the scheduler. When it is resumed, the scheduler clears
the preemption request, unclobbers the stack guard. But the
resumed goroutine will still call morestack, as it is already on
its way. morestack will, as there is no preemption request,
double the stack unnecessarily. If this happens multiple times,
the stack may grow too big, although only a small amount is
actually used.
To fix this, we mark the stack bounds check and the call to
morestack async-nonpreemptible, starting after the memory
instruction (mostly a load, on x86 CMP with memory).
Not done for Wasm as it does not support async preemption.
Fixes #35470.
Change-Id: Ibd7f3d935a3649b80f47539116ec9b9556680cf2
Reviewed-on: https://go-review.googlesource.com/c/go/+/207350
Reviewed-by: David Chase <drchase@google.com>
2019-11-14 20:23:17 -05:00
|
|
|
pcdata := ctxt.EmitEntryStackMap(s, p, newprog)
|
2020-10-21 20:15:48 -04:00
|
|
|
pcdata = ctxt.EmitEntryUnsafePoint(s, pcdata, newprog)
|
cmd/internal/obj: mark split-stack prologue nonpreemptible
When there are both a synchronous preemption request (by
clobbering the stack guard) and an asynchronous one (by signal),
the running goroutine may observe the synchronous request first
in stack bounds check, and go to the path of calling morestack.
If the preemption signal arrives at this point before the call to
morestack, the goroutine will be asynchronously preempted,
entering the scheduler. When it is resumed, the scheduler clears
the preemption request, unclobbers the stack guard. But the
resumed goroutine will still call morestack, as it is already on
its way. morestack will, as there is no preemption request,
double the stack unnecessarily. If this happens multiple times,
the stack may grow too big, although only a small amount is
actually used.
To fix this, we mark the stack bounds check and the call to
morestack async-nonpreemptible, starting after the memory
instruction (mostly a load, on x86 CMP with memory).
Not done for Wasm as it does not support async preemption.
Fixes #35470.
Change-Id: Ibd7f3d935a3649b80f47539116ec9b9556680cf2
Reviewed-on: https://go-review.googlesource.com/c/go/+/207350
Reviewed-by: David Chase <drchase@google.com>
2019-11-14 20:23:17 -05:00
|
|
|
return pcdata
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Similar to EmitEntryLiveness, but just emit stack map.
|
|
|
|
|
func (ctxt *Link) EmitEntryStackMap(s *LSym, p *Prog, newprog ProgAlloc) *Prog {
|
2018-04-21 21:13:56 -04:00
|
|
|
pcdata := Appendp(p, newprog)
|
2020-07-19 00:30:12 -04:00
|
|
|
pcdata.Pos = s.Func().Text.Pos
|
2018-04-21 21:13:56 -04:00
|
|
|
pcdata.As = APCDATA
|
|
|
|
|
pcdata.From.Type = TYPE_CONST
|
|
|
|
|
pcdata.From.Offset = objabi.PCDATA_StackMapIndex
|
|
|
|
|
pcdata.To.Type = TYPE_CONST
|
|
|
|
|
pcdata.To.Offset = -1 // pcdata starts at -1 at function entry
|
|
|
|
|
|
cmd/internal/obj: mark split-stack prologue nonpreemptible
When there are both a synchronous preemption request (by
clobbering the stack guard) and an asynchronous one (by signal),
the running goroutine may observe the synchronous request first
in stack bounds check, and go to the path of calling morestack.
If the preemption signal arrives at this point before the call to
morestack, the goroutine will be asynchronously preempted,
entering the scheduler. When it is resumed, the scheduler clears
the preemption request, unclobbers the stack guard. But the
resumed goroutine will still call morestack, as it is already on
its way. morestack will, as there is no preemption request,
double the stack unnecessarily. If this happens multiple times,
the stack may grow too big, although only a small amount is
actually used.
To fix this, we mark the stack bounds check and the call to
morestack async-nonpreemptible, starting after the memory
instruction (mostly a load, on x86 CMP with memory).
Not done for Wasm as it does not support async preemption.
Fixes #35470.
Change-Id: Ibd7f3d935a3649b80f47539116ec9b9556680cf2
Reviewed-on: https://go-review.googlesource.com/c/go/+/207350
Reviewed-by: David Chase <drchase@google.com>
2019-11-14 20:23:17 -05:00
|
|
|
return pcdata
|
|
|
|
|
}
|
|
|
|
|
|
2020-10-21 20:15:48 -04:00
|
|
|
// Similar to EmitEntryLiveness, but just emit unsafe point map.
|
|
|
|
|
func (ctxt *Link) EmitEntryUnsafePoint(s *LSym, p *Prog, newprog ProgAlloc) *Prog {
|
cmd/internal/obj: mark split-stack prologue nonpreemptible
When there are both a synchronous preemption request (by
clobbering the stack guard) and an asynchronous one (by signal),
the running goroutine may observe the synchronous request first
in stack bounds check, and go to the path of calling morestack.
If the preemption signal arrives at this point before the call to
morestack, the goroutine will be asynchronously preempted,
entering the scheduler. When it is resumed, the scheduler clears
the preemption request, unclobbers the stack guard. But the
resumed goroutine will still call morestack, as it is already on
its way. morestack will, as there is no preemption request,
double the stack unnecessarily. If this happens multiple times,
the stack may grow too big, although only a small amount is
actually used.
To fix this, we mark the stack bounds check and the call to
morestack async-nonpreemptible, starting after the memory
instruction (mostly a load, on x86 CMP with memory).
Not done for Wasm as it does not support async preemption.
Fixes #35470.
Change-Id: Ibd7f3d935a3649b80f47539116ec9b9556680cf2
Reviewed-on: https://go-review.googlesource.com/c/go/+/207350
Reviewed-by: David Chase <drchase@google.com>
2019-11-14 20:23:17 -05:00
|
|
|
pcdata := Appendp(p, newprog)
|
2020-07-19 00:30:12 -04:00
|
|
|
pcdata.Pos = s.Func().Text.Pos
|
2018-03-27 15:50:45 -04:00
|
|
|
pcdata.As = APCDATA
|
|
|
|
|
pcdata.From.Type = TYPE_CONST
|
2020-10-21 20:15:48 -04:00
|
|
|
pcdata.From.Offset = objabi.PCDATA_UnsafePoint
|
2018-03-27 15:50:45 -04:00
|
|
|
pcdata.To.Type = TYPE_CONST
|
|
|
|
|
pcdata.To.Offset = -1
|
|
|
|
|
|
2018-04-21 21:13:56 -04:00
|
|
|
return pcdata
|
|
|
|
|
}
|
2019-10-21 14:08:11 -04:00
|
|
|
|
|
|
|
|
// StartUnsafePoint generates PCDATA Progs after p to mark the
|
|
|
|
|
// beginning of an unsafe point. The unsafe point starts immediately
|
|
|
|
|
// after p.
|
|
|
|
|
// It returns the last Prog generated.
|
|
|
|
|
func (ctxt *Link) StartUnsafePoint(p *Prog, newprog ProgAlloc) *Prog {
|
|
|
|
|
pcdata := Appendp(p, newprog)
|
|
|
|
|
pcdata.As = APCDATA
|
|
|
|
|
pcdata.From.Type = TYPE_CONST
|
2020-10-21 20:15:48 -04:00
|
|
|
pcdata.From.Offset = objabi.PCDATA_UnsafePoint
|
2019-10-21 14:08:11 -04:00
|
|
|
pcdata.To.Type = TYPE_CONST
|
2020-10-21 20:15:48 -04:00
|
|
|
pcdata.To.Offset = objabi.PCDATA_UnsafePointUnsafe
|
2019-10-21 14:08:11 -04:00
|
|
|
|
|
|
|
|
return pcdata
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// EndUnsafePoint generates PCDATA Progs after p to mark the end of an
|
cmd/internal/obj, runtime: preempt & restart some instruction sequences
On some architectures, for async preemption the injected call
needs to clobber a register (usually REGTMP) in order to return
to the preempted function. As a consequence, the PC ranges where
REGTMP is live are not preemptible.
The uses of REGTMP are usually generated by the assembler, where
it needs to load or materialize a large constant or offset that
doesn't fit into the instruction. In those cases, REGTMP is not
live at the start of the instruction sequence. Instead of giving
up preemption in those cases, we could preempt it and restart the
sequence when resuming the execution. Basically, this is like
reissuing an interrupted instruction, except that here the
"instruction" is a Prog that consists of multiple machine
instructions. For this to work, we need to generate PC data to
mark the start of the Prog.
Currently this is only done for ARM64.
TODO: the split-stack function prologue is currently not async
preemptible. We could use this mechanism, preempt it and restart
at the function entry.
Change-Id: I37cb282f8e606e7ab6f67b3edfdc6063097b4bd1
Reviewed-on: https://go-review.googlesource.com/c/go/+/208126
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>
2019-11-20 17:10:34 -05:00
|
|
|
// unsafe point, restoring the register map index to oldval.
|
2019-10-21 14:08:11 -04:00
|
|
|
// The unsafe point ends right after p.
|
|
|
|
|
// It returns the last Prog generated.
|
|
|
|
|
func (ctxt *Link) EndUnsafePoint(p *Prog, newprog ProgAlloc, oldval int64) *Prog {
|
|
|
|
|
pcdata := Appendp(p, newprog)
|
|
|
|
|
pcdata.As = APCDATA
|
|
|
|
|
pcdata.From.Type = TYPE_CONST
|
2020-10-21 20:15:48 -04:00
|
|
|
pcdata.From.Offset = objabi.PCDATA_UnsafePoint
|
2019-10-21 14:08:11 -04:00
|
|
|
pcdata.To.Type = TYPE_CONST
|
|
|
|
|
pcdata.To.Offset = oldval
|
|
|
|
|
|
|
|
|
|
return pcdata
|
|
|
|
|
}
|
|
|
|
|
|
cmd/internal/obj, runtime: preempt & restart some instruction sequences
On some architectures, for async preemption the injected call
needs to clobber a register (usually REGTMP) in order to return
to the preempted function. As a consequence, the PC ranges where
REGTMP is live are not preemptible.
The uses of REGTMP are usually generated by the assembler, where
it needs to load or materialize a large constant or offset that
doesn't fit into the instruction. In those cases, REGTMP is not
live at the start of the instruction sequence. Instead of giving
up preemption in those cases, we could preempt it and restart the
sequence when resuming the execution. Basically, this is like
reissuing an interrupted instruction, except that here the
"instruction" is a Prog that consists of multiple machine
instructions. For this to work, we need to generate PC data to
mark the start of the Prog.
Currently this is only done for ARM64.
TODO: the split-stack function prologue is currently not async
preemptible. We could use this mechanism, preempt it and restart
at the function entry.
Change-Id: I37cb282f8e606e7ab6f67b3edfdc6063097b4bd1
Reviewed-on: https://go-review.googlesource.com/c/go/+/208126
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>
2019-11-20 17:10:34 -05:00
|
|
|
// MarkUnsafePoints inserts PCDATAs to mark nonpreemptible and restartable
|
|
|
|
|
// instruction sequences, based on isUnsafePoint and isRestartable predicate.
|
|
|
|
|
// p0 is the start of the instruction stream.
|
|
|
|
|
// isUnsafePoint(p) returns true if p is not safe for async preemption.
|
|
|
|
|
// isRestartable(p) returns true if we can restart at the start of p (this Prog)
|
|
|
|
|
// upon async preemption. (Currently multi-Prog restartable sequence is not
|
|
|
|
|
// supported.)
|
|
|
|
|
// isRestartable can be nil. In this case it is treated as always returning false.
|
|
|
|
|
// If isUnsafePoint(p) and isRestartable(p) are both true, it is treated as
|
|
|
|
|
// an unsafe point.
|
|
|
|
|
func MarkUnsafePoints(ctxt *Link, p0 *Prog, newprog ProgAlloc, isUnsafePoint, isRestartable func(*Prog) bool) {
|
|
|
|
|
if isRestartable == nil {
|
|
|
|
|
// Default implementation: nothing is restartable.
|
|
|
|
|
isRestartable = func(*Prog) bool { return false }
|
|
|
|
|
}
|
2019-10-21 14:08:11 -04:00
|
|
|
prev := p0
|
cmd/internal/obj, runtime: preempt & restart some instruction sequences
On some architectures, for async preemption the injected call
needs to clobber a register (usually REGTMP) in order to return
to the preempted function. As a consequence, the PC ranges where
REGTMP is live are not preemptible.
The uses of REGTMP are usually generated by the assembler, where
it needs to load or materialize a large constant or offset that
doesn't fit into the instruction. In those cases, REGTMP is not
live at the start of the instruction sequence. Instead of giving
up preemption in those cases, we could preempt it and restart the
sequence when resuming the execution. Basically, this is like
reissuing an interrupted instruction, except that here the
"instruction" is a Prog that consists of multiple machine
instructions. For this to work, we need to generate PC data to
mark the start of the Prog.
Currently this is only done for ARM64.
TODO: the split-stack function prologue is currently not async
preemptible. We could use this mechanism, preempt it and restart
at the function entry.
Change-Id: I37cb282f8e606e7ab6f67b3edfdc6063097b4bd1
Reviewed-on: https://go-review.googlesource.com/c/go/+/208126
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>
2019-11-20 17:10:34 -05:00
|
|
|
prevPcdata := int64(-1) // entry PC data value
|
|
|
|
|
prevRestart := int64(0)
|
2019-10-21 14:08:11 -04:00
|
|
|
for p := prev.Link; p != nil; p, prev = p.Link, p {
|
2020-10-21 20:15:48 -04:00
|
|
|
if p.As == APCDATA && p.From.Offset == objabi.PCDATA_UnsafePoint {
|
cmd/internal/obj, runtime: preempt & restart some instruction sequences
On some architectures, for async preemption the injected call
needs to clobber a register (usually REGTMP) in order to return
to the preempted function. As a consequence, the PC ranges where
REGTMP is live are not preemptible.
The uses of REGTMP are usually generated by the assembler, where
it needs to load or materialize a large constant or offset that
doesn't fit into the instruction. In those cases, REGTMP is not
live at the start of the instruction sequence. Instead of giving
up preemption in those cases, we could preempt it and restart the
sequence when resuming the execution. Basically, this is like
reissuing an interrupted instruction, except that here the
"instruction" is a Prog that consists of multiple machine
instructions. For this to work, we need to generate PC data to
mark the start of the Prog.
Currently this is only done for ARM64.
TODO: the split-stack function prologue is currently not async
preemptible. We could use this mechanism, preempt it and restart
at the function entry.
Change-Id: I37cb282f8e606e7ab6f67b3edfdc6063097b4bd1
Reviewed-on: https://go-review.googlesource.com/c/go/+/208126
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>
2019-11-20 17:10:34 -05:00
|
|
|
prevPcdata = p.To.Offset
|
2019-10-21 14:08:11 -04:00
|
|
|
continue
|
|
|
|
|
}
|
2020-10-21 20:15:48 -04:00
|
|
|
if prevPcdata == objabi.PCDATA_UnsafePointUnsafe {
|
2019-10-21 14:08:11 -04:00
|
|
|
continue // already unsafe
|
|
|
|
|
}
|
|
|
|
|
if isUnsafePoint(p) {
|
|
|
|
|
q := ctxt.StartUnsafePoint(prev, newprog)
|
|
|
|
|
q.Pc = p.Pc
|
|
|
|
|
q.Link = p
|
|
|
|
|
// Advance to the end of unsafe point.
|
|
|
|
|
for p.Link != nil && isUnsafePoint(p.Link) {
|
|
|
|
|
p = p.Link
|
|
|
|
|
}
|
|
|
|
|
if p.Link == nil {
|
|
|
|
|
break // Reached the end, don't bother marking the end
|
|
|
|
|
}
|
cmd/internal/obj, runtime: preempt & restart some instruction sequences
On some architectures, for async preemption the injected call
needs to clobber a register (usually REGTMP) in order to return
to the preempted function. As a consequence, the PC ranges where
REGTMP is live are not preemptible.
The uses of REGTMP are usually generated by the assembler, where
it needs to load or materialize a large constant or offset that
doesn't fit into the instruction. In those cases, REGTMP is not
live at the start of the instruction sequence. Instead of giving
up preemption in those cases, we could preempt it and restart the
sequence when resuming the execution. Basically, this is like
reissuing an interrupted instruction, except that here the
"instruction" is a Prog that consists of multiple machine
instructions. For this to work, we need to generate PC data to
mark the start of the Prog.
Currently this is only done for ARM64.
TODO: the split-stack function prologue is currently not async
preemptible. We could use this mechanism, preempt it and restart
at the function entry.
Change-Id: I37cb282f8e606e7ab6f67b3edfdc6063097b4bd1
Reviewed-on: https://go-review.googlesource.com/c/go/+/208126
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>
2019-11-20 17:10:34 -05:00
|
|
|
p = ctxt.EndUnsafePoint(p, newprog, prevPcdata)
|
|
|
|
|
p.Pc = p.Link.Pc
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
if isRestartable(p) {
|
|
|
|
|
val := int64(objabi.PCDATA_Restart1)
|
|
|
|
|
if val == prevRestart {
|
|
|
|
|
val = objabi.PCDATA_Restart2
|
|
|
|
|
}
|
|
|
|
|
prevRestart = val
|
|
|
|
|
q := Appendp(prev, newprog)
|
|
|
|
|
q.As = APCDATA
|
|
|
|
|
q.From.Type = TYPE_CONST
|
2020-10-21 20:15:48 -04:00
|
|
|
q.From.Offset = objabi.PCDATA_UnsafePoint
|
cmd/internal/obj, runtime: preempt & restart some instruction sequences
On some architectures, for async preemption the injected call
needs to clobber a register (usually REGTMP) in order to return
to the preempted function. As a consequence, the PC ranges where
REGTMP is live are not preemptible.
The uses of REGTMP are usually generated by the assembler, where
it needs to load or materialize a large constant or offset that
doesn't fit into the instruction. In those cases, REGTMP is not
live at the start of the instruction sequence. Instead of giving
up preemption in those cases, we could preempt it and restart the
sequence when resuming the execution. Basically, this is like
reissuing an interrupted instruction, except that here the
"instruction" is a Prog that consists of multiple machine
instructions. For this to work, we need to generate PC data to
mark the start of the Prog.
Currently this is only done for ARM64.
TODO: the split-stack function prologue is currently not async
preemptible. We could use this mechanism, preempt it and restart
at the function entry.
Change-Id: I37cb282f8e606e7ab6f67b3edfdc6063097b4bd1
Reviewed-on: https://go-review.googlesource.com/c/go/+/208126
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>
2019-11-20 17:10:34 -05:00
|
|
|
q.To.Type = TYPE_CONST
|
|
|
|
|
q.To.Offset = val
|
|
|
|
|
q.Pc = p.Pc
|
|
|
|
|
q.Link = p
|
|
|
|
|
|
|
|
|
|
if p.Link == nil {
|
|
|
|
|
break // Reached the end, don't bother marking the end
|
|
|
|
|
}
|
|
|
|
|
if isRestartable(p.Link) {
|
|
|
|
|
// Next Prog is also restartable. No need to mark the end
|
|
|
|
|
// of this sequence. We'll just go ahead mark the next one.
|
|
|
|
|
continue
|
|
|
|
|
}
|
|
|
|
|
p = Appendp(p, newprog)
|
|
|
|
|
p.As = APCDATA
|
|
|
|
|
p.From.Type = TYPE_CONST
|
2020-10-21 20:15:48 -04:00
|
|
|
p.From.Offset = objabi.PCDATA_UnsafePoint
|
cmd/internal/obj, runtime: preempt & restart some instruction sequences
On some architectures, for async preemption the injected call
needs to clobber a register (usually REGTMP) in order to return
to the preempted function. As a consequence, the PC ranges where
REGTMP is live are not preemptible.
The uses of REGTMP are usually generated by the assembler, where
it needs to load or materialize a large constant or offset that
doesn't fit into the instruction. In those cases, REGTMP is not
live at the start of the instruction sequence. Instead of giving
up preemption in those cases, we could preempt it and restart the
sequence when resuming the execution. Basically, this is like
reissuing an interrupted instruction, except that here the
"instruction" is a Prog that consists of multiple machine
instructions. For this to work, we need to generate PC data to
mark the start of the Prog.
Currently this is only done for ARM64.
TODO: the split-stack function prologue is currently not async
preemptible. We could use this mechanism, preempt it and restart
at the function entry.
Change-Id: I37cb282f8e606e7ab6f67b3edfdc6063097b4bd1
Reviewed-on: https://go-review.googlesource.com/c/go/+/208126
Run-TryBot: Cherry Zhang <cherryyz@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Austin Clements <austin@google.com>
2019-11-20 17:10:34 -05:00
|
|
|
p.To.Type = TYPE_CONST
|
|
|
|
|
p.To.Offset = prevPcdata
|
2019-10-21 14:08:11 -04:00
|
|
|
p.Pc = p.Link.Pc
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|