From 4c20f7f15a9a8eed50d8cbb8be8f74d449093a5c Mon Sep 17 00:00:00 2001 From: matloob Date: Mon, 25 Aug 2025 17:22:01 -0400 Subject: [PATCH] cmd/cgo: run gcc to get errors and debug info in parallel This change kicks off the work to load the debug info when processing each file, and then waits for all the files to be processed before starting the single-goroutined part that processes them. The processing is very order dependent so we won't try to make it concurrent. Though in a later CL we can wait for only the relevant package to have been processed concurrently before doing the single-goroutined processing for it instead of waiting for all packages to be processed concurrently before the single goroutine section. We use a par.Queue to make sure we're not running too many gcc compiles at the same time. The change to cmd/dist makes the par package available to cgo. Fixes #75167 Change-Id: I6a6a6964fb7f3a3684118b5ee66f1ad856b3ee59 Reviewed-on: https://go-review.googlesource.com/c/go/+/699020 Reviewed-by: Michael Matloob LUCI-TryBot-Result: Go LUCI Reviewed-by: Ian Lance Taylor Reviewed-by: Michael Pratt --- src/cmd/cgo/gcc.go | 25 ++++++++++++++++++------- src/cmd/cgo/main.go | 15 +++++++++------ src/cmd/dist/buildtool.go | 1 + 3 files changed, 28 insertions(+), 13 deletions(-) diff --git a/src/cmd/cgo/gcc.go b/src/cmd/cgo/gcc.go index aa418af93b..1f18657400 100644 --- a/src/cmd/cgo/gcc.go +++ b/src/cmd/cgo/gcc.go @@ -183,16 +183,16 @@ func splitQuoted(s string) (r []string, err error) { return args, err } -// Translate rewrites f.AST, the original Go input, to remove -// references to the imported package C, replacing them with -// references to the equivalent Go types, functions, and variables. -func (p *Package) Translate(f *File) { +// loadDebug runs gcc to load debug information for the File. The debug +// information will be saved to the debugs field of the file, and be +// processed when Translate is called on the file later. +// loadDebug is called concurrently with different files. +func (f *File) loadDebug(p *Package) { for _, cref := range f.Ref { // Convert C.ulong to C.unsigned long, etc. cref.Name.C = cname(cref.Name.Go) } - var debugs []*debug // debug data from iterations of gccDebug ft := fileTypedefs{typedefs: make(map[string]bool)} numTypedefs := -1 for len(ft.typedefs) > numTypedefs { @@ -211,7 +211,7 @@ func (p *Package) Translate(f *File) { } needType := p.guessKinds(f) if len(needType) > 0 { - debugs = append(debugs, p.loadDWARF(f, &ft, needType)) + f.debugs = append(f.debugs, p.loadDWARF(f, &ft, needType)) } // In godefs mode we're OK with the typedefs, which @@ -221,10 +221,16 @@ func (p *Package) Translate(f *File) { break } } +} +// Translate rewrites f.AST, the original Go input, to remove +// references to the imported package C, replacing them with +// references to the equivalent Go types, functions, and variables. +// Preconditions: File.loadDebug must be called prior to translate. +func (p *Package) Translate(f *File) { var conv typeConv conv.Init(p.PtrSize, p.IntSize) - for _, d := range debugs { + for _, d := range f.debugs { p.recordTypes(f, d, &conv) } p.prepareNames(f) @@ -283,6 +289,7 @@ func (f *File) loadDefines(gccOptions []string) bool { // guessKinds tricks gcc into revealing the kind of each // name xxx for the references C.xxx in the Go input. // The kind is either a constant, type, or variable. +// guessKinds is called concurrently with different files. func (p *Package) guessKinds(f *File) []*Name { // Determine kinds for names we already know about, // like #defines or 'struct foo', before bothering with gcc. @@ -526,6 +533,7 @@ func (p *Package) guessKinds(f *File) []*Name { // loadDWARF parses the DWARF debug information generated // by gcc to learn the details of the constants, variables, and types // being referred to as C.xxx. +// loadDwarf is called concurrently with different files. func (p *Package) loadDWARF(f *File, ft *fileTypedefs, names []*Name) *debug { // Extract the types from the DWARF section of an object // from a well-formed C program. Gcc only generates DWARF info @@ -1789,6 +1797,7 @@ func gccTmp() string { // gccCmd returns the gcc command line to use for compiling // the input. +// gccCommand is called concurrently for different files. func (p *Package) gccCmd(ofile string) []string { c := append(gccBaseCmd, "-w", // no warnings @@ -1832,6 +1841,7 @@ func (p *Package) gccCmd(ofile string) []string { // gccDebug runs gcc -gdwarf-2 over the C program stdin and // returns the corresponding DWARF data and, if present, debug data block. +// gccDebug is called concurrently with different C programs. func (p *Package) gccDebug(stdin []byte, nnames int) (d *dwarf.Data, ints []int64, floats []float64, strs []string) { ofile := gccTmp() runGcc(stdin, p.gccCmd(ofile)) @@ -2222,6 +2232,7 @@ func gccDefines(stdin []byte, gccOptions []string) string { // gccErrors runs gcc over the C program stdin and returns // the errors that gcc prints. That is, this function expects // gcc to fail. +// gccErrors is called concurrently with different C programs. func (p *Package) gccErrors(stdin []byte, extraArgs ...string) string { // TODO(rsc): require failure args := p.gccCmd(gccTmp()) diff --git a/src/cmd/cgo/main.go b/src/cmd/cgo/main.go index df3b7fbd1d..5e08427daf 100644 --- a/src/cmd/cgo/main.go +++ b/src/cmd/cgo/main.go @@ -30,6 +30,7 @@ import ( "cmd/internal/edit" "cmd/internal/hash" "cmd/internal/objabi" + "cmd/internal/par" "cmd/internal/telemetry/counter" ) @@ -74,6 +75,8 @@ type File struct { NoCallbacks map[string]bool // C function names that with #cgo nocallback directive NoEscapes map[string]bool // C function names that with #cgo noescape directive Edit *edit.Buffer + + debugs []*debug // debug data from iterations of gccDebug. Initialized by File.loadDebug. } func (f *File) offset(p token.Pos) int { @@ -391,7 +394,7 @@ func main() { h := hash.New32() io.WriteString(h, *importPath) var once sync.Once - var wg sync.WaitGroup + q := par.NewQueue(runtime.GOMAXPROCS(0)) fs := make([]*File, len(goFiles)) for i, input := range goFiles { if *srcDir != "" { @@ -413,9 +416,7 @@ func main() { fatalf("%s", err) } - wg.Add(1) - go func() { - defer wg.Done() + q.Add(func() { // Apply trimpath to the file path. The path won't be read from after this point. input, _ = objabi.ApplyRewrites(input, *trimpath) if strings.ContainsAny(input, "\r\n") { @@ -436,10 +437,12 @@ func main() { }) fs[i] = f - }() + + f.loadDebug(p) + }) } - wg.Wait() + <-q.Idle() cPrefix = fmt.Sprintf("_%x", h.Sum(nil)[0:6]) diff --git a/src/cmd/dist/buildtool.go b/src/cmd/dist/buildtool.go index b7e5891981..080de832b2 100644 --- a/src/cmd/dist/buildtool.go +++ b/src/cmd/dist/buildtool.go @@ -49,6 +49,7 @@ var bootstrapDirs = []string{ "cmd/internal/macho", "cmd/internal/obj/...", "cmd/internal/objabi", + "cmd/internal/par", "cmd/internal/pgo", "cmd/internal/pkgpath", "cmd/internal/quoted",