go/src/cmd/compile/internal/gc/lex.go

217 lines
6 KiB
Go
Raw Normal View History

// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package gc
import (
"cmd/compile/internal/syntax"
"cmd/internal/objabi"
"cmd/internal/src"
"fmt"
"strings"
)
[dev.inline] cmd/internal/src: replace src.Pos with syntax.Pos This replaces the src.Pos LineHist-based position tracking with the syntax.Pos implementation and updates all uses. The LineHist table is not used anymore - the respective code is still there but should be removed eventually. CL forthcoming. Passes toolstash -cmp when comparing to the master repo (with the exception of a couple of swapped assembly instructions, likely due to different instruction scheduling because the line-based sorting has changed; though this is won't affect correctness). The sizes of various important compiler data structures have increased significantly (see the various sizes_test.go files); this is probably the reason for an increase of compilation times (to be addressed). Here are the results of compilebench -count 5, run on a "quiet" machine (no apps running besides a terminal): name old time/op new time/op delta Template 256ms ± 1% 280ms ±15% +9.54% (p=0.008 n=5+5) Unicode 132ms ± 1% 132ms ± 1% ~ (p=0.690 n=5+5) GoTypes 891ms ± 1% 917ms ± 2% +2.88% (p=0.008 n=5+5) Compiler 3.84s ± 2% 3.99s ± 2% +3.95% (p=0.016 n=5+5) MakeBash 47.1s ± 1% 47.2s ± 2% ~ (p=0.841 n=5+5) name old user-ns/op new user-ns/op delta Template 309M ± 1% 326M ± 2% +5.18% (p=0.008 n=5+5) Unicode 165M ± 1% 168M ± 4% ~ (p=0.421 n=5+5) GoTypes 1.14G ± 2% 1.18G ± 1% +3.47% (p=0.008 n=5+5) Compiler 5.00G ± 1% 5.16G ± 1% +3.12% (p=0.008 n=5+5) Change-Id: I241c4246cdff627d7ecb95cac23060b38f9775ec Reviewed-on: https://go-review.googlesource.com/34273 Run-TryBot: Robert Griesemer <gri@golang.org> TryBot-Result: Gobot Gobot <gobot@golang.org> Reviewed-by: Matthew Dempsky <mdempsky@google.com>
2016-12-09 17:15:05 -08:00
// lineno is the source position at the start of the most recently lexed token.
// TODO(gri) rename and eventually remove
2016-12-15 17:17:01 -08:00
var lineno src.XPos
func makePos(base *src.PosBase, line, col uint) src.XPos {
2016-12-15 17:17:01 -08:00
return Ctxt.PosTable.XPos(src.MakePos(base, line, col))
}
func isSpace(c rune) bool {
return c == ' ' || c == '\t' || c == '\n' || c == '\r'
}
func isQuoted(s string) bool {
return len(s) >= 2 && s[0] == '"' && s[len(s)-1] == '"'
}
func plan9quote(s string) string {
if s == "" {
return "''"
}
for _, c := range s {
if c <= ' ' || c == '\'' {
return "'" + strings.Replace(s, "'", "''", -1) + "'"
}
}
return s
}
const (
cmd/compile: add go:notinheap type pragma This adds a //go:notinheap pragma for declarations of types that must not be heap allocated. We ensure these rules by disallowing new(T), make([]T), append([]T), or implicit allocation of T, by disallowing conversions to notinheap types, and by propagating notinheap to any struct or array that contains notinheap elements. The utility of this pragma is that we can eliminate write barriers for writes to pointers to go:notinheap types, since the write barrier is guaranteed to be a no-op. This will let us mark several scheduler and memory allocator structures as go:notinheap, which will let us disallow write barriers in the scheduler and memory allocator much more thoroughly and also eliminate some problematic hybrid write barriers. This also makes go:nowritebarrierrec and go:yeswritebarrierrec much more powerful. Currently we use go:nowritebarrier all over the place, but it's almost never what you actually want: when write barriers are illegal, they're typically illegal for a whole dynamic scope. Partly this is because go:nowritebarrier has been around longer, but it's also because go:nowritebarrierrec couldn't be used in situations that had no-op write barriers or where some nested scope did allow write barriers. go:notinheap eliminates many no-op write barriers and go:yeswritebarrierrec makes it possible to opt back in to write barriers, so these two changes will let us use go:nowritebarrierrec far more liberally. This updates #13386, which is about controlling pointers from non-GC'd memory to GC'd memory. That would require some additional pragma (or pragmas), but could build on this pragma. Change-Id: I6314f8f4181535dd166887c9ec239977b54940bd Reviewed-on: https://go-review.googlesource.com/30939 Reviewed-by: Keith Randall <khr@golang.org> Reviewed-by: Matthew Dempsky <mdempsky@google.com>
2016-10-11 22:53:27 -04:00
// Func pragmas.
Nointerface syntax.Pragma = 1 << iota
Noescape // func parameters don't escape
Norace // func must not have race detector annotations
Nosplit // func should not execute on separate stack
Noinline // func should not be inlined
CgoUnsafeArgs // treat a pointer to one arg as a pointer to them all
UintptrEscapes // pointers converted to uintptr escape
cmd/compile: add go:notinheap type pragma This adds a //go:notinheap pragma for declarations of types that must not be heap allocated. We ensure these rules by disallowing new(T), make([]T), append([]T), or implicit allocation of T, by disallowing conversions to notinheap types, and by propagating notinheap to any struct or array that contains notinheap elements. The utility of this pragma is that we can eliminate write barriers for writes to pointers to go:notinheap types, since the write barrier is guaranteed to be a no-op. This will let us mark several scheduler and memory allocator structures as go:notinheap, which will let us disallow write barriers in the scheduler and memory allocator much more thoroughly and also eliminate some problematic hybrid write barriers. This also makes go:nowritebarrierrec and go:yeswritebarrierrec much more powerful. Currently we use go:nowritebarrier all over the place, but it's almost never what you actually want: when write barriers are illegal, they're typically illegal for a whole dynamic scope. Partly this is because go:nowritebarrier has been around longer, but it's also because go:nowritebarrierrec couldn't be used in situations that had no-op write barriers or where some nested scope did allow write barriers. go:notinheap eliminates many no-op write barriers and go:yeswritebarrierrec makes it possible to opt back in to write barriers, so these two changes will let us use go:nowritebarrierrec far more liberally. This updates #13386, which is about controlling pointers from non-GC'd memory to GC'd memory. That would require some additional pragma (or pragmas), but could build on this pragma. Change-Id: I6314f8f4181535dd166887c9ec239977b54940bd Reviewed-on: https://go-review.googlesource.com/30939 Reviewed-by: Keith Randall <khr@golang.org> Reviewed-by: Matthew Dempsky <mdempsky@google.com>
2016-10-11 22:53:27 -04:00
// Runtime-only func pragmas.
// See ../../../../runtime/README.md for detailed descriptions.
Systemstack // func must run on system stack
Nowritebarrier // emit compiler error instead of write barrier
Nowritebarrierrec // error on write barrier in this or recursive callees
Yeswritebarrierrec // cancels Nowritebarrierrec in this function and callees
cmd/compile: add go:notinheap type pragma This adds a //go:notinheap pragma for declarations of types that must not be heap allocated. We ensure these rules by disallowing new(T), make([]T), append([]T), or implicit allocation of T, by disallowing conversions to notinheap types, and by propagating notinheap to any struct or array that contains notinheap elements. The utility of this pragma is that we can eliminate write barriers for writes to pointers to go:notinheap types, since the write barrier is guaranteed to be a no-op. This will let us mark several scheduler and memory allocator structures as go:notinheap, which will let us disallow write barriers in the scheduler and memory allocator much more thoroughly and also eliminate some problematic hybrid write barriers. This also makes go:nowritebarrierrec and go:yeswritebarrierrec much more powerful. Currently we use go:nowritebarrier all over the place, but it's almost never what you actually want: when write barriers are illegal, they're typically illegal for a whole dynamic scope. Partly this is because go:nowritebarrier has been around longer, but it's also because go:nowritebarrierrec couldn't be used in situations that had no-op write barriers or where some nested scope did allow write barriers. go:notinheap eliminates many no-op write barriers and go:yeswritebarrierrec makes it possible to opt back in to write barriers, so these two changes will let us use go:nowritebarrierrec far more liberally. This updates #13386, which is about controlling pointers from non-GC'd memory to GC'd memory. That would require some additional pragma (or pragmas), but could build on this pragma. Change-Id: I6314f8f4181535dd166887c9ec239977b54940bd Reviewed-on: https://go-review.googlesource.com/30939 Reviewed-by: Keith Randall <khr@golang.org> Reviewed-by: Matthew Dempsky <mdempsky@google.com>
2016-10-11 22:53:27 -04:00
// Runtime-only type pragmas
NotInHeap // values of this type must not be heap allocated
)
func pragmaValue(verb string) syntax.Pragma {
switch verb {
case "go:nointerface":
if objabi.Fieldtrack_enabled != 0 {
return Nointerface
}
case "go:noescape":
return Noescape
case "go:norace":
return Norace
case "go:nosplit":
return Nosplit
case "go:noinline":
return Noinline
case "go:systemstack":
return Systemstack
case "go:nowritebarrier":
return Nowritebarrier
case "go:nowritebarrierrec":
return Nowritebarrierrec | Nowritebarrier // implies Nowritebarrier
case "go:yeswritebarrierrec":
return Yeswritebarrierrec
case "go:cgo_unsafe_args":
return CgoUnsafeArgs
case "go:uintptrescapes":
// For the next function declared in the file
// any uintptr arguments may be pointer values
// converted to uintptr. This directive
// ensures that the referenced allocated
// object, if any, is retained and not moved
// until the call completes, even though from
// the types alone it would appear that the
// object is no longer needed during the
// call. The conversion to uintptr must appear
// in the argument list.
// Used in syscall/dll_windows.go.
return UintptrEscapes
cmd/compile: add go:notinheap type pragma This adds a //go:notinheap pragma for declarations of types that must not be heap allocated. We ensure these rules by disallowing new(T), make([]T), append([]T), or implicit allocation of T, by disallowing conversions to notinheap types, and by propagating notinheap to any struct or array that contains notinheap elements. The utility of this pragma is that we can eliminate write barriers for writes to pointers to go:notinheap types, since the write barrier is guaranteed to be a no-op. This will let us mark several scheduler and memory allocator structures as go:notinheap, which will let us disallow write barriers in the scheduler and memory allocator much more thoroughly and also eliminate some problematic hybrid write barriers. This also makes go:nowritebarrierrec and go:yeswritebarrierrec much more powerful. Currently we use go:nowritebarrier all over the place, but it's almost never what you actually want: when write barriers are illegal, they're typically illegal for a whole dynamic scope. Partly this is because go:nowritebarrier has been around longer, but it's also because go:nowritebarrierrec couldn't be used in situations that had no-op write barriers or where some nested scope did allow write barriers. go:notinheap eliminates many no-op write barriers and go:yeswritebarrierrec makes it possible to opt back in to write barriers, so these two changes will let us use go:nowritebarrierrec far more liberally. This updates #13386, which is about controlling pointers from non-GC'd memory to GC'd memory. That would require some additional pragma (or pragmas), but could build on this pragma. Change-Id: I6314f8f4181535dd166887c9ec239977b54940bd Reviewed-on: https://go-review.googlesource.com/30939 Reviewed-by: Keith Randall <khr@golang.org> Reviewed-by: Matthew Dempsky <mdempsky@google.com>
2016-10-11 22:53:27 -04:00
case "go:notinheap":
return NotInHeap
}
return 0
}
// pragcgo is called concurrently if files are parsed concurrently.
cmd/compile/internal/syntax: remove dependency on cmd/internal/src For dependency reasons, the data structure implementing source positions in the compiler is in cmd/internal/src. It contains highly compiler specific details (e.g. inlining index). This change introduces a parallel but simpler position representation, defined in the syntax package, which removes that package's dependency on cmd/internal/src, and also removes the need to deal with certain filename-specific operations (defined by the needs of the compiler) in the syntax package. As a result, the syntax package becomes again a compiler- independent, stand-alone package that at some point might replace (or augment) the existing top-level go/* syntax-related packages. Additionally, line directives that update column numbers are now correctly tracked through the syntax package, with additional tests added. (The respective changes also need to be made in cmd/internal/src; i.e., the compiler accepts but still ignores column numbers in line directives.) This change comes at the cost of a new position translation step, but that step is cheap because it only needs to do real work if the position base changed (i.e., if there is a new file, or new line directive). There is no noticeable impact on overall compiler performance measured with `compilebench -count 5 -alloc`: name old time/op new time/op delta Template 220ms ± 8% 228ms ±18% ~ (p=0.548 n=5+5) Unicode 119ms ±11% 113ms ± 5% ~ (p=0.056 n=5+5) GoTypes 684ms ± 6% 677ms ± 3% ~ (p=0.841 n=5+5) Compiler 3.19s ± 7% 3.01s ± 1% ~ (p=0.095 n=5+5) SSA 7.92s ± 8% 7.79s ± 1% ~ (p=0.690 n=5+5) Flate 141ms ± 7% 139ms ± 4% ~ (p=0.548 n=5+5) GoParser 173ms ±12% 171ms ± 4% ~ (p=1.000 n=5+5) Reflect 417ms ± 5% 411ms ± 3% ~ (p=0.548 n=5+5) Tar 205ms ± 5% 198ms ± 2% ~ (p=0.690 n=5+5) XML 232ms ± 4% 229ms ± 4% ~ (p=0.690 n=5+5) StdCmd 28.7s ± 5% 28.2s ± 2% ~ (p=0.421 n=5+5) name old user-time/op new user-time/op delta Template 269ms ± 4% 265ms ± 5% ~ (p=0.421 n=5+5) Unicode 153ms ± 7% 149ms ± 3% ~ (p=0.841 n=5+5) GoTypes 850ms ± 7% 862ms ± 4% ~ (p=0.841 n=5+5) Compiler 4.01s ± 5% 3.86s ± 0% ~ (p=0.190 n=5+4) SSA 10.9s ± 4% 10.8s ± 2% ~ (p=0.548 n=5+5) Flate 166ms ± 7% 167ms ± 6% ~ (p=1.000 n=5+5) GoParser 204ms ± 8% 206ms ± 7% ~ (p=0.841 n=5+5) Reflect 514ms ± 5% 508ms ± 4% ~ (p=0.548 n=5+5) Tar 245ms ± 6% 244ms ± 3% ~ (p=0.690 n=5+5) XML 280ms ± 4% 278ms ± 4% ~ (p=0.841 n=5+5) name old alloc/op new alloc/op delta Template 37.9MB ± 0% 37.9MB ± 0% ~ (p=0.841 n=5+5) Unicode 28.8MB ± 0% 28.8MB ± 0% ~ (p=0.841 n=5+5) GoTypes 113MB ± 0% 113MB ± 0% ~ (p=0.151 n=5+5) Compiler 468MB ± 0% 468MB ± 0% -0.01% (p=0.032 n=5+5) SSA 1.50GB ± 0% 1.50GB ± 0% ~ (p=0.548 n=5+5) Flate 24.4MB ± 0% 24.4MB ± 0% ~ (p=1.000 n=5+5) GoParser 30.7MB ± 0% 30.7MB ± 0% ~ (p=1.000 n=5+5) Reflect 76.5MB ± 0% 76.5MB ± 0% ~ (p=0.548 n=5+5) Tar 38.9MB ± 0% 38.9MB ± 0% ~ (p=0.222 n=5+5) XML 41.6MB ± 0% 41.6MB ± 0% ~ (p=0.548 n=5+5) name old allocs/op new allocs/op delta Template 382k ± 0% 382k ± 0% +0.01% (p=0.008 n=5+5) Unicode 343k ± 0% 343k ± 0% ~ (p=0.841 n=5+5) GoTypes 1.19M ± 0% 1.19M ± 0% +0.01% (p=0.008 n=5+5) Compiler 4.53M ± 0% 4.53M ± 0% +0.03% (p=0.008 n=5+5) SSA 12.4M ± 0% 12.4M ± 0% +0.00% (p=0.008 n=5+5) Flate 235k ± 0% 235k ± 0% ~ (p=0.079 n=5+5) GoParser 318k ± 0% 318k ± 0% ~ (p=0.730 n=5+5) Reflect 978k ± 0% 978k ± 0% ~ (p=1.000 n=5+5) Tar 393k ± 0% 393k ± 0% ~ (p=0.056 n=5+5) XML 405k ± 0% 405k ± 0% ~ (p=0.548 n=5+5) name old text-bytes new text-bytes delta HelloSize 672kB ± 0% 672kB ± 0% ~ (all equal) CmdGoSize 7.12MB ± 0% 7.12MB ± 0% ~ (all equal) name old data-bytes new data-bytes delta HelloSize 133kB ± 0% 133kB ± 0% ~ (all equal) CmdGoSize 390kB ± 0% 390kB ± 0% ~ (all equal) name old exe-bytes new exe-bytes delta HelloSize 1.07MB ± 0% 1.07MB ± 0% ~ (all equal) CmdGoSize 11.2MB ± 0% 11.2MB ± 0% ~ (all equal) Passes toolstash compare. For #22662. Change-Id: I19edb53dd9675af57f7122cb7dba2a6d8bdcc3da Reviewed-on: https://go-review.googlesource.com/94515 Reviewed-by: Matthew Dempsky <mdempsky@google.com>
2018-01-02 16:58:37 -08:00
func (p *noder) pragcgo(pos syntax.Pos, text string) string {
f := pragmaFields(text)
verb := f[0][3:] // skip "go:"
switch verb {
case "cgo_export_static", "cgo_export_dynamic":
switch {
case len(f) == 2 && !isQuoted(f[1]):
local := plan9quote(f[1])
return fmt.Sprintln(verb, local)
case len(f) == 3 && !isQuoted(f[1]) && !isQuoted(f[2]):
local := plan9quote(f[1])
remote := plan9quote(f[2])
return fmt.Sprintln(verb, local, remote)
default:
p.error(syntax.Error{Pos: pos, Msg: fmt.Sprintf(`usage: //go:%s local [remote]`, verb)})
}
case "cgo_import_dynamic":
switch {
case len(f) == 2 && !isQuoted(f[1]):
local := plan9quote(f[1])
return fmt.Sprintln(verb, local)
case len(f) == 3 && !isQuoted(f[1]) && !isQuoted(f[2]):
local := plan9quote(f[1])
remote := plan9quote(f[2])
return fmt.Sprintln(verb, local, remote)
case len(f) == 4 && !isQuoted(f[1]) && !isQuoted(f[2]) && isQuoted(f[3]):
local := plan9quote(f[1])
remote := plan9quote(f[2])
library := plan9quote(strings.Trim(f[3], `"`))
return fmt.Sprintln(verb, local, remote, library)
default:
p.error(syntax.Error{Pos: pos, Msg: `usage: //go:cgo_import_dynamic local [remote ["library"]]`})
}
case "cgo_import_static":
switch {
case len(f) == 2 && !isQuoted(f[1]):
local := plan9quote(f[1])
return fmt.Sprintln(verb, local)
default:
p.error(syntax.Error{Pos: pos, Msg: `usage: //go:cgo_import_static local`})
}
case "cgo_dynamic_linker":
switch {
case len(f) == 2 && isQuoted(f[1]):
path := plan9quote(strings.Trim(f[1], `"`))
return fmt.Sprintln(verb, path)
default:
p.error(syntax.Error{Pos: pos, Msg: `usage: //go:cgo_dynamic_linker "path"`})
}
case "cgo_ldflag":
switch {
case len(f) == 2 && isQuoted(f[1]):
arg := plan9quote(strings.Trim(f[1], `"`))
return fmt.Sprintln(verb, arg)
default:
p.error(syntax.Error{Pos: pos, Msg: `usage: //go:cgo_ldflag "arg"`})
}
}
return ""
}
// pragmaFields is similar to strings.FieldsFunc(s, isSpace)
// but does not split when inside double quoted regions and always
// splits before the start and after the end of a double quoted region.
// pragmaFields does not recognize escaped quotes. If a quote in s is not
// closed the part after the opening quote will not be returned as a field.
func pragmaFields(s string) []string {
var a []string
inQuote := false
fieldStart := -1 // Set to -1 when looking for start of field.
for i, c := range s {
switch {
case c == '"':
if inQuote {
inQuote = false
a = append(a, s[fieldStart:i+1])
fieldStart = -1
} else {
inQuote = true
if fieldStart >= 0 {
a = append(a, s[fieldStart:i])
}
fieldStart = i
}
case !inQuote && isSpace(c):
if fieldStart >= 0 {
a = append(a, s[fieldStart:i])
fieldStart = -1
}
default:
if fieldStart == -1 {
fieldStart = i
}
}
}
if !inQuote && fieldStart >= 0 { // Last field might end at the end of the string.
a = append(a, s[fieldStart:])
}
return a
}