Stowage/go
2
0
Fork 0
mirror of https://github.com/golang/go.git synced 2025-12-08 06:10:04 +00:00
Code Issues Projects Releases Packages Wiki Activity

[dev.inline] cmd/compile/internal/syntax: process //line pragmas in scanner

Browse source 
Reviewed in and cherry-picked from https://go-review.googlesource.com/#/c/33764/.

Minor adjustment in noder.go to make merge compile again.

Change-Id: Ib5029b52b59944f207b0f2438c8a5aa576eb25b8
Reviewed-on: https://go-review.googlesource.com/34233
Run-TryBot: Robert Griesemer <gri@golang.org>
Reviewed-by: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
This commit is contained in:
Robert Griesemer 2016-11-30 23:28:40 -08:00
parent 8d20b25779
commit 32bf2829a1
8 changed files with 89 additions and 54 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/cmd/compile/internal/gc/noder.go
View file

@ -23,7 +23,7 @@ func parseFile(filename string) {
defer src.Close()
p := noder{baseline: lexlineno.Line()}
file, _ := syntax.Parse(src, p.error, p.pragma, 0) // errors are tracked via p.error
file, _ := syntax.Parse(filename, src, p.error, p.pragma, 0) // errors are tracked via p.error
p.file(file)

2
src/cmd/compile/internal/syntax/nodes.go
View file

@ -26,7 +26,7 @@ func (n *node) Pos() *Pos {
func (*node) aNode() {}
func (n *node) init(p *parser) {
n.pos = MakePos(nil, p.line, p.col)
n.pos = MakePos(p.base, p.line, p.col)
}
// ----------------------------------------------------------------------------

4
src/cmd/compile/internal/syntax/parser.go
View file

@ -26,8 +26,8 @@ type parser struct {
indent []byte // tracing support
}
func (p *parser) init(src io.Reader, errh ErrorHandler, pragh PragmaHandler) {
p.scanner.init(src, errh, pragh)
func (p *parser) init(filename string, src io.Reader, errh ErrorHandler, pragh PragmaHandler) {
p.scanner.init(filename, src, errh, pragh)
p.fnest = 0
p.xnest = 0

4
src/cmd/compile/internal/syntax/parser_test.go
View file

@ -133,7 +133,7 @@ func verifyPrint(filename string, ast1 *File) {
panic(err)
}
ast2, err := ParseBytes(buf1.Bytes(), nil, nil, 0)
ast2, err := ParseBytes(filename, buf1.Bytes(), nil, nil, 0)
if err != nil {
panic(err)
}
@ -157,7 +157,7 @@ func verifyPrint(filename string, ast1 *File) {
}
func TestIssue17697(t *testing.T) {
_, err := ParseBytes(nil, nil, nil, 0) // return with parser error, don't panic
_, err := ParseBytes("", nil, nil, nil, 0) // return with parser error, don't panic
if err == nil {
t.Errorf("no error reported")
}

12
src/cmd/compile/internal/syntax/pos.go
View file

@ -103,13 +103,15 @@ func NewLinePragmaBase(pos Pos, filename string, line uint) *PosBase {
return &PosBase{pos, filename, line - 1}
}
var noPos Pos
// Pos returns the position at which base is located.
// If b == nil, the result is the empty position.
func (b *PosBase) Pos() Pos {
func (b *PosBase) Pos() *Pos {
if b != nil {
return b.pos
return &b.pos
}
return Pos{}
return &noPos
}
// Filename returns the filename recorded with the base.
@ -136,13 +138,13 @@ func (b *PosBase) Line() uint {
// A lico is a compact encoding of a LIne and COlumn number.
type lico uint32
// Layout constants: 23 bits for line, 9 bits for column.
// Layout constants: 24 bits for line, 8 bits for column.
// (If this is too tight, we can either make lico 64b wide,
// or we can introduce a tiered encoding where we remove column
// information as line numbers grow bigger; similar to what gcc
// does.)
const (
lineW, lineM = 23, 1<<lineW - 1
lineW, lineM = 24, 1<<lineW - 1
colW, colM = 32 - lineW, 1<<colW - 1
)

93
src/cmd/compile/internal/syntax/scanner.go
View file

@ -7,6 +7,7 @@ package syntax
import (
"fmt"
"io"
"strconv"
"strings"
"unicode"
"unicode/utf8"
@ -18,6 +19,7 @@ type scanner struct {
pragma Pragma
// current token, valid after calling next()
base *PosBase
line, col uint
tok token
lit string // valid if tok is _Name or _Literal
@ -28,9 +30,10 @@ type scanner struct {
pragh PragmaHandler
}
func (s *scanner) init(src io.Reader, errh ErrorHandler, pragh PragmaHandler) {
func (s *scanner) init(filename string, src io.Reader, errh ErrorHandler, pragh PragmaHandler) {
s.source.init(src, errh)
s.nlsemi = false
s.base = NewFileBase(filename)
s.pragh = pragh
}
@ -524,48 +527,70 @@ func (s *scanner) rune() {
s.tok = _Literal
}
func (s *scanner) lineComment() {
// recognize pragmas
var prefix string
r := s.getr()
if s.pragh == nil {
goto skip
}
switch r {
case 'g':
prefix = "go:"
case 'l':
prefix = "line "
default:
goto skip
}
s.startLit()
for _, m := range prefix {
if r != m {
s.stopLit()
goto skip
}
r = s.getr()
}
func (s *scanner) skipLine(r rune) {
for r >= 0 {
if r == '\n' {
s.ungetr()
s.ungetr() // don't consume '\n' - needed for nlsemi logic
break
}
r = s.getr()
}
s.pragma |= s.pragh(s.line, strings.TrimSuffix(string(s.stopLit()), "\r"))
return
}
skip:
// consume line
for r != '\n' && r >= 0 {
func (s *scanner) lineComment() {
// recognize pragmas
prefix := ""
r := s.getr()
switch r {
case 'g':
if s.pragh == nil {
s.skipLine(r)
return
}
prefix = "go:"
case 'l':
prefix = "line "
default:
s.skipLine(r)
return
}
for _, m := range prefix {
if r != m {
s.skipLine(r)
return
}
r = s.getr()
}
s.ungetr() // don't consume '\n' - needed for nlsemi logic
// pragma text without prefix and line ending (which may be "\r\n" if Windows)
s.startLit()
s.skipLine(r)
text := strings.TrimSuffix(string(s.stopLit()), "\r")
// process //line filename:line pragma
if prefix[0] == 'l' {
// Want to use LastIndexByte below but it's not defined in Go1.4 and bootstrap fails.
i := strings.LastIndex(text, ":") // look from right (Windows filenames may contain ':')
if i < 0 {
return
}
nstr := text[i+1:]
n, err := strconv.Atoi(nstr)
if err != nil || n <= 0 || n > lineM {
s.error_at(s.line0, s.col0-uint(len(nstr)), "invalid line number: "+nstr)
return
}
s.base = NewLinePragmaBase(MakePos(s.base.Pos().Base(), s.line, s.col), text[:i], uint(n))
// TODO(gri) Return here once we rely exclusively
// on node positions for line number information,
// and remove //line pragma handling elsewhere.
if s.pragh == nil {
return
}
}
s.pragma |= s.pragh(s.line, prefix+text)
}
func (s *scanner) fullComment() {

14
src/cmd/compile/internal/syntax/scanner_test.go
View file

@ -22,7 +22,7 @@ func TestScanner(t *testing.T) {
defer src.Close()
var s scanner
s.init(src, nil, nil)
s.init("parser.go", src, nil, nil)
for {
s.next()
if s.tok == _EOF {
@ -51,7 +51,7 @@ func TestTokens(t *testing.T) {
// scan source
var got scanner
got.init(&bytesReader{buf}, nil, nil)
got.init("", &bytesReader{buf}, nil, nil)
got.next()
for i, want := range sampleTokens {
nlsemi := false
@ -317,12 +317,20 @@ func TestScanErrors(t *testing.T) {
{`var s string = "\x"`, "non-hex character in escape sequence: \"", 1, 19},
{`return "\Uffffffff"`, "escape sequence is invalid Unicode code point", 1, 19},
{`//line :`, "invalid line number: ", 1, 9},
{`//line :x`, "invalid line number: x", 1, 9},
{`//line foo :`, "invalid line number: ", 1, 13},
{`//line foo:123abc`, "invalid line number: 123abc", 1, 12},
{`/**///line foo:x`, "invalid line number: x", 1, 16},
{`//line foo:0`, "invalid line number: 0", 1, 12},
{fmt.Sprintf(`//line foo:%d`, lineM+1), fmt.Sprintf("invalid line number: %d", lineM+1), 1, 12},
// former problem cases
{"package p\n\n\xef", "invalid UTF-8 encoding", 3, 1},
} {
var s scanner
nerrors := 0
s.init(&bytesReader{[]byte(test.src)}, func(err error) {
s.init("", &bytesReader{[]byte(test.src)}, func(err error) {
nerrors++
// only check the first error
e := err.(Error) // we know it's an Error

12
src/cmd/compile/internal/syntax/syntax.go
View file

@ -41,7 +41,7 @@ type PragmaHandler func(line uint, text string) Pragma
// Parse parses a single Go source file from src and returns the corresponding
// syntax tree. If there are syntax errors, Parse will return the first error
// encountered.
// encountered. The filename is only used for position information.
//
// If errh != nil, it is called with each error encountered, and Parse will
// process as much source as possible. If errh is nil, Parse will terminate
@ -50,7 +50,7 @@ type PragmaHandler func(line uint, text string) Pragma
// If a PragmaHandler is provided, it is called with each pragma encountered.
//
// The Mode argument is currently ignored.
func Parse(src io.Reader, errh ErrorHandler, pragh PragmaHandler, mode Mode) (_ *File, err error) {
func Parse(filename string, src io.Reader, errh ErrorHandler, pragh PragmaHandler, mode Mode) (_ *File, err error) {
defer func() {
if p := recover(); p != nil {
var ok bool
@ -62,14 +62,14 @@ func Parse(src io.Reader, errh ErrorHandler, pragh PragmaHandler, mode Mode) (_
}()
var p parser
p.init(src, errh, pragh)
p.init(filename, src, errh, pragh)
p.next()
return p.file(), p.first
}
// ParseBytes behaves like Parse but it reads the source from the []byte slice provided.
func ParseBytes(src []byte, errh ErrorHandler, pragh PragmaHandler, mode Mode) (*File, error) {
return Parse(&bytesReader{src}, errh, pragh, mode)
func ParseBytes(filename string, src []byte, errh ErrorHandler, pragh PragmaHandler, mode Mode) (*File, error) {
return Parse(filename, &bytesReader{src}, errh, pragh, mode)
}
type bytesReader struct {
@ -95,5 +95,5 @@ func ParseFile(filename string, errh ErrorHandler, pragh PragmaHandler, mode Mod
return nil, err
}
defer src.Close()
return Parse(src, errh, pragh, mode)
return Parse(filename, src, errh, pragh, mode)
}