Stowage/go
2
0
Fork 0
mirror of https://github.com/golang/go.git synced 2025-12-08 06:10:04 +00:00
Code Issues Projects Releases Packages Wiki Activity

go/ast: use token.Pos instead of token.Position; adjust all dependent code

Browse source 
Specifically:

	* lib/godoc:
	- provide file set (FSet) argument to formatters where needed

	* src/cmd:
	- cgo, ebnflint, godoc, gofmt, goinstall: provide file set (fset) where needed
	- godoc: remove local binary search with sort.Search (change by rsc),
	  extract file set for formatters

	* src/pkg:
	- exp/eval: remove embedded token.Position fields from nodes and replace
	  with named token.Pos fields; add corresponding Pos() accessor methods
	- go/token: added file.Line(), changed signature of File.Position()

	* test/fixedbugs/:
	- bug206.go: change test to not rely on token.Pos details

	* added various extra comments
	* Runs all.bash
	* gofmt formats all of src, misc w/o changes
	* godoc runs

	* performance:
	- The new version of godoc consumes about the same space after indexing
	  has completed, but indexing is half the speed. Significant space savings
	  are expected from smaller ASTs, but since they are thrown away after a
	  file has been indexed, this is not visible anymore. The slower indexing
	  time is due to the much more expensive computation of line information.
	  However, with the new compressed position information, indexing can be
	  rewritten and simplified. Furthermore, computing the line info can be
	  done more efficiently.

        New godoc, immediately after indexing completed (best of three runs):

	  PID COMMAND      %CPU   TIME   #TH #PRTS #MREGS RPRVT  RSHRD  RSIZE  VSIZE
	44381 godoc        0.0%  0:38.00   4    19    149  145M   184K   148M   176M

	2010/12/03 17:58:35 index updated (39.231s, 18505 unique words, 386387 spots)
	2010/12/03 17:58:35 bytes=90858456 footprint=199182584
	2010/12/03 17:58:36 bytes=47858568 footprint=167295224

	Old godoc, immediately after indexing completed (best of three runs):

	  PID COMMAND      %CPU   TIME   #TH #PRTS #MREGS RPRVT  RSHRD  RSIZE  VSIZE
	23167 godoc        0.0%  0:22.02   4    17    132  129M   184K   132M   173M

	2010/12/03 14:51:32 index updated (24.892s, 18765 unique words, 393830 spots)
	2010/12/03 14:51:32 bytes=66404528 footprint=163907832
	2010/12/03 14:51:32 bytes=46282224 footprint=163907832

	The different numbers for unique words/spots stem from the fact the the
	two workspaces are not exactly identical. The new godoc maintains a large
	file set data structure during indexing which (probably) is the reason
	for the larger heap (90858456 vs 66404528) before garbage collection.

R=rsc, r
CC=golang-dev
https://golang.org/cl/3050041
This commit is contained in:
Robert Griesemer 2010-12-06 14:23:18 -08:00
parent 92bfd850e0
commit 5a9ad8b9e3
51 changed files with 782 additions and 691 deletions
Download patch file Download diff file Expand all files Collapse all files

34
lib/godoc/package.html
View file

@ -6,17 +6,19 @@
{.section PAst} {.section PAst}
<pre> <pre>
{@|html} {@ FSet|html}
</pre> </pre>
{.end} {.end}
{.section PDoc} {.section PDoc}
<!-- PackageName is printed as title by the top-level template --> <!-- PackageName is printed as title by the top-level template -->
{.section IsPkg} {.section IsPkg}
{# ImportPath is a string - no need for FSet}
<p><code>import "{ImportPath|html-esc}"</code></p> <p><code>import "{ImportPath|html-esc}"</code></p>
{.end} {.end}
{Doc|html-comment} {Doc|html-comment}
{.section IsPkg} {.section IsPkg}
{.section Filenames} {.section Filenames}
{# Filenames are strings - no need for FSet}
<p> <p>
<h4>Package files</h4> <h4>Package files</h4>
<span style="font-size:90%"> <span style="font-size:90%">
@ -31,44 +33,46 @@
<h2 id="Constants">Constants</h2> <h2 id="Constants">Constants</h2>
{.repeated section @} {.repeated section @}
{Doc|html-comment} {Doc|html-comment}
<pre>{Decl|html}</pre> <pre>{Decl FSet|html}</pre>
{.end} {.end}
{.end} {.end}
{.section Vars} {.section Vars}
<h2 id="Variables">Variables</h2> <h2 id="Variables">Variables</h2>
{.repeated section @} {.repeated section @}
{Doc|html-comment} {Doc|html-comment}
<pre>{Decl|html}</pre> <pre>{Decl FSet|html}</pre>
{.end} {.end}
{.end} {.end}
{.section Funcs} {.section Funcs}
{.repeated section @} {.repeated section @}
<h2 id="{Name|html-esc}">func <a href="/{Decl|url-pos}">{Name|html-esc}</a></h2> {# Name is a string - no need for FSet}
<p><code>{Decl|html}</code></p> <h2 id="{Name|html-esc}">func <a href="/{Decl FSet|url-pos}">{Name|html-esc}</a></h2>
<p><code>{Decl FSet|html}</code></p>
{Doc|html-comment} {Doc|html-comment}
{.end} {.end}
{.end} {.end}
{.section Types} {.section Types}
{.repeated section @} {.repeated section @}
<h2 id="{Type.Name|html-esc}">type <a href="/{Decl|url-pos}">{Type.Name|html-esc}</a></h2> {# Type.Name is a string - no need for FSet}
<h2 id="{Type.Name FSet|html-esc}">type <a href="/{Decl FSet|url-pos}">{Type.Name FSet|html-esc}</a></h2>
{Doc|html-comment} {Doc|html-comment}
<p><pre>{Decl|html}</pre></p> <p><pre>{Decl FSet|html}</pre></p>
{.repeated section Consts} {.repeated section Consts}
{Doc|html-comment} {Doc|html-comment}
<pre>{Decl|html}</pre> <pre>{Decl FSet|html}</pre>
{.end} {.end}
{.repeated section Vars} {.repeated section Vars}
{Doc|html-comment} {Doc|html-comment}
<pre>{Decl|html}</pre> <pre>{Decl FSet|html}</pre>
{.end} {.end}
{.repeated section Factories} {.repeated section Factories}
<h3 id="{Type.Name|html-esc}.{Name|html-esc}">func <a href="/{Decl|url-pos}">{Name|html-esc}</a></h3> <h3 id="{Type.Name FSet|html-esc}.{Name|html-esc}">func <a href="/{Decl FSet|url-pos}">{Name|html-esc}</a></h3>
<p><code>{Decl|html}</code></p> <p><code>{Decl FSet|html}</code></p>
{Doc|html-comment} {Doc|html-comment}
{.end} {.end}
{.repeated section Methods} {.repeated section Methods}
<h3 id="{Type.Name|html-esc}.{Name|html-esc}">func ({Recv|html}) <a href="/{Decl|url-pos}">{Name|html-esc}</a></h3> <h3 id="{Type.Name FSet|html-esc}.{Name|html-esc}">func ({Recv FSet|html}) <a href="/{Decl FSet|url-pos}">{Name|html-esc}</a></h3>
<p><code>{Decl|html}</code></p> <p><code>{Decl FSet|html}</code></p>
{Doc|html-comment} {Doc|html-comment}
{.end} {.end}
{.end} {.end}
@ -83,12 +87,14 @@
{.section PList} {.section PList}
<h2>Other packages</h2> <h2>Other packages</h2>
<p> <p>
{# PLIst entries are strings - no need for FSet}
{.repeated section @} {.repeated section @}
<a href="?p={@|html}">{@|html}</a><br /> <a href="?p={@|html-esc}">{@|html-esc}</a><br />
{.end} {.end}
</p> </p>
{.end} {.end}
{.section Dirs} {.section Dirs}
{# DirList entries are numbers and strings - no need for FSet}
<h2 id="Subdirectories">Subdirectories</h2> <h2 id="Subdirectories">Subdirectories</h2>
<p> <p>
<table class="layout"> <table class="layout">

16
lib/godoc/package.txt
View file

@ -19,7 +19,7 @@ COMMAND DOCUMENTATION
CONSTANTS CONSTANTS
{.repeated section @} {.repeated section @}
{Decl} {Decl FSet}
{Doc} {Doc}
{.end} {.end}
{.end} {.end}
@ -28,7 +28,7 @@ CONSTANTS
VARIABLES VARIABLES
{.repeated section @} {.repeated section @}
{Decl} {Decl FSet}
{Doc} {Doc}
{.end} {.end}
{.end} {.end}
@ -37,7 +37,7 @@ VARIABLES
FUNCTIONS FUNCTIONS
{.repeated section @} {.repeated section @}
{Decl} {Decl FSet}
{Doc} {Doc}
{.end} {.end}
{.end} {.end}
@ -46,22 +46,22 @@ FUNCTIONS
TYPES TYPES
{.repeated section @} {.repeated section @}
{Decl} {Decl FSet}
{Doc} {Doc}
{.repeated section Consts} {.repeated section Consts}
{Decl} {Decl FSet}
{Doc} {Doc}
{.end} {.end}
{.repeated section Vars} {.repeated section Vars}
{Decl} {Decl FSet}
{Doc} {Doc}
{.end} {.end}
{.repeated section Factories} {.repeated section Factories}
{Decl} {Decl FSet}
{Doc} {Doc}
{.end} {.end}
{.repeated section Methods} {.repeated section Methods}
{Decl} {Decl FSet}
{Doc} {Doc}
{.end} {.end}
{.end} {.end}

7
src/cmd/cgo/ast.go
View file

@ -12,12 +12,13 @@ import (
"go/doc" "go/doc"
"go/parser" "go/parser"
"go/scanner" "go/scanner"
"go/token"
"os" "os"
"strings" "strings"
) )
func parse(name string, flags uint) *ast.File { func parse(name string, flags uint) *ast.File {
ast1, err := parser.ParseFile(name, nil, flags) ast1, err := parser.ParseFile(fset, name, nil, flags)
if err != nil { if err != nil {
if list, ok := err.(scanner.ErrorList); ok { if list, ok := err.(scanner.ErrorList); ok {
// If err is a scanner.ErrorList, its String will print just // If err is a scanner.ErrorList, its String will print just
@ -76,7 +77,7 @@ func (f *File) ReadGo(name string) {
} }
} }
if !sawC { if !sawC {
error(noPos, `cannot find import "C"`) error(token.NoPos, `cannot find import "C"`)
} }
// In ast2, strip the import "C" line. // In ast2, strip the import "C" line.
@ -209,7 +210,7 @@ func (f *File) walk(x interface{}, context string, visit func(*File, interface{}
// everything else just recurs // everything else just recurs
default: default:
error(noPos, "unexpected type %T in walk", x, visit) error(token.NoPos, "unexpected type %T in walk", x, visit)
panic("unexpected type") panic("unexpected type")
case nil: case nil:

6
src/cmd/cgo/gcc.go
View file

@ -146,7 +146,7 @@ func (p *Package) guessKinds(f *File) []*Name {
if _, err := strconv.Atoi(n.Define); err == nil { if _, err := strconv.Atoi(n.Define); err == nil {
ok = true ok = true
} else if n.Define[0] == '"' || n.Define[0] == '\'' { } else if n.Define[0] == '"' || n.Define[0] == '\'' {
_, err := parser.ParseExpr("", n.Define) _, err := parser.ParseExpr(fset, "", n.Define)
if err == nil { if err == nil {
ok = true ok = true
} }
@ -229,7 +229,7 @@ func (p *Package) guessKinds(f *File) []*Name {
case strings.Contains(line, ": statement with no effect"): case strings.Contains(line, ": statement with no effect"):
what = "not-type" // const or func or var what = "not-type" // const or func or var
case strings.Contains(line, "undeclared"): case strings.Contains(line, "undeclared"):
error(noPos, "%s", strings.TrimSpace(line[colon+1:])) error(token.NoPos, "%s", strings.TrimSpace(line[colon+1:]))
case strings.Contains(line, "is not an integer constant"): case strings.Contains(line, "is not an integer constant"):
isConst[i] = false isConst[i] = false
continue continue
@ -257,7 +257,7 @@ func (p *Package) guessKinds(f *File) []*Name {
if n.Kind != "" { if n.Kind != "" {
continue continue
} }
error(noPos, "could not determine kind of name for C.%s", n.Go) error(token.NoPos, "could not determine kind of name for C.%s", n.Go)
} }
if nerrors > 0 { if nerrors > 0 {
fatal("unresolved names") fatal("unresolved names")

10
src/cmd/cgo/main.go
View file

@ -51,7 +51,7 @@ type Ref struct {
Context string // "type", "expr", "call", or "call2" Context string // "type", "expr", "call", or "call2"
} }
func (r *Ref) Pos() token.Position { func (r *Ref) Pos() token.Pos {
return (*r.Expr).Pos() return (*r.Expr).Pos()
} }
@ -103,6 +103,8 @@ var ptrSizeMap = map[string]int64{
"arm": 4, "arm": 4,
} }
var fset = token.NewFileSet()
func main() { func main() {
flag.Usage = usage flag.Usage = usage
flag.Parse() flag.Parse()
@ -180,7 +182,7 @@ func (p *Package) Record(f *File) {
if p.PackageName == "" { if p.PackageName == "" {
p.PackageName = f.Package p.PackageName = f.Package
} else if p.PackageName != f.Package { } else if p.PackageName != f.Package {
error(noPos, "inconsistent package names: %s, %s", p.PackageName, f.Package) error(token.NoPos, "inconsistent package names: %s, %s", p.PackageName, f.Package)
} }
if p.Typedef == nil { if p.Typedef == nil {
@ -190,7 +192,7 @@ func (p *Package) Record(f *File) {
if p.Typedef[k] == nil { if p.Typedef[k] == nil {
p.Typedef[k] = v p.Typedef[k] = v
} else if !reflect.DeepEqual(p.Typedef[k], v) { } else if !reflect.DeepEqual(p.Typedef[k], v) {
error(noPos, "inconsistent definitions for C type %s", k) error(token.NoPos, "inconsistent definitions for C type %s", k)
} }
} }
} }
@ -202,7 +204,7 @@ func (p *Package) Record(f *File) {
if p.Name[k] == nil { if p.Name[k] == nil {
p.Name[k] = v p.Name[k] = v
} else if !reflect.DeepEqual(p.Name[k], v) { } else if !reflect.DeepEqual(p.Name[k], v) {
error(noPos, "inconsistent definitions for C.%s", k) error(token.NoPos, "inconsistent definitions for C.%s", k)
} }
} }
} }

14
src/cmd/cgo/out.go
View file

@ -41,7 +41,7 @@ func (p *Package) writeDefs() {
for name, def := range p.Typedef { for name, def := range p.Typedef {
fmt.Fprintf(fgo2, "type %s ", name) fmt.Fprintf(fgo2, "type %s ", name)
printer.Fprint(fgo2, def) printer.Fprint(fgo2, fset, def)
fmt.Fprintf(fgo2, "\n") fmt.Fprintf(fgo2, "\n")
} }
fmt.Fprintf(fgo2, "type _Ctype_void [0]byte\n") fmt.Fprintf(fgo2, "type _Ctype_void [0]byte\n")
@ -54,7 +54,7 @@ func (p *Package) writeDefs() {
} }
fmt.Fprintf(fc, "#pragma dynimport ·%s %s \"%s%s.so\"\n", n.Mangle, n.C, soprefix, sopath) fmt.Fprintf(fc, "#pragma dynimport ·%s %s \"%s%s.so\"\n", n.Mangle, n.C, soprefix, sopath)
fmt.Fprintf(fgo2, "var %s ", n.Mangle) fmt.Fprintf(fgo2, "var %s ", n.Mangle)
printer.Fprint(fgo2, &ast.StarExpr{X: n.Type.Go}) printer.Fprint(fgo2, fset, &ast.StarExpr{X: n.Type.Go})
fmt.Fprintf(fgo2, "\n") fmt.Fprintf(fgo2, "\n")
} }
fmt.Fprintf(fc, "\n") fmt.Fprintf(fc, "\n")
@ -155,7 +155,7 @@ func (p *Package) writeDefsFunc(fc, fgo2 *os.File, n *Name, soprefix, sopath str
Name: ast.NewIdent(n.Mangle), Name: ast.NewIdent(n.Mangle),
Type: gtype, Type: gtype,
} }
printer.Fprint(fgo2, d) printer.Fprint(fgo2, fset, d)
fmt.Fprintf(fgo2, "\n") fmt.Fprintf(fgo2, "\n")
if name == "CString" || name == "GoString" || name == "GoStringN" { if name == "CString" || name == "GoString" || name == "GoStringN" {
@ -215,7 +215,7 @@ func (p *Package) writeOutput(f *File, srcfile string) {
// Write Go output: Go input with rewrites of C.xxx to _C_xxx. // Write Go output: Go input with rewrites of C.xxx to _C_xxx.
fmt.Fprintf(fgo1, "// Created by cgo - DO NOT EDIT\n") fmt.Fprintf(fgo1, "// Created by cgo - DO NOT EDIT\n")
fmt.Fprintf(fgo1, "//line %s:1\n", srcfile) fmt.Fprintf(fgo1, "//line %s:1\n", srcfile)
printer.Fprint(fgo1, f.AST) printer.Fprint(fgo1, fset, f.AST)
// While we process the vars and funcs, also write 6c and gcc output. // While we process the vars and funcs, also write 6c and gcc output.
// Gcc output starts with the preamble. // Gcc output starts with the preamble.
@ -423,11 +423,11 @@ func (p *Package) writeExports(fgo2, fc *os.File) {
// a Go wrapper function. // a Go wrapper function.
if fn.Recv != nil { if fn.Recv != nil {
fmt.Fprintf(fgo2, "func %s(recv ", goname) fmt.Fprintf(fgo2, "func %s(recv ", goname)
printer.Fprint(fgo2, fn.Recv.List[0].Type) printer.Fprint(fgo2, fset, fn.Recv.List[0].Type)
forFieldList(fntype.Params, forFieldList(fntype.Params,
func(i int, atype ast.Expr) { func(i int, atype ast.Expr) {
fmt.Fprintf(fgo2, ", p%d ", i) fmt.Fprintf(fgo2, ", p%d ", i)
printer.Fprint(fgo2, atype) printer.Fprint(fgo2, fset, atype)
}) })
fmt.Fprintf(fgo2, ")") fmt.Fprintf(fgo2, ")")
if gccResult != "void" { if gccResult != "void" {
@ -437,7 +437,7 @@ func (p *Package) writeExports(fgo2, fc *os.File) {
if i > 0 { if i > 0 {
fmt.Fprint(fgo2, ", ") fmt.Fprint(fgo2, ", ")
} }
printer.Fprint(fgo2, atype) printer.Fprint(fgo2, fset, atype)
}) })
fmt.Fprint(fgo2, ")") fmt.Fprint(fgo2, ")")
} }

5
src/cmd/cgo/util.go
View file

@ -72,12 +72,11 @@ func fatal(msg string, args ...interface{}) {
} }
var nerrors int var nerrors int
var noPos token.Position
func error(pos token.Position, msg string, args ...interface{}) { func error(pos token.Pos, msg string, args ...interface{}) {
nerrors++ nerrors++
if pos.IsValid() { if pos.IsValid() {
fmt.Fprintf(os.Stderr, "%s: ", pos) fmt.Fprintf(os.Stderr, "%s: ", fset.Position(pos).String())
} }
fmt.Fprintf(os.Stderr, msg, args...) fmt.Fprintf(os.Stderr, msg, args...)
fmt.Fprintf(os.Stderr, "\n") fmt.Fprintf(os.Stderr, "\n")

6
src/cmd/ebnflint/ebnflint.go
View file

@ -10,12 +10,14 @@ import (
"flag" "flag"
"fmt" "fmt"
"go/scanner" "go/scanner"
"go/token"
"io/ioutil" "io/ioutil"
"os" "os"
"path" "path"
) )
var fset = token.NewFileSet()
var start = flag.String("start", "Start", "name of start production") var start = flag.String("start", "Start", "name of start production")
@ -92,12 +94,12 @@ func main() {
src = extractEBNF(src) src = extractEBNF(src)
} }
grammar, err := ebnf.Parse(filename, src) grammar, err := ebnf.Parse(fset, filename, src)
if err != nil { if err != nil {
scanner.PrintError(os.Stderr, err) scanner.PrintError(os.Stderr, err)
} }
if err = ebnf.Verify(grammar, *start); err != nil { if err = ebnf.Verify(fset, grammar, *start); err != nil {
scanner.PrintError(os.Stderr, err) scanner.PrintError(os.Stderr, err)
} }
} }

11
src/cmd/godoc/dirtrees.go
View file

@ -10,6 +10,7 @@ import (
"bytes" "bytes"
"go/doc" "go/doc"
"go/parser" "go/parser"
"go/token"
"io/ioutil" "io/ioutil"
"os" "os"
pathutil "path" pathutil "path"
@ -87,7 +88,7 @@ type treeBuilder struct {
} }
func (b *treeBuilder) newDirTree(path, name string, depth int) *Directory { func (b *treeBuilder) newDirTree(fset *token.FileSet, path, name string, depth int) *Directory {
if b.pathFilter != nil && !b.pathFilter(path) { if b.pathFilter != nil && !b.pathFilter(path) {
return nil return nil
} }
@ -115,7 +116,7 @@ func (b *treeBuilder) newDirTree(path, name string, depth int) *Directory {
// though the directory doesn't contain any real package files - was bug) // though the directory doesn't contain any real package files - was bug)
if synopses[0] == "" { if synopses[0] == "" {
// no "optimal" package synopsis yet; continue to collect synopses // no "optimal" package synopsis yet; continue to collect synopses
file, err := parser.ParseFile(pathutil.Join(path, d.Name), nil, file, err := parser.ParseFile(fset, pathutil.Join(path, d.Name), nil,
parser.ParseComments|parser.PackageClauseOnly) parser.ParseComments|parser.PackageClauseOnly)
if err == nil { if err == nil {
hasPkgFiles = true hasPkgFiles = true
@ -148,7 +149,7 @@ func (b *treeBuilder) newDirTree(path, name string, depth int) *Directory {
i := 0 i := 0
for _, d := range list { for _, d := range list {
if isPkgDir(d) { if isPkgDir(d) {
dd := b.newDirTree(pathutil.Join(path, d.Name), d.Name, depth+1) dd := b.newDirTree(fset, pathutil.Join(path, d.Name), d.Name, depth+1)
if dd != nil { if dd != nil {
dirs[i] = dd dirs[i] = dd
i++ i++
@ -195,7 +196,9 @@ func newDirectory(root string, pathFilter func(string) bool, maxDepth int) *Dire
maxDepth = 1e6 // "infinity" maxDepth = 1e6 // "infinity"
} }
b := treeBuilder{pathFilter, maxDepth} b := treeBuilder{pathFilter, maxDepth}
return b.newDirTree(root, d.Name, 0) // the file set provided is only for local parsing, no position
// information escapes and thus we don't need to save the set
return b.newDirTree(token.NewFileSet(), root, d.Name, 0)
} }

89
src/cmd/godoc/godoc.go
View file

@ -21,6 +21,7 @@ import (
pathutil "path" pathutil "path"
"regexp" "regexp"
"runtime" "runtime"
"sort"
"strings" "strings"
"template" "template"
"time" "time"
@ -104,27 +105,6 @@ func isParentOf(p, q string) bool {
} }
// binarySearch returns an index i such that (a[i] <= s < a[i+1]) || (s is not in a).
// The slice a must not be empty and sorted in increasing order.
// (See "A Method of Programming", E.W. Dijkstra).
//
func binarySearch(a []string, s string) int {
i, j := 0, len(a)
// i < j for non-empty a
for i+1 < j {
// 0 <= i < j <= len(a) && (a[i] <= s < a[j] || (s is not in a))
h := i + (j-i)/2 // i < h < j
if a[h] <= s {
i = h
} else { // s < a[h]
j = h
}
}
// i+1 == j for non-empty a
return i
}
func setPathFilter(list []string) { func setPathFilter(list []string) {
if len(list) == 0 { if len(list) == 0 {
pathFilter.set(nil) pathFilter.set(nil)
@ -134,14 +114,10 @@ func setPathFilter(list []string) {
// len(list) > 0 // len(list) > 0
pathFilter.set(func(path string) bool { pathFilter.set(func(path string) bool {
// list is sorted in increasing order and for each path all its children are removed // list is sorted in increasing order and for each path all its children are removed
i := binarySearch(list, path) i := sort.Search(len(list), func(i int) bool { return list[i] > path })
// At this point we have (list[i] <= path < list[i+1]) || (path is not in list), // Now we have list[i-1] <= path < list[i].
// thus path must be either longer (a child) of list[i], or shorter (a parent) // Path may be a child of list[i-1] or a parent of list[i].
// of list[i+1] - assuming an "infinitely extended" list. However, binarySearch return i > 0 && isParentOf(list[i-1], path) || i < len(list) && isParentOf(path, list[i])
// will return a 0 if path < list[0], so we must be careful in that case.
return i == 0 && isParentOf(path, list[0]) ||
isParentOf(list[i], path) ||
i+1 < len(list) && isParentOf(path, list[i+1])
}) })
} }
@ -362,7 +338,7 @@ func (s *Styler) identId(name *ast.Ident) int {
// writeObjInfo writes the popup info corresponding to obj to w. // writeObjInfo writes the popup info corresponding to obj to w.
// The text is HTML-escaped and does not contain single quotes. // The text is HTML-escaped and does not contain single quotes.
func writeObjInfo(w io.Writer, obj *ast.Object) { func writeObjInfo(w io.Writer, fset *token.FileSet, obj *ast.Object) {
// for now, show object kind and name; eventually // for now, show object kind and name; eventually
// do something more interesting (show declaration, // do something more interesting (show declaration,
// for instance) // for instance)
@ -373,7 +349,7 @@ func writeObjInfo(w io.Writer, obj *ast.Object) {
// show type if we know it // show type if we know it
if obj.Type != nil && obj.Type.Expr != nil { if obj.Type != nil && obj.Type.Expr != nil {
fmt.Fprint(w, " ") fmt.Fprint(w, " ")
writeNode(&aposescaper{w}, obj.Type.Expr, true, &defaultStyler) writeNode(&aposescaper{w}, fset, obj.Type.Expr, true, &defaultStyler)
} }
} }
@ -382,7 +358,7 @@ func writeObjInfo(w io.Writer, obj *ast.Object) {
// information: The i'th array entry is a single-quoted string with // information: The i'th array entry is a single-quoted string with
// the popup information for an identifier x with s.identId(x) == i, // the popup information for an identifier x with s.identId(x) == i,
// for 0 <= i < s.idcount. // for 0 <= i < s.idcount.
func (s *Styler) idList() []byte { func (s *Styler) idList(fset *token.FileSet) []byte {
var buf bytes.Buffer var buf bytes.Buffer
fmt.Fprintln(&buf, "[") fmt.Fprintln(&buf, "[")
@ -400,7 +376,7 @@ func (s *Styler) idList() []byte {
fmt.Fprintf(&buf, "/* %4d */ ", id) fmt.Fprintf(&buf, "/* %4d */ ", id)
} }
fmt.Fprint(&buf, "'") fmt.Fprint(&buf, "'")
writeObjInfo(&buf, obj) writeObjInfo(&buf, fset, obj)
fmt.Fprint(&buf, "',\n") fmt.Fprint(&buf, "',\n")
} }
} }
@ -539,7 +515,7 @@ func (p *tconv) Write(data []byte) (n int, err os.Error) {
// Templates // Templates
// Write an AST-node to w; optionally html-escaped. // Write an AST-node to w; optionally html-escaped.
func writeNode(w io.Writer, node interface{}, html bool, styler printer.Styler) { func writeNode(w io.Writer, fset *token.FileSet, node interface{}, html bool, styler printer.Styler) {
mode := printer.TabIndent | printer.UseSpaces mode := printer.TabIndent | printer.UseSpaces
if html { if html {
mode |= printer.GenHTML mode |= printer.GenHTML
@ -548,7 +524,7 @@ func writeNode(w io.Writer, node interface{}, html bool, styler printer.Styler)
// to ensure a good outcome in most browsers (there may still // to ensure a good outcome in most browsers (there may still
// be tabs in comments and strings, but converting those into // be tabs in comments and strings, but converting those into
// the right number of spaces is much harder) // the right number of spaces is much harder)
(&printer.Config{mode, *tabwidth, styler}).Fprint(&tconv{output: w}, node) (&printer.Config{mode, *tabwidth, styler}).Fprint(&tconv{output: w}, fset, node)
} }
@ -563,14 +539,14 @@ func writeText(w io.Writer, text []byte, html bool) {
// Write anything to w; optionally html-escaped. // Write anything to w; optionally html-escaped.
func writeAny(w io.Writer, html bool, x interface{}) { func writeAny(w io.Writer, fset *token.FileSet, html bool, x interface{}) {
switch v := x.(type) { switch v := x.(type) {
case []byte: case []byte:
writeText(w, v, html) writeText(w, v, html)
case string: case string:
writeText(w, []byte(v), html) writeText(w, []byte(v), html)
case ast.Decl, ast.Expr, ast.Stmt, *ast.File: case ast.Decl, ast.Expr, ast.Stmt, *ast.File:
writeNode(w, x, html, &defaultStyler) writeNode(w, fset, x, html, &defaultStyler)
default: default:
if html { if html {
var buf bytes.Buffer var buf bytes.Buffer
@ -583,16 +559,26 @@ func writeAny(w io.Writer, html bool, x interface{}) {
} }
func fileset(x []interface{}) *token.FileSet {
if len(x) > 1 {
if fset, ok := x[1].(*token.FileSet); ok {
return fset
}
}
return nil
}
// Template formatter for "html" format. // Template formatter for "html" format.
func htmlFmt(w io.Writer, format string, x ...interface{}) { func htmlFmt(w io.Writer, format string, x ...interface{}) {
writeAny(w, true, x[0]) writeAny(w, fileset(x), true, x[0])
} }
// Template formatter for "html-esc" format. // Template formatter for "html-esc" format.
func htmlEscFmt(w io.Writer, format string, x ...interface{}) { func htmlEscFmt(w io.Writer, format string, x ...interface{}) {
var buf bytes.Buffer var buf bytes.Buffer
writeAny(&buf, false, x[0]) writeAny(&buf, fileset(x), false, x[0])
template.HTMLEscape(w, buf.Bytes()) template.HTMLEscape(w, buf.Bytes())
} }
@ -600,7 +586,7 @@ func htmlEscFmt(w io.Writer, format string, x ...interface{}) {
// Template formatter for "html-comment" format. // Template formatter for "html-comment" format.
func htmlCommentFmt(w io.Writer, format string, x ...interface{}) { func htmlCommentFmt(w io.Writer, format string, x ...interface{}) {
var buf bytes.Buffer var buf bytes.Buffer
writeAny(&buf, false, x[0]) writeAny(&buf, fileset(x), false, x[0])
// TODO(gri) Provide list of words (e.g. function parameters) // TODO(gri) Provide list of words (e.g. function parameters)
// to be emphasized by ToHTML. // to be emphasized by ToHTML.
doc.ToHTML(w, buf.Bytes(), nil) // does html-escaping doc.ToHTML(w, buf.Bytes(), nil) // does html-escaping
@ -609,7 +595,7 @@ func htmlCommentFmt(w io.Writer, format string, x ...interface{}) {
// Template formatter for "" (default) format. // Template formatter for "" (default) format.
func textFmt(w io.Writer, format string, x ...interface{}) { func textFmt(w io.Writer, format string, x ...interface{}) {
writeAny(w, false, x[0]) writeAny(w, fileset(x), false, x[0])
} }
@ -620,7 +606,7 @@ func urlFmt(w io.Writer, format string, x ...interface{}) {
// determine path and position info, if any // determine path and position info, if any
type positioner interface { type positioner interface {
Pos() token.Position Pos() token.Pos
} }
switch t := x[0].(type) { switch t := x[0].(type) {
case string: case string:
@ -628,9 +614,15 @@ func urlFmt(w io.Writer, format string, x ...interface{}) {
case positioner: case positioner:
pos := t.Pos() pos := t.Pos()
if pos.IsValid() { if pos.IsValid() {
pos := fileset(x).Position(pos)
path = pos.Filename path = pos.Filename
line = pos.Line line = pos.Line
} }
default:
// we should never reach here, but be resilient
// and assume the position is invalid (empty path,
// and line 0)
log.Printf("INTERNAL ERROR: urlFmt(%s) without a string or positioner", format)
} }
// map path // map path
@ -902,7 +894,8 @@ func applyTemplate(t *template.Template, name string, data interface{}) []byte {
func serveGoSource(w http.ResponseWriter, r *http.Request, abspath, relpath string) { func serveGoSource(w http.ResponseWriter, r *http.Request, abspath, relpath string) {
file, err := parser.ParseFile(abspath, nil, parser.ParseComments) fset := token.NewFileSet()
file, err := parser.ParseFile(fset, abspath, nil, parser.ParseComments)
if err != nil { if err != nil {
log.Printf("parser.ParseFile: %s", err) log.Printf("parser.ParseFile: %s", err)
serveError(w, r, relpath, err) serveError(w, r, relpath, err)
@ -915,13 +908,13 @@ func serveGoSource(w http.ResponseWriter, r *http.Request, abspath, relpath stri
var buf bytes.Buffer var buf bytes.Buffer
styler := newStyler(r.FormValue("h")) styler := newStyler(r.FormValue("h"))
writeNode(&buf, file, true, styler) writeNode(&buf, fset, file, true, styler)
type SourceInfo struct { type SourceInfo struct {
IdList []byte IdList []byte
Source []byte Source []byte
} }
info := &SourceInfo{styler.idList(), buf.Bytes()} info := &SourceInfo{styler.idList(fset), buf.Bytes()}
contents := applyTemplate(sourceHTML, "sourceHTML", info) contents := applyTemplate(sourceHTML, "sourceHTML", info)
servePage(w, "Source file "+relpath, "", "", contents) servePage(w, "Source file "+relpath, "", "", contents)
@ -1102,6 +1095,7 @@ const (
type PageInfo struct { type PageInfo struct {
Dirname string // directory containing the package Dirname string // directory containing the package
PList []string // list of package names found PList []string // list of package names found
FSet *token.FileSet // corresponding file set
PAst *ast.File // nil if no single AST with package exports PAst *ast.File // nil if no single AST with package exports
PDoc *doc.PackageDoc // nil if no single package documentation PDoc *doc.PackageDoc // nil if no single package documentation
Dirs *DirList // nil if no directory information Dirs *DirList // nil if no directory information
@ -1135,7 +1129,8 @@ func (h *httpHandler) getPageInfo(abspath, relpath, pkgname string, mode PageInf
} }
// get package ASTs // get package ASTs
pkgs, err := parser.ParseDir(abspath, filter, parser.ParseComments) fset := token.NewFileSet()
pkgs, err := parser.ParseDir(fset, abspath, filter, parser.ParseComments)
if err != nil && pkgs == nil { if err != nil && pkgs == nil {
// only report directory read errors, ignore parse errors // only report directory read errors, ignore parse errors
// (may be able to extract partial package information) // (may be able to extract partial package information)
@ -1252,7 +1247,7 @@ func (h *httpHandler) getPageInfo(abspath, relpath, pkgname string, mode PageInf
timestamp = time.Seconds() timestamp = time.Seconds()
} }
return PageInfo{abspath, plist, past, pdoc, dir.listing(true), timestamp, h.isPkg, nil} return PageInfo{abspath, plist, fset, past, pdoc, dir.listing(true), timestamp, h.isPkg, nil}
} }

19
src/cmd/godoc/index.go
View file

@ -429,6 +429,7 @@ type IndexResult struct {
// interface for walking file trees, and the ast.Visitor interface for // interface for walking file trees, and the ast.Visitor interface for
// walking Go ASTs. // walking Go ASTs.
type Indexer struct { type Indexer struct {
fset *token.FileSet // file set for all indexed files
words map[string]*IndexResult // RunLists of Spots words map[string]*IndexResult // RunLists of Spots
snippets vector.Vector // vector of *Snippets, indexed by snippet indices snippets vector.Vector // vector of *Snippets, indexed by snippet indices
file *File // current file file *File // current file
@ -461,11 +462,11 @@ func (x *Indexer) visitIdent(kind SpotKind, id *ast.Ident) {
if kind == Use || x.decl == nil { if kind == Use || x.decl == nil {
// not a declaration or no snippet required // not a declaration or no snippet required
info := makeSpotInfo(kind, id.Pos().Line, false) info := makeSpotInfo(kind, x.fset.Position(id.Pos()).Line, false)
lists.Others.Push(Spot{x.file, info}) lists.Others.Push(Spot{x.file, info})
} else { } else {
// a declaration with snippet // a declaration with snippet
index := x.addSnippet(NewSnippet(x.decl, id)) index := x.addSnippet(NewSnippet(x.fset, x.decl, id))
info := makeSpotInfo(kind, index, true) info := makeSpotInfo(kind, index, true)
lists.Decls.Push(Spot{x.file, info}) lists.Decls.Push(Spot{x.file, info})
} }
@ -579,8 +580,8 @@ func (x *Indexer) Visit(node interface{}) ast.Visitor {
} }
func pkgName(filename string) string { func pkgName(fset *token.FileSet, filename string) string {
file, err := parser.ParseFile(filename, nil, parser.PackageClauseOnly) file, err := parser.ParseFile(fset, filename, nil, parser.PackageClauseOnly)
if err != nil || file == nil { if err != nil || file == nil {
return "" return ""
} }
@ -598,11 +599,11 @@ func (x *Indexer) visitFile(dirname string, f *os.FileInfo) {
return return
} }
if excludeMainPackages && pkgName(path) == "main" { if excludeMainPackages && pkgName(x.fset, path) == "main" {
return return
} }
file, err := parser.ParseFile(path, nil, parser.ParseComments) file, err := parser.ParseFile(x.fset, path, nil, parser.ParseComments)
if err != nil { if err != nil {
return // ignore files with (parse) errors return // ignore files with (parse) errors
} }
@ -641,6 +642,7 @@ func NewIndex(dirnames <-chan string) *Index {
var x Indexer var x Indexer
// initialize Indexer // initialize Indexer
x.fset = token.NewFileSet()
x.words = make(map[string]*IndexResult) x.words = make(map[string]*IndexResult)
// index all files in the directories given by dirnames // index all files in the directories given by dirnames
@ -656,6 +658,9 @@ func NewIndex(dirnames <-chan string) *Index {
} }
} }
// the file set is not needed after indexing - help GC and clear it
x.fset = nil
// for each word, reduce the RunLists into a LookupResult; // for each word, reduce the RunLists into a LookupResult;
// also collect the word with its canonical spelling in a // also collect the word with its canonical spelling in a
// word list for later computation of alternative spellings // word list for later computation of alternative spellings
@ -714,7 +719,7 @@ func (x *Index) LookupWord(w string) (match *LookupResult, alt *AltWords) {
func isIdentifier(s string) bool { func isIdentifier(s string) bool {
var S scanner.Scanner var S scanner.Scanner
S.Init("", []byte(s), nil, 0) S.Init(token.NewFileSet(), "", []byte(s), nil, 0)
if _, tok, _ := S.Scan(); tok == token.IDENT { if _, tok, _ := S.Scan(); tok == token.IDENT {
_, tok, _ := S.Scan() _, tok, _ := S.Scan()
return tok == token.EOF return tok == token.EOF

2
src/cmd/godoc/main.go
View file

@ -367,7 +367,7 @@ func main() {
if i > 0 { if i > 0 {
fmt.Println() fmt.Println()
} }
writeAny(os.Stdout, *html, d) writeAny(os.Stdout, info.FSet, *html, d)
fmt.Println() fmt.Println()
} }
return return

23
src/cmd/godoc/snippet.go
View file

@ -12,6 +12,7 @@ package main
import ( import (
"bytes" "bytes"
"go/ast" "go/ast"
"go/token"
"go/printer" "go/printer"
"fmt" "fmt"
) )
@ -43,10 +44,10 @@ func (s *snippetStyler) Ident(id *ast.Ident) (text []byte, tag printer.HTMLTag)
} }
func newSnippet(decl ast.Decl, id *ast.Ident) *Snippet { func newSnippet(fset *token.FileSet, decl ast.Decl, id *ast.Ident) *Snippet {
var buf bytes.Buffer var buf bytes.Buffer
writeNode(&buf, decl, true, &snippetStyler{highlight: id}) writeNode(&buf, fset, decl, true, &snippetStyler{highlight: id})
return &Snippet{id.Pos().Line, buf.String()} return &Snippet{fset.Position(id.Pos()).Line, buf.String()}
} }
@ -73,7 +74,7 @@ func findSpec(list []ast.Spec, id *ast.Ident) ast.Spec {
} }
func genSnippet(d *ast.GenDecl, id *ast.Ident) *Snippet { func genSnippet(fset *token.FileSet, d *ast.GenDecl, id *ast.Ident) *Snippet {
s := findSpec(d.Specs, id) s := findSpec(d.Specs, id)
if s == nil { if s == nil {
return nil // declaration doesn't contain id - exit gracefully return nil // declaration doesn't contain id - exit gracefully
@ -82,11 +83,11 @@ func genSnippet(d *ast.GenDecl, id *ast.Ident) *Snippet {
// only use the spec containing the id for the snippet // only use the spec containing the id for the snippet
dd := &ast.GenDecl{d.Doc, d.Pos(), d.Tok, d.Lparen, []ast.Spec{s}, d.Rparen} dd := &ast.GenDecl{d.Doc, d.Pos(), d.Tok, d.Lparen, []ast.Spec{s}, d.Rparen}
return newSnippet(dd, id) return newSnippet(fset, dd, id)
} }
func funcSnippet(d *ast.FuncDecl, id *ast.Ident) *Snippet { func funcSnippet(fset *token.FileSet, d *ast.FuncDecl, id *ast.Ident) *Snippet {
if d.Name != id { if d.Name != id {
return nil // declaration doesn't contain id - exit gracefully return nil // declaration doesn't contain id - exit gracefully
} }
@ -94,7 +95,7 @@ func funcSnippet(d *ast.FuncDecl, id *ast.Ident) *Snippet {
// only use the function signature for the snippet // only use the function signature for the snippet
dd := &ast.FuncDecl{d.Doc, d.Recv, d.Name, d.Type, nil} dd := &ast.FuncDecl{d.Doc, d.Recv, d.Name, d.Type, nil}
return newSnippet(dd, id) return newSnippet(fset, dd, id)
} }
@ -102,18 +103,18 @@ func funcSnippet(d *ast.FuncDecl, id *ast.Ident) *Snippet {
// identifier id. Parts of the declaration not containing the identifier // identifier id. Parts of the declaration not containing the identifier
// may be removed for a more compact snippet. // may be removed for a more compact snippet.
// //
func NewSnippet(decl ast.Decl, id *ast.Ident) (s *Snippet) { func NewSnippet(fset *token.FileSet, decl ast.Decl, id *ast.Ident) (s *Snippet) {
switch d := decl.(type) { switch d := decl.(type) {
case *ast.GenDecl: case *ast.GenDecl:
s = genSnippet(d, id) s = genSnippet(fset, d, id)
case *ast.FuncDecl: case *ast.FuncDecl:
s = funcSnippet(d, id) s = funcSnippet(fset, d, id)
} }
// handle failure gracefully // handle failure gracefully
if s == nil { if s == nil {
s = &Snippet{ s = &Snippet{
id.Pos().Line, fset.Position(id.Pos()).Line,
fmt.Sprintf(`could not generate a snippet for <span class="highlight">%s</span>`, id.Name), fmt.Sprintf(`could not generate a snippet for <span class="highlight">%s</span>`, id.Name),
} }
} }

33
src/cmd/godoc/spec.go
View file

@ -20,19 +20,21 @@ import (
type ebnfParser struct { type ebnfParser struct {
out io.Writer // parser output out io.Writer // parser output
src []byte // parser source src []byte // parser source
file *token.File // for position information
scanner scanner.Scanner scanner scanner.Scanner
prev int // offset of previous token prev int // offset of previous token
pos token.Position // token position pos token.Pos // token position
tok token.Token // one token look-ahead tok token.Token // one token look-ahead
lit []byte // token literal lit []byte // token literal
} }
func (p *ebnfParser) flush() { func (p *ebnfParser) flush() {
p.out.Write(p.src[p.prev:p.pos.Offset]) offs := p.file.Offset(p.pos)
p.prev = p.pos.Offset p.out.Write(p.src[p.prev:offs])
p.prev = offs
} }
@ -52,9 +54,9 @@ func (p *ebnfParser) Error(pos token.Position, msg string) {
} }
func (p *ebnfParser) errorExpected(pos token.Position, msg string) { func (p *ebnfParser) errorExpected(pos token.Pos, msg string) {
msg = "expected " + msg msg = "expected " + msg
if pos.Offset == p.pos.Offset { if pos == p.pos {
// the error happened at the current position; // the error happened at the current position;
// make the error message more specific // make the error message more specific
msg += ", found '" + p.tok.String() + "'" msg += ", found '" + p.tok.String() + "'"
@ -62,11 +64,11 @@ func (p *ebnfParser) errorExpected(pos token.Position, msg string) {
msg += " " + string(p.lit) msg += " " + string(p.lit)
} }
} }
p.Error(pos, msg) p.Error(p.file.Position(pos), msg)
} }
func (p *ebnfParser) expect(tok token.Token) token.Position { func (p *ebnfParser) expect(tok token.Token) token.Pos {
pos := p.pos pos := p.pos
if p.tok != tok { if p.tok != tok {
p.errorExpected(pos, "'"+tok.String()+"'") p.errorExpected(pos, "'"+tok.String()+"'")
@ -148,11 +150,11 @@ func (p *ebnfParser) parseProduction() {
} }
func (p *ebnfParser) parse(out io.Writer, src []byte) { func (p *ebnfParser) parse(fset *token.FileSet, out io.Writer, src []byte) {
// initialize ebnfParser // initialize ebnfParser
p.out = out p.out = out
p.src = src p.src = src
p.scanner.Init("", src, p, 0) p.file = p.scanner.Init(fset, "", src, p, 0)
p.next() // initializes pos, tok, lit p.next() // initializes pos, tok, lit
// process source // process source
@ -171,6 +173,7 @@ var (
func linkify(out io.Writer, src []byte) { func linkify(out io.Writer, src []byte) {
fset := token.NewFileSet()
for len(src) > 0 { for len(src) > 0 {
n := len(src) n := len(src)
@ -192,7 +195,7 @@ func linkify(out io.Writer, src []byte) {
out.Write(src[0:i]) out.Write(src[0:i])
// parse and write EBNF // parse and write EBNF
var p ebnfParser var p ebnfParser
p.parse(out, src[i:j]) p.parse(fset, out, src[i:j])
// advance // advance
src = src[j:n] src = src[j:n]

6
src/cmd/gofmt/gofmt.go
View file

@ -12,6 +12,7 @@ import (
"go/parser" "go/parser"
"go/printer" "go/printer"
"go/scanner" "go/scanner"
"go/token"
"io/ioutil" "io/ioutil"
"os" "os"
pathutil "path" pathutil "path"
@ -39,6 +40,7 @@ var (
var ( var (
fset = token.NewFileSet()
exitCode = 0 exitCode = 0
rewrite func(*ast.File) *ast.File rewrite func(*ast.File) *ast.File
parserMode uint parserMode uint
@ -93,7 +95,7 @@ func processFile(f *os.File) os.Error {
return err return err
} }
file, err := parser.ParseFile(f.Name(), src, parserMode) file, err := parser.ParseFile(fset, f.Name(), src, parserMode)
if err != nil { if err != nil {
return err return err
@ -112,7 +114,7 @@ func processFile(f *os.File) os.Error {
} }
var res bytes.Buffer var res bytes.Buffer
_, err = (&printer.Config{printerMode, *tabWidth, nil}).Fprint(&res, file) _, err = (&printer.Config{printerMode, *tabWidth, nil}).Fprint(&res, fset, file)
if err != nil { if err != nil {
return err return err
} }

4
src/cmd/gofmt/rewrite.go
View file

@ -37,7 +37,7 @@ func initRewrite() {
// but there are problems with preserving formatting and also // but there are problems with preserving formatting and also
// with what a wildcard for a statement looks like. // with what a wildcard for a statement looks like.
func parseExpr(s string, what string) ast.Expr { func parseExpr(s string, what string) ast.Expr {
x, err := parser.ParseExpr("input", s) x, err := parser.ParseExpr(fset, "input", s)
if err != nil { if err != nil {
fmt.Fprintf(os.Stderr, "parsing %s %s: %s\n", what, s, err) fmt.Fprintf(os.Stderr, "parsing %s %s: %s\n", what, s, err)
os.Exit(2) os.Exit(2)
@ -66,7 +66,7 @@ func rewriteFile(pattern, replace ast.Expr, p *ast.File) *ast.File {
} }
var positionType = reflect.Typeof(token.Position{}) var positionType = reflect.Typeof(token.NoPos)
var identType = reflect.Typeof((*ast.Ident)(nil)) var identType = reflect.Typeof((*ast.Ident)(nil))

2
src/cmd/goinstall/main.go
View file

@ -11,6 +11,7 @@ import (
"exec" "exec"
"flag" "flag"
"fmt" "fmt"
"go/token"
"io" "io"
"io/ioutil" "io/ioutil"
"os" "os"
@ -27,6 +28,7 @@ func usage() {
} }
var ( var (
fset = token.NewFileSet()
argv0 = os.Args[0] argv0 = os.Args[0]
errors = false errors = false
parents = make(map[string]string) parents = make(map[string]string)

2
src/cmd/goinstall/parse.go
View file

@ -41,7 +41,7 @@ func goFiles(dir string, allowMain bool) (files []string, imports map[string]str
continue continue
} }
filename := path.Join(dir, d.Name) filename := path.Join(dir, d.Name)
pf, err := parser.ParseFile(filename, nil, parser.ImportsOnly) pf, err := parser.ParseFile(fset, filename, nil, parser.ImportsOnly)
if err != nil { if err != nil {
return nil, nil, "", err return nil, nil, "", err
} }

75
src/pkg/ebnf/ebnf.go
View file

@ -38,7 +38,7 @@ type (
// An Expression node represents a production expression. // An Expression node represents a production expression.
Expression interface { Expression interface {
// Pos is the position of the first character of the syntactic construct // Pos is the position of the first character of the syntactic construct
Pos() token.Position Pos() token.Pos
} }
// An Alternative node represents a non-empty list of alternative expressions. // An Alternative node represents a non-empty list of alternative expressions.
@ -49,14 +49,14 @@ type (
// A Name node represents a production name. // A Name node represents a production name.
Name struct { Name struct {
token.Position StringPos token.Pos
String string String string
} }
// A Token node represents a literal. // A Token node represents a literal.
Token struct { Token struct {
token.Position StringPos token.Pos
String string String string
} }
// A List node represents a range of characters. // A List node represents a range of characters.
@ -66,20 +66,20 @@ type (
// A Group node represents a grouped expression. // A Group node represents a grouped expression.
Group struct { Group struct {
token.Position Lparen token.Pos
Body Expression // (body) Body Expression // (body)
} }
// An Option node represents an optional expression. // An Option node represents an optional expression.
Option struct { Option struct {
token.Position Lbrack token.Pos
Body Expression // [body] Body Expression // [body]
} }
// A Repetition node represents a repeated expression. // A Repetition node represents a repeated expression.
Repetition struct { Repetition struct {
token.Position Lbrace token.Pos
Body Expression // {body} Body Expression // {body}
} }
// A Production node represents an EBNF production. // A Production node represents an EBNF production.
@ -95,20 +95,15 @@ type (
) )
func (x Alternative) Pos() token.Position { func (x Alternative) Pos() token.Pos { return x[0].Pos() } // the parser always generates non-empty Alternative
return x[0].Pos() // the parser always generates non-empty Alternative func (x Sequence) Pos() token.Pos { return x[0].Pos() } // the parser always generates non-empty Sequences
} func (x *Name) Pos() token.Pos { return x.StringPos }
func (x *Token) Pos() token.Pos { return x.StringPos }
func (x *Range) Pos() token.Pos { return x.Begin.Pos() }
func (x Sequence) Pos() token.Position { func (x *Group) Pos() token.Pos { return x.Lparen }
return x[0].Pos() // the parser always generates non-empty Sequences func (x *Option) Pos() token.Pos { return x.Lbrack }
} func (x *Repetition) Pos() token.Pos { return x.Lbrace }
func (x *Production) Pos() token.Pos { return x.Name.Pos() }
func (x Range) Pos() token.Position { return x.Begin.Pos() }
func (p *Production) Pos() token.Position { return p.Name.Pos() }
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
@ -121,6 +116,7 @@ func isLexical(name string) bool {
type verifier struct { type verifier struct {
fset *token.FileSet
scanner.ErrorVector scanner.ErrorVector
worklist []*Production worklist []*Production
reached Grammar // set of productions reached from (and including) the root production reached Grammar // set of productions reached from (and including) the root production
@ -128,6 +124,11 @@ type verifier struct {
} }
func (v *verifier) error(pos token.Pos, msg string) {
v.Error(v.fset.Position(pos), msg)
}
func (v *verifier) push(prod *Production) { func (v *verifier) push(prod *Production) {
name := prod.Name.String name := prod.Name.String
if _, found := v.reached[name]; !found { if _, found := v.reached[name]; !found {
@ -140,7 +141,7 @@ func (v *verifier) push(prod *Production) {
func (v *verifier) verifyChar(x *Token) int { func (v *verifier) verifyChar(x *Token) int {
s := x.String s := x.String
if utf8.RuneCountInString(s) != 1 { if utf8.RuneCountInString(s) != 1 {
v.Error(x.Pos(), "single char expected, found "+s) v.error(x.Pos(), "single char expected, found "+s)
return 0 return 0
} }
ch, _ := utf8.DecodeRuneInString(s) ch, _ := utf8.DecodeRuneInString(s)
@ -166,12 +167,12 @@ func (v *verifier) verifyExpr(expr Expression, lexical bool) {
if prod, found := v.grammar[x.String]; found { if prod, found := v.grammar[x.String]; found {
v.push(prod) v.push(prod)
} else { } else {
v.Error(x.Pos(), "missing production "+x.String) v.error(x.Pos(), "missing production "+x.String)
} }
// within a lexical production references // within a lexical production references
// to non-lexical productions are invalid // to non-lexical productions are invalid
if lexical && !isLexical(x.String) { if lexical && !isLexical(x.String) {
v.Error(x.Pos(), "reference to non-lexical production "+x.String) v.error(x.Pos(), "reference to non-lexical production "+x.String)
} }
case *Token: case *Token:
// nothing to do for now // nothing to do for now
@ -179,7 +180,7 @@ func (v *verifier) verifyExpr(expr Expression, lexical bool) {
i := v.verifyChar(x.Begin) i := v.verifyChar(x.Begin)
j := v.verifyChar(x.End) j := v.verifyChar(x.End)
if i >= j { if i >= j {
v.Error(x.Pos(), "decreasing character range") v.error(x.Pos(), "decreasing character range")
} }
case *Group: case *Group:
v.verifyExpr(x.Body, lexical) v.verifyExpr(x.Body, lexical)
@ -193,16 +194,18 @@ func (v *verifier) verifyExpr(expr Expression, lexical bool) {
} }
func (v *verifier) verify(grammar Grammar, start string) { func (v *verifier) verify(fset *token.FileSet, grammar Grammar, start string) {
// find root production // find root production
root, found := grammar[start] root, found := grammar[start]
if !found { if !found {
var noPos token.Position // token.NoPos doesn't require a file set;
v.Error(noPos, "no start production "+start) // ok to set v.fset only afterwards
v.error(token.NoPos, "no start production "+start)
return return
} }
// initialize verifier // initialize verifier
v.fset = fset
v.ErrorVector.Reset() v.ErrorVector.Reset()
v.worklist = v.worklist[0:0] v.worklist = v.worklist[0:0]
v.reached = make(Grammar) v.reached = make(Grammar)
@ -224,7 +227,7 @@ func (v *verifier) verify(grammar Grammar, start string) {
if len(v.reached) < len(v.grammar) { if len(v.reached) < len(v.grammar) {
for name, prod := range v.grammar { for name, prod := range v.grammar {
if _, found := v.reached[name]; !found { if _, found := v.reached[name]; !found {
v.Error(prod.Pos(), name+" is unreachable") v.error(prod.Pos(), name+" is unreachable")
} }
} }
} }
@ -236,8 +239,10 @@ func (v *verifier) verify(grammar Grammar, start string) {
// - all productions defined are used when beginning at start // - all productions defined are used when beginning at start
// - lexical productions refer only to other lexical productions // - lexical productions refer only to other lexical productions
// //
func Verify(grammar Grammar, start string) os.Error { // Position information is interpreted relative to the file set fset.
//
func Verify(fset *token.FileSet, grammar Grammar, start string) os.Error {
var v verifier var v verifier
v.verify(grammar, start) v.verify(fset, grammar, start)
return v.GetError(scanner.Sorted) return v.GetError(scanner.Sorted)
} }

8
src/pkg/ebnf/ebnf_test.go
View file

@ -5,11 +5,15 @@
package ebnf package ebnf
import ( import (
"go/token"
"io/ioutil" "io/ioutil"
"testing" "testing"
) )
var fset = token.NewFileSet()
var grammars = []string{ var grammars = []string{
`Program = . `Program = .
`, `,
@ -40,11 +44,11 @@ var grammars = []string{
func check(t *testing.T, filename string, src []byte) { func check(t *testing.T, filename string, src []byte) {
grammar, err := Parse(filename, src) grammar, err := Parse(fset, filename, src)
if err != nil { if err != nil {
t.Errorf("Parse(%s) failed: %v", src, err) t.Errorf("Parse(%s) failed: %v", src, err)
} }
if err = Verify(grammar, "Program"); err != nil { if err = Verify(fset, grammar, "Program"); err != nil {
t.Errorf("Verify(%s) failed: %v", src, err) t.Errorf("Verify(%s) failed: %v", src, err)
} }
} }

34
src/pkg/ebnf/parser.go
View file

@ -13,11 +13,12 @@ import (
type parser struct { type parser struct {
fset *token.FileSet
scanner.ErrorVector scanner.ErrorVector
scanner scanner.Scanner scanner scanner.Scanner
pos token.Position // token position pos token.Pos // token position
tok token.Token // one token look-ahead tok token.Token // one token look-ahead
lit []byte // token literal lit []byte // token literal
} }
@ -31,9 +32,14 @@ func (p *parser) next() {
} }
func (p *parser) errorExpected(pos token.Position, msg string) { func (p *parser) error(pos token.Pos, msg string) {
p.Error(p.fset.Position(pos), msg)
}
func (p *parser) errorExpected(pos token.Pos, msg string) {
msg = "expected " + msg msg = "expected " + msg
if pos.Offset == p.pos.Offset { if pos == p.pos {
// the error happened at the current position; // the error happened at the current position;
// make the error message more specific // make the error message more specific
msg += ", found '" + p.tok.String() + "'" msg += ", found '" + p.tok.String() + "'"
@ -41,11 +47,11 @@ func (p *parser) errorExpected(pos token.Position, msg string) {
msg += " " + string(p.lit) msg += " " + string(p.lit)
} }
} }
p.Error(pos, msg) p.error(pos, msg)
} }
func (p *parser) expect(tok token.Token) token.Position { func (p *parser) expect(tok token.Token) token.Pos {
pos := p.pos pos := p.pos
if p.tok != tok { if p.tok != tok {
p.errorExpected(pos, "'"+tok.String()+"'") p.errorExpected(pos, "'"+tok.String()+"'")
@ -167,10 +173,11 @@ func (p *parser) parseProduction() *Production {
} }
func (p *parser) parse(filename string, src []byte) Grammar { func (p *parser) parse(fset *token.FileSet, filename string, src []byte) Grammar {
// initialize parser // initialize parser
p.fset = fset
p.ErrorVector.Reset() p.ErrorVector.Reset()
p.scanner.Init(filename, src, p, 0) p.scanner.Init(fset, filename, src, p, 0)
p.next() // initializes pos, tok, lit p.next() // initializes pos, tok, lit
grammar := make(Grammar) grammar := make(Grammar)
@ -180,7 +187,7 @@ func (p *parser) parse(filename string, src []byte) Grammar {
if _, found := grammar[name]; !found { if _, found := grammar[name]; !found {
grammar[name] = prod grammar[name] = prod
} else { } else {
p.Error(prod.Pos(), name+" declared already") p.error(prod.Pos(), name+" declared already")
} }
} }
@ -191,10 +198,11 @@ func (p *parser) parse(filename string, src []byte) Grammar {
// Parse parses a set of EBNF productions from source src. // Parse parses a set of EBNF productions from source src.
// It returns a set of productions. Errors are reported // It returns a set of productions. Errors are reported
// for incorrect syntax and if a production is declared // for incorrect syntax and if a production is declared
// more than once. // more than once. Position information is recorded relative
// to the file set fset.
// //
func Parse(filename string, src []byte) (Grammar, os.Error) { func Parse(fset *token.FileSet, filename string, src []byte) (Grammar, os.Error) {
var p parser var p parser
grammar := p.parse(filename, src) grammar := p.parse(fset, filename, src)
return grammar, p.GetError(scanner.Sorted) return grammar, p.GetError(scanner.Sorted)
} }

6
src/pkg/exp/datafmt/datafmt_test.go
View file

@ -7,11 +7,15 @@ package datafmt
import ( import (
"fmt" "fmt"
"testing" "testing"
"go/token"
) )
var fset = token.NewFileSet()
func parse(t *testing.T, form string, fmap FormatterMap) Format { func parse(t *testing.T, form string, fmap FormatterMap) Format {
f, err := Parse("", []byte(form), fmap) f, err := Parse(fset, "", []byte(form), fmap)
if err != nil { if err != nil {
t.Errorf("Parse(%s): %v", form, err) t.Errorf("Parse(%s): %v", form, err)
return nil return nil

38
src/pkg/exp/datafmt/parser.go
View file

@ -19,9 +19,10 @@ import (
type parser struct { type parser struct {
scanner.ErrorVector scanner.ErrorVector
scanner scanner.Scanner scanner scanner.Scanner
pos token.Position // token position file *token.File
tok token.Token // one token look-ahead pos token.Pos // token position
lit []byte // token literal tok token.Token // one token look-ahead
lit []byte // token literal
packs map[string]string // PackageName -> ImportPath packs map[string]string // PackageName -> ImportPath
rules map[string]expr // RuleName -> Expression rules map[string]expr // RuleName -> Expression
@ -39,18 +40,23 @@ func (p *parser) next() {
} }
func (p *parser) init(filename string, src []byte) { func (p *parser) init(fset *token.FileSet, filename string, src []byte) {
p.ErrorVector.Reset() p.ErrorVector.Reset()
p.scanner.Init(filename, src, p, scanner.AllowIllegalChars) // return '@' as token.ILLEGAL w/o error message p.file = p.scanner.Init(fset, filename, src, p, scanner.AllowIllegalChars) // return '@' as token.ILLEGAL w/o error message
p.next() // initializes pos, tok, lit p.next() // initializes pos, tok, lit
p.packs = make(map[string]string) p.packs = make(map[string]string)
p.rules = make(map[string]expr) p.rules = make(map[string]expr)
} }
func (p *parser) errorExpected(pos token.Position, msg string) { func (p *parser) error(pos token.Pos, msg string) {
p.Error(p.file.Position(pos), msg)
}
func (p *parser) errorExpected(pos token.Pos, msg string) {
msg = "expected " + msg msg = "expected " + msg
if pos.Offset == p.pos.Offset { if pos == p.pos {
// the error happened at the current position; // the error happened at the current position;
// make the error message more specific // make the error message more specific
msg += ", found '" + p.tok.String() + "'" msg += ", found '" + p.tok.String() + "'"
@ -58,11 +64,11 @@ func (p *parser) errorExpected(pos token.Position, msg string) {
msg += " " + string(p.lit) msg += " " + string(p.lit)
} }
} }
p.Error(pos, msg) p.error(pos, msg)
} }
func (p *parser) expect(tok token.Token) token.Position { func (p *parser) expect(tok token.Token) token.Pos {
pos := p.pos pos := p.pos
if p.tok != tok { if p.tok != tok {
p.errorExpected(pos, "'"+tok.String()+"'") p.errorExpected(pos, "'"+tok.String()+"'")
@ -87,7 +93,7 @@ func (p *parser) parseTypeName() (string, bool) {
if importPath, found := p.packs[name]; found { if importPath, found := p.packs[name]; found {
name = importPath name = importPath
} else { } else {
p.Error(pos, "package not declared: "+name) p.error(pos, "package not declared: "+name)
} }
p.next() p.next()
name, isIdent = name+"."+p.parseIdentifier(), false name, isIdent = name+"."+p.parseIdentifier(), false
@ -303,11 +309,11 @@ func (p *parser) parseFormat() {
// add package declaration // add package declaration
if !isIdent { if !isIdent {
p.Error(pos, "illegal package name: "+name) p.error(pos, "illegal package name: "+name)
} else if _, found := p.packs[name]; !found { } else if _, found := p.packs[name]; !found {
p.packs[name] = importPath p.packs[name] = importPath
} else { } else {
p.Error(pos, "package already declared: "+name) p.error(pos, "package already declared: "+name)
} }
case token.ASSIGN: case token.ASSIGN:
@ -319,7 +325,7 @@ func (p *parser) parseFormat() {
if _, found := p.rules[name]; !found { if _, found := p.rules[name]; !found {
p.rules[name] = x p.rules[name] = x
} else { } else {
p.Error(pos, "format rule already declared: "+name) p.error(pos, "format rule already declared: "+name)
} }
default: default:
@ -358,10 +364,10 @@ func remap(p *parser, name string) string {
// there are no errors, the result is a Format and the error is nil. // there are no errors, the result is a Format and the error is nil.
// Otherwise the format is nil and a non-empty ErrorList is returned. // Otherwise the format is nil and a non-empty ErrorList is returned.
// //
func Parse(filename string, src []byte, fmap FormatterMap) (Format, os.Error) { func Parse(fset *token.FileSet, filename string, src []byte, fmap FormatterMap) (Format, os.Error) {
// parse source // parse source
var p parser var p parser
p.init(filename, src) p.init(fset, filename, src)
p.parseFormat() p.parseFormat()
// add custom formatters, if any // add custom formatters, if any

2
src/pkg/exp/eval/bridge.go
View file

@ -29,7 +29,7 @@ func TypeFromNative(t reflect.Type) Type {
var nt *NamedType var nt *NamedType
if t.Name() != "" { if t.Name() != "" {
name := t.PkgPath() + "·" + t.Name() name := t.PkgPath() + "·" + t.Name()
nt = &NamedType{token.Position{}, name, nil, true, make(map[string]Method)} nt = &NamedType{token.NoPos, name, nil, true, make(map[string]Method)}
evalTypes[t] = nt evalTypes[t] = nt
} }

14
src/pkg/exp/eval/compiler.go
View file

@ -11,24 +11,20 @@ import (
) )
type positioned interface {
Pos() token.Position
}
// A compiler captures information used throughout an entire // A compiler captures information used throughout an entire
// compilation. Currently it includes only the error handler. // compilation. Currently it includes only the error handler.
// //
// TODO(austin) This might actually represent package level, in which // TODO(austin) This might actually represent package level, in which
// case it should be package compiler. // case it should be package compiler.
type compiler struct { type compiler struct {
fset *token.FileSet
errors scanner.ErrorHandler errors scanner.ErrorHandler
numErrors int numErrors int
silentErrors int silentErrors int
} }
func (a *compiler) diagAt(pos positioned, format string, args ...interface{}) { func (a *compiler) diagAt(pos token.Pos, format string, args ...interface{}) {
a.errors.Error(pos.Pos(), fmt.Sprintf(format, args...)) a.errors.Error(a.fset.Position(pos), fmt.Sprintf(format, args...))
a.numErrors++ a.numErrors++
} }
@ -64,9 +60,9 @@ type label struct {
continuePC *uint continuePC *uint
// The position where this label was resolved. If it has not // The position where this label was resolved. If it has not
// been resolved yet, an invalid position. // been resolved yet, an invalid position.
resolved token.Position resolved token.Pos
// The position where this label was first jumped to. // The position where this label was first jumped to.
used token.Position used token.Pos
} }
// A funcCompiler captures information used throughout the compilation // A funcCompiler captures information used throughout the compilation

6
src/pkg/exp/eval/eval_test.go
View file

@ -8,6 +8,7 @@ import (
"big" "big"
"flag" "flag"
"fmt" "fmt"
"go/token"
"log" "log"
"os" "os"
"reflect" "reflect"
@ -15,6 +16,9 @@ import (
"testing" "testing"
) )
// All tests are done using the same file set.
var fset = token.NewFileSet()
// Print each statement or expression before parsing it // Print each statement or expression before parsing it
var noisy = false var noisy = false
@ -49,7 +53,7 @@ func (a test) run(t *testing.T, name string) {
println("code:", src) println("code:", src)
} }
code, err := w.Compile(src) code, err := w.Compile(fset, src)
if err != nil { if err != nil {
if j.cerr == "" { if j.cerr == "" {
t.Errorf("%s: Compile %s: %v", name, src, err) t.Errorf("%s: Compile %s: %v", name, src, err)

24
src/pkg/exp/eval/expr.go
View file

@ -57,7 +57,7 @@ type expr struct {
// compiled from it. // compiled from it.
type exprInfo struct { type exprInfo struct {
*compiler *compiler
pos token.Position pos token.Pos
} }
func (a *exprInfo) newExpr(t Type, desc string) *expr { func (a *exprInfo) newExpr(t Type, desc string) *expr {
@ -65,7 +65,7 @@ func (a *exprInfo) newExpr(t Type, desc string) *expr {
} }
func (a *exprInfo) diag(format string, args ...interface{}) { func (a *exprInfo) diag(format string, args ...interface{}) {
a.diagAt(&a.pos, format, args...) a.diagAt(a.pos, format, args...)
} }
func (a *exprInfo) diagOpType(op token.Token, vt Type) { func (a *exprInfo) diagOpType(op token.Token, vt Type) {
@ -229,7 +229,7 @@ func (a *expr) derefArray() *expr {
// multi-valued type. // multi-valued type.
type assignCompiler struct { type assignCompiler struct {
*compiler *compiler
pos token.Position pos token.Pos
// The RHS expressions. This may include nil's for // The RHS expressions. This may include nil's for
// expressions that failed to compile. // expressions that failed to compile.
rs []*expr rs []*expr
@ -254,7 +254,7 @@ type assignCompiler struct {
// assignCompiler with rmt set, but if type checking fails, slots in // assignCompiler with rmt set, but if type checking fails, slots in
// the MultiType may be nil. If rs contains nil's, type checking will // the MultiType may be nil. If rs contains nil's, type checking will
// fail and these expressions given a nil type. // fail and these expressions given a nil type.
func (a *compiler) checkAssign(pos token.Position, rs []*expr, errOp, errPosName string) (*assignCompiler, bool) { func (a *compiler) checkAssign(pos token.Pos, rs []*expr, errOp, errPosName string) (*assignCompiler, bool) {
c := &assignCompiler{ c := &assignCompiler{
compiler: a, compiler: a,
pos: pos, pos: pos,
@ -331,7 +331,7 @@ func (a *assignCompiler) compile(b *block, lt Type) func(Value, *Thread) {
pos = a.rs[lcount-1].pos pos = a.rs[lcount-1].pos
} }
} }
a.diagAt(&pos, "%s %ss for %s\n\t%s\n\t%s", msg, a.errPosName, a.errOp, lt, rmt) a.diagAt(pos, "%s %ss for %s\n\t%s\n\t%s", msg, a.errPosName, a.errOp, lt, rmt)
return nil return nil
} }
@ -453,7 +453,7 @@ func (a *assignCompiler) compile(b *block, lt Type) func(Value, *Thread) {
// compileAssign compiles an assignment operation without the full // compileAssign compiles an assignment operation without the full
// generality of an assignCompiler. See assignCompiler for a // generality of an assignCompiler. See assignCompiler for a
// description of the arguments. // description of the arguments.
func (a *compiler) compileAssign(pos token.Position, b *block, lt Type, rs []*expr, errOp, errPosName string) func(Value, *Thread) { func (a *compiler) compileAssign(pos token.Pos, b *block, lt Type, rs []*expr, errOp, errPosName string) func(Value, *Thread) {
ac, ok := a.checkAssign(pos, rs, errOp, errPosName) ac, ok := a.checkAssign(pos, rs, errOp, errPosName)
if !ok { if !ok {
return nil return nil
@ -514,7 +514,7 @@ func (a *exprCompiler) compile(x ast.Expr, callCtx bool) *expr {
return nil return nil
} }
if a.constant { if a.constant {
a.diagAt(x, "function literal used in constant expression") a.diagAt(x.Pos(), "function literal used in constant expression")
return nil return nil
} }
return ei.compileFuncLit(decl, fn) return ei.compileFuncLit(decl, fn)
@ -571,12 +571,12 @@ func (a *exprCompiler) compile(x ast.Expr, callCtx bool) *expr {
return nil return nil
} }
if a.constant { if a.constant {
a.diagAt(x, "function call in constant context") a.diagAt(x.Pos(), "function call in constant context")
return nil return nil
} }
if l.valType != nil { if l.valType != nil {
a.diagAt(x, "type conversions not implemented") a.diagAt(x.Pos(), "type conversions not implemented")
return nil return nil
} else if ft, ok := l.t.(*FuncType); ok && ft.builtin != "" { } else if ft, ok := l.t.(*FuncType); ok && ft.builtin != "" {
return ei.compileBuiltinCallExpr(a.block, ft, args) return ei.compileBuiltinCallExpr(a.block, ft, args)
@ -654,13 +654,13 @@ func (a *exprCompiler) compile(x ast.Expr, callCtx bool) *expr {
typeexpr: typeexpr:
if !callCtx { if !callCtx {
a.diagAt(x, "type used as expression") a.diagAt(x.Pos(), "type used as expression")
return nil return nil
} }
return ei.exprFromType(a.compileType(a.block, x)) return ei.exprFromType(a.compileType(a.block, x))
notimpl: notimpl:
a.diagAt(x, "%T expression node not implemented", x) a.diagAt(x.Pos(), "%T expression node not implemented", x)
return nil return nil
} }
@ -1920,7 +1920,7 @@ func (a *compiler) compileArrayLen(b *block, expr ast.Expr) (int64, bool) {
} }
if !lenExpr.t.isInteger() { if !lenExpr.t.isInteger() {
a.diagAt(expr, "array size must be an integer") a.diagAt(expr.Pos(), "array size must be an integer")
return 0, false return 0, false
} }

12
src/pkg/exp/eval/main.go
View file

@ -10,10 +10,12 @@ import (
"flag" "flag"
"go/parser" "go/parser"
"go/scanner" "go/scanner"
"go/token"
"io/ioutil" "io/ioutil"
"os" "os"
) )
var fset = token.NewFileSet()
var filename = flag.String("f", "", "file to run") var filename = flag.String("f", "", "file to run")
func main() { func main() {
@ -25,12 +27,12 @@ func main() {
println(err.String()) println(err.String())
os.Exit(1) os.Exit(1)
} }
file, err := parser.ParseFile(*filename, data, 0) file, err := parser.ParseFile(fset, *filename, data, 0)
if err != nil { if err != nil {
println(err.String()) println(err.String())
os.Exit(1) os.Exit(1)
} }
code, err := w.CompileDeclList(file.Decls) code, err := w.CompileDeclList(fset, file.Decls)
if err != nil { if err != nil {
if list, ok := err.(scanner.ErrorList); ok { if list, ok := err.(scanner.ErrorList); ok {
for _, e := range list { for _, e := range list {
@ -46,7 +48,7 @@ func main() {
println(err.String()) println(err.String())
os.Exit(1) os.Exit(1)
} }
code, err = w.Compile("init()") code, err = w.Compile(fset, "init()")
if code != nil { if code != nil {
_, err := code.Run() _, err := code.Run()
if err != nil { if err != nil {
@ -54,7 +56,7 @@ func main() {
os.Exit(1) os.Exit(1)
} }
} }
code, err = w.Compile("main()") code, err = w.Compile(fset, "main()")
if err != nil { if err != nil {
println(err.String()) println(err.String())
os.Exit(1) os.Exit(1)
@ -74,7 +76,7 @@ func main() {
if err != nil { if err != nil {
break break
} }
code, err := w.Compile(line) code, err := w.Compile(fset, line)
if err != nil { if err != nil {
println(err.String()) println(err.String())
continue continue

28
src/pkg/exp/eval/scope.go
View file

@ -15,11 +15,11 @@ import (
// A definition can be a *Variable, *Constant, or Type. // A definition can be a *Variable, *Constant, or Type.
type Def interface { type Def interface {
Pos() token.Position Pos() token.Pos
} }
type Variable struct { type Variable struct {
token.Position VarPos token.Pos
// Index of this variable in the Frame structure // Index of this variable in the Frame structure
Index int Index int
// Static type of this variable // Static type of this variable
@ -30,10 +30,18 @@ type Variable struct {
Init Value Init Value
} }
func (v *Variable) Pos() token.Pos {
return v.VarPos
}
type Constant struct { type Constant struct {
token.Position ConstPos token.Pos
Type Type Type Type
Value Value Value Value
}
func (c *Constant) Pos() token.Pos {
return c.ConstPos
} }
// A block represents a definition block in which a name may not be // A block represents a definition block in which a name may not be
@ -111,12 +119,12 @@ func (b *block) ChildScope() *Scope {
return sub.scope return sub.scope
} }
func (b *block) DefineVar(name string, pos token.Position, t Type) (*Variable, Def) { func (b *block) DefineVar(name string, pos token.Pos, t Type) (*Variable, Def) {
if prev, ok := b.defs[name]; ok { if prev, ok := b.defs[name]; ok {
return nil, prev return nil, prev
} }
v := b.defineSlot(t, false) v := b.defineSlot(t, false)
v.Position = pos v.VarPos = pos
b.defs[name] = v b.defs[name] = v
return v, nil return v, nil
} }
@ -135,11 +143,11 @@ func (b *block) defineSlot(t Type, temp bool) *Variable {
b.scope.maxVars = index + 1 b.scope.maxVars = index + 1
} }
} }
v := &Variable{token.Position{}, index, t, nil} v := &Variable{token.NoPos, index, t, nil}
return v return v
} }
func (b *block) DefineConst(name string, pos token.Position, t Type, v Value) (*Constant, Def) { func (b *block) DefineConst(name string, pos token.Pos, t Type, v Value) (*Constant, Def) {
if prev, ok := b.defs[name]; ok { if prev, ok := b.defs[name]; ok {
return nil, prev return nil, prev
} }
@ -148,7 +156,7 @@ func (b *block) DefineConst(name string, pos token.Position, t Type, v Value) (*
return c, nil return c, nil
} }
func (b *block) DefineType(name string, pos token.Position, t Type) Type { func (b *block) DefineType(name string, pos token.Pos, t Type) Type {
if _, ok := b.defs[name]; ok { if _, ok := b.defs[name]; ok {
return nil return nil
} }

41
src/pkg/exp/eval/stmt.go
View file

@ -22,13 +22,13 @@ const (
type stmtCompiler struct { type stmtCompiler struct {
*blockCompiler *blockCompiler
pos token.Position pos token.Pos
// This statement's label, or nil if it is not labeled. // This statement's label, or nil if it is not labeled.
stmtLabel *label stmtLabel *label
} }
func (a *stmtCompiler) diag(format string, args ...interface{}) { func (a *stmtCompiler) diag(format string, args ...interface{}) {
a.diagAt(&a.pos, format, args...) a.diagAt(a.pos, format, args...)
} }
/* /*
@ -65,7 +65,7 @@ type flowBuf struct {
ents map[uint]*flowEnt ents map[uint]*flowEnt
// gotos is a map from goto positions to information on the // gotos is a map from goto positions to information on the
// block at the point of the goto. // block at the point of the goto.
gotos map[*token.Position]*flowBlock gotos map[token.Pos]*flowBlock
// labels is a map from label name to information on the block // labels is a map from label name to information on the block
// at the point of the label. labels are tracked by name, // at the point of the label. labels are tracked by name,
// since mutliple labels at the same PC can have different // since mutliple labels at the same PC can have different
@ -74,7 +74,7 @@ type flowBuf struct {
} }
func newFlowBuf(cb *codeBuf) *flowBuf { func newFlowBuf(cb *codeBuf) *flowBuf {
return &flowBuf{cb, make(map[uint]*flowEnt), make(map[*token.Position]*flowBlock), make(map[string]*flowBlock)} return &flowBuf{cb, make(map[uint]*flowEnt), make(map[token.Pos]*flowBlock), make(map[string]*flowBlock)}
} }
// put creates a flow control point for the next PC in the code buffer. // put creates a flow control point for the next PC in the code buffer.
@ -123,8 +123,8 @@ func newFlowBlock(target string, b *block) *flowBlock {
// putGoto captures the block at a goto statement. This should be // putGoto captures the block at a goto statement. This should be
// called in addition to putting a flow control point. // called in addition to putting a flow control point.
func (f *flowBuf) putGoto(pos token.Position, target string, b *block) { func (f *flowBuf) putGoto(pos token.Pos, target string, b *block) {
f.gotos[&pos] = newFlowBlock(target, b) f.gotos[pos] = newFlowBlock(target, b)
} }
// putLabel captures the block at a label. // putLabel captures the block at a label.
@ -212,13 +212,10 @@ func (f *flowBuf) gotosObeyScopes(a *compiler) {
func (a *stmtCompiler) defineVar(ident *ast.Ident, t Type) *Variable { func (a *stmtCompiler) defineVar(ident *ast.Ident, t Type) *Variable {
v, prev := a.block.DefineVar(ident.Name, ident.Pos(), t) v, prev := a.block.DefineVar(ident.Name, ident.Pos(), t)
if prev != nil { if prev != nil {
// TODO(austin) It's silly that we have to capture if prev.Pos().IsValid() {
// Pos() in a variable. a.diagAt(ident.Pos(), "variable %s redeclared in this block\n\tprevious declaration at %s", ident.Name, a.fset.Position(prev.Pos()))
pos := prev.Pos()
if pos.IsValid() {
a.diagAt(ident, "variable %s redeclared in this block\n\tprevious declaration at %s", ident.Name, &pos)
} else { } else {
a.diagAt(ident, "variable %s redeclared in this block", ident.Name) a.diagAt(ident.Pos(), "variable %s redeclared in this block", ident.Name)
} }
return nil return nil
} }
@ -385,9 +382,9 @@ func (a *stmtCompiler) compileDecl(decl ast.Decl) {
if prev != nil { if prev != nil {
pos := prev.Pos() pos := prev.Pos()
if pos.IsValid() { if pos.IsValid() {
a.diagAt(d.Name, "identifier %s redeclared in this block\n\tprevious declaration at %s", d.Name.Name, &pos) a.diagAt(d.Name.Pos(), "identifier %s redeclared in this block\n\tprevious declaration at %s", d.Name.Name, a.fset.Position(pos))
} else { } else {
a.diagAt(d.Name, "identifier %s redeclared in this block", d.Name.Name) a.diagAt(d.Name.Pos(), "identifier %s redeclared in this block", d.Name.Name)
} }
} }
fn := a.compileFunc(a.block, decl, d.Body) fn := a.compileFunc(a.block, decl, d.Body)
@ -419,7 +416,7 @@ func (a *stmtCompiler) compileLabeledStmt(s *ast.LabeledStmt) {
l, ok := a.labels[s.Label.Name] l, ok := a.labels[s.Label.Name]
if ok { if ok {
if l.resolved.IsValid() { if l.resolved.IsValid() {
a.diag("label %s redeclared in this block\n\tprevious declaration at %s", s.Label.Name, &l.resolved) a.diag("label %s redeclared in this block\n\tprevious declaration at %s", s.Label.Name, a.fset.Position(l.resolved))
} }
} else { } else {
pc := badPC pc := badPC
@ -555,7 +552,7 @@ func (a *stmtCompiler) doAssign(lhs []ast.Expr, rhs []ast.Expr, tok token.Token,
// Check that it's an identifier // Check that it's an identifier
ident, ok = le.(*ast.Ident) ident, ok = le.(*ast.Ident)
if !ok { if !ok {
a.diagAt(le, "left side of := must be a name") a.diagAt(le.Pos(), "left side of := must be a name")
// Suppress new defitions errors // Suppress new defitions errors
nDefs++ nDefs++
continue continue
@ -1012,12 +1009,12 @@ func (a *stmtCompiler) compileSwitchStmt(s *ast.SwitchStmt) {
for _, c := range s.Body.List { for _, c := range s.Body.List {
clause, ok := c.(*ast.CaseClause) clause, ok := c.(*ast.CaseClause)
if !ok { if !ok {
a.diagAt(clause, "switch statement must contain case clauses") a.diagAt(clause.Pos(), "switch statement must contain case clauses")
continue continue
} }
if clause.Values == nil { if clause.Values == nil {
if hasDefault { if hasDefault {
a.diagAt(clause, "switch statement contains more than one default case") a.diagAt(clause.Pos(), "switch statement contains more than one default case")
} }
hasDefault = true hasDefault = true
} else { } else {
@ -1039,7 +1036,7 @@ func (a *stmtCompiler) compileSwitchStmt(s *ast.SwitchStmt) {
case e == nil: case e == nil:
// Error reported by compileExpr // Error reported by compileExpr
case cond == nil && !e.t.isBoolean(): case cond == nil && !e.t.isBoolean():
a.diagAt(v, "'case' condition must be boolean") a.diagAt(v.Pos(), "'case' condition must be boolean")
case cond == nil: case cond == nil:
cases[i] = e.asBool() cases[i] = e.asBool()
case cond != nil: case cond != nil:
@ -1104,7 +1101,7 @@ func (a *stmtCompiler) compileSwitchStmt(s *ast.SwitchStmt) {
// empty blocks to be empty // empty blocks to be empty
// statements. // statements.
if _, ok := s2.(*ast.EmptyStmt); !ok { if _, ok := s2.(*ast.EmptyStmt); !ok {
a.diagAt(s, "fallthrough statement must be final statement in case") a.diagAt(s.Pos(), "fallthrough statement must be final statement in case")
break break
} }
} }
@ -1275,7 +1272,7 @@ func (a *compiler) compileFunc(b *block, decl *FuncDecl, body *ast.BlockStmt) fu
// this if there were no errors compiling the body. // this if there were no errors compiling the body.
if len(decl.Type.Out) > 0 && fc.flow.reachesEnd(0) { if len(decl.Type.Out) > 0 && fc.flow.reachesEnd(0) {
// XXX(Spec) Not specified. // XXX(Spec) Not specified.
a.diagAt(&body.Rbrace, "function ends without a return statement") a.diagAt(body.Rbrace, "function ends without a return statement")
return nil return nil
} }
@ -1290,7 +1287,7 @@ func (a *funcCompiler) checkLabels() {
nerr := a.numError() nerr := a.numError()
for _, l := range a.labels { for _, l := range a.labels {
if !l.resolved.IsValid() { if !l.resolved.IsValid() {
a.diagAt(&l.used, "label %s not defined", l.name) a.diagAt(l.used, "label %s not defined", l.name)
} }
} }
if nerr != a.numError() { if nerr != a.numError() {

16
src/pkg/exp/eval/type.go
View file

@ -54,7 +54,7 @@ type Type interface {
// String returns the string representation of this type. // String returns the string representation of this type.
String() string String() string
// The position where this type was defined, if any. // The position where this type was defined, if any.
Pos() token.Position Pos() token.Pos
} }
type BoundedType interface { type BoundedType interface {
@ -65,7 +65,7 @@ type BoundedType interface {
maxVal() *big.Rat maxVal() *big.Rat
} }
var universePos = token.Position{"<universe>", 0, 0, 0} var universePos = token.NoPos
/* /*
* Type array maps. These are used to memoize composite types. * Type array maps. These are used to memoize composite types.
@ -140,7 +140,7 @@ func (commonType) isFloat() bool { return false }
func (commonType) isIdeal() bool { return false } func (commonType) isIdeal() bool { return false }
func (commonType) Pos() token.Position { return token.Position{} } func (commonType) Pos() token.Pos { return token.NoPos }
/* /*
* Bool * Bool
@ -1100,8 +1100,8 @@ type Method struct {
} }
type NamedType struct { type NamedType struct {
token.Position NamePos token.Pos
Name string Name string
// Underlying type. If incomplete is true, this will be nil. // Underlying type. If incomplete is true, this will be nil.
// If incomplete is false and this is still nil, then this is // If incomplete is false and this is still nil, then this is
// a placeholder type representing an error. // a placeholder type representing an error.
@ -1114,7 +1114,11 @@ type NamedType struct {
// TODO(austin) This is temporarily needed by the debugger's remote // TODO(austin) This is temporarily needed by the debugger's remote
// type parser. This should only be possible with block.DefineType. // type parser. This should only be possible with block.DefineType.
func NewNamedType(name string) *NamedType { func NewNamedType(name string) *NamedType {
return &NamedType{token.Position{}, name, nil, true, make(map[string]Method)} return &NamedType{token.NoPos, name, nil, true, make(map[string]Method)}
}
func (t *NamedType) Pos() token.Pos {
return t.NamePos
} }
func (t *NamedType) Complete(def Type) { func (t *NamedType) Complete(def Type) {

44
src/pkg/exp/eval/typec.go
View file

@ -28,19 +28,19 @@ type typeCompiler struct {
func (a *typeCompiler) compileIdent(x *ast.Ident, allowRec bool) Type { func (a *typeCompiler) compileIdent(x *ast.Ident, allowRec bool) Type {
_, _, def := a.block.Lookup(x.Name) _, _, def := a.block.Lookup(x.Name)
if def == nil { if def == nil {
a.diagAt(x, "%s: undefined", x.Name) a.diagAt(x.Pos(), "%s: undefined", x.Name)
return nil return nil
} }
switch def := def.(type) { switch def := def.(type) {
case *Constant: case *Constant:
a.diagAt(x, "constant %v used as type", x.Name) a.diagAt(x.Pos(), "constant %v used as type", x.Name)
return nil return nil
case *Variable: case *Variable:
a.diagAt(x, "variable %v used as type", x.Name) a.diagAt(x.Pos(), "variable %v used as type", x.Name)
return nil return nil
case *NamedType: case *NamedType:
if !allowRec && def.incomplete { if !allowRec && def.incomplete {
a.diagAt(x, "illegal recursive type") a.diagAt(x.Pos(), "illegal recursive type")
return nil return nil
} }
if !def.incomplete && def.Def == nil { if !def.incomplete && def.Def == nil {
@ -68,7 +68,7 @@ func (a *typeCompiler) compileArrayType(x *ast.ArrayType, allowRec bool) Type {
} }
if _, ok := x.Len.(*ast.Ellipsis); ok { if _, ok := x.Len.(*ast.Ellipsis); ok {
a.diagAt(x.Len, "... array initailizers not implemented") a.diagAt(x.Len.Pos(), "... array initailizers not implemented")
return nil return nil
} }
l, ok := a.compileArrayLen(a.block, x.Len) l, ok := a.compileArrayLen(a.block, x.Len)
@ -76,7 +76,7 @@ func (a *typeCompiler) compileArrayType(x *ast.ArrayType, allowRec bool) Type {
return nil return nil
} }
if l < 0 { if l < 0 {
a.diagAt(x.Len, "array length must be non-negative") a.diagAt(x.Len.Pos(), "array length must be non-negative")
return nil return nil
} }
if elem == nil { if elem == nil {
@ -86,11 +86,11 @@ func (a *typeCompiler) compileArrayType(x *ast.ArrayType, allowRec bool) Type {
return NewArrayType(l, elem) return NewArrayType(l, elem)
} }
func (a *typeCompiler) compileFields(fields *ast.FieldList, allowRec bool) ([]Type, []*ast.Ident, []token.Position, bool) { func (a *typeCompiler) compileFields(fields *ast.FieldList, allowRec bool) ([]Type, []*ast.Ident, []token.Pos, bool) {
n := fields.NumFields() n := fields.NumFields()
ts := make([]Type, n) ts := make([]Type, n)
ns := make([]*ast.Ident, n) ns := make([]*ast.Ident, n)
ps := make([]token.Position, n) ps := make([]token.Pos, n)
bad := false bad := false
if fields != nil { if fields != nil {
@ -132,7 +132,7 @@ func (a *typeCompiler) compileStructType(x *ast.StructType, allowRec bool) Type
// uniqueness of field names inherited from anonymous fields // uniqueness of field names inherited from anonymous fields
// at use time. // at use time.
fields := make([]StructField, len(ts)) fields := make([]StructField, len(ts))
nameSet := make(map[string]token.Position, len(ts)) nameSet := make(map[string]token.Pos, len(ts))
for i := range fields { for i := range fields {
// Compute field name and check anonymous fields // Compute field name and check anonymous fields
var name string var name string
@ -162,7 +162,7 @@ func (a *typeCompiler) compileStructType(x *ast.StructType, allowRec bool) Type
// *T, and T itself, may not be a pointer or // *T, and T itself, may not be a pointer or
// interface type. // interface type.
if nt == nil { if nt == nil {
a.diagAt(&poss[i], "embedded type must T or *T, where T is a named type") a.diagAt(poss[i], "embedded type must T or *T, where T is a named type")
bad = true bad = true
continue continue
} }
@ -172,7 +172,7 @@ func (a *typeCompiler) compileStructType(x *ast.StructType, allowRec bool) Type
lateCheck := a.lateCheck lateCheck := a.lateCheck
a.lateCheck = func() bool { a.lateCheck = func() bool {
if _, ok := nt.lit().(*PtrType); ok { if _, ok := nt.lit().(*PtrType); ok {
a.diagAt(&poss[i], "embedded type %v is a pointer type", nt) a.diagAt(poss[i], "embedded type %v is a pointer type", nt)
return false return false
} }
return lateCheck() return lateCheck()
@ -181,7 +181,7 @@ func (a *typeCompiler) compileStructType(x *ast.StructType, allowRec bool) Type
// Check name uniqueness // Check name uniqueness
if prev, ok := nameSet[name]; ok { if prev, ok := nameSet[name]; ok {
a.diagAt(&poss[i], "field %s redeclared\n\tprevious declaration at %s", name, &prev) a.diagAt(poss[i], "field %s redeclared\n\tprevious declaration at %s", name, a.fset.Position(prev))
bad = true bad = true
continue continue
} }
@ -227,7 +227,7 @@ func (a *typeCompiler) compileInterfaceType(x *ast.InterfaceType, allowRec bool)
ts, names, poss, bad := a.compileFields(x.Methods, allowRec) ts, names, poss, bad := a.compileFields(x.Methods, allowRec)
methods := make([]IMethod, len(ts)) methods := make([]IMethod, len(ts))
nameSet := make(map[string]token.Position, len(ts)) nameSet := make(map[string]token.Pos, len(ts))
embeds := make([]*InterfaceType, len(ts)) embeds := make([]*InterfaceType, len(ts))
var nm, ne int var nm, ne int
@ -242,7 +242,7 @@ func (a *typeCompiler) compileInterfaceType(x *ast.InterfaceType, allowRec bool)
methods[nm].Type = ts[i].(*FuncType) methods[nm].Type = ts[i].(*FuncType)
nm++ nm++
if prev, ok := nameSet[name]; ok { if prev, ok := nameSet[name]; ok {
a.diagAt(&poss[i], "method %s redeclared\n\tprevious declaration at %s", name, &prev) a.diagAt(poss[i], "method %s redeclared\n\tprevious declaration at %s", name, a.fset.Position(prev))
bad = true bad = true
continue continue
} }
@ -251,7 +251,7 @@ func (a *typeCompiler) compileInterfaceType(x *ast.InterfaceType, allowRec bool)
// Embedded interface // Embedded interface
it, ok := ts[i].lit().(*InterfaceType) it, ok := ts[i].lit().(*InterfaceType)
if !ok { if !ok {
a.diagAt(&poss[i], "embedded type must be an interface") a.diagAt(poss[i], "embedded type must be an interface")
bad = true bad = true
continue continue
} }
@ -259,7 +259,7 @@ func (a *typeCompiler) compileInterfaceType(x *ast.InterfaceType, allowRec bool)
ne++ ne++
for _, m := range it.methods { for _, m := range it.methods {
if prev, ok := nameSet[m.Name]; ok { if prev, ok := nameSet[m.Name]; ok {
a.diagAt(&poss[i], "method %s redeclared\n\tprevious declaration at %s", m.Name, &prev) a.diagAt(poss[i], "method %s redeclared\n\tprevious declaration at %s", m.Name, a.fset.Position(prev))
bad = true bad = true
continue continue
} }
@ -288,13 +288,13 @@ func (a *typeCompiler) compileMapType(x *ast.MapType) Type {
// that can be map keys except for function types. // that can be map keys except for function types.
switch key.lit().(type) { switch key.lit().(type) {
case *StructType: case *StructType:
a.diagAt(x, "map key cannot be a struct type") a.diagAt(x.Pos(), "map key cannot be a struct type")
return nil return nil
case *ArrayType: case *ArrayType:
a.diagAt(x, "map key cannot be an array type") a.diagAt(x.Pos(), "map key cannot be an array type")
return nil return nil
case *SliceType: case *SliceType:
a.diagAt(x, "map key cannot be a slice type") a.diagAt(x.Pos(), "map key cannot be a slice type")
return nil return nil
} }
return NewMapType(key, val) return NewMapType(key, val)
@ -339,14 +339,14 @@ func (a *typeCompiler) compileType(x ast.Expr, allowRec bool) Type {
return a.compileType(x.X, allowRec) return a.compileType(x.X, allowRec)
case *ast.Ellipsis: case *ast.Ellipsis:
a.diagAt(x, "illegal use of ellipsis") a.diagAt(x.Pos(), "illegal use of ellipsis")
return nil return nil
} }
a.diagAt(x, "expression used as type") a.diagAt(x.Pos(), "expression used as type")
return nil return nil
notimpl: notimpl:
a.diagAt(x, "compileType: %T not implemented", x) a.diagAt(x.Pos(), "compileType: %T not implemented", x)
return nil return nil
} }

33
src/pkg/exp/eval/world.go
View file

@ -41,14 +41,14 @@ type stmtCode struct {
code code code code
} }
func (w *World) CompileStmtList(stmts []ast.Stmt) (Code, os.Error) { func (w *World) CompileStmtList(fset *token.FileSet, stmts []ast.Stmt) (Code, os.Error) {
if len(stmts) == 1 { if len(stmts) == 1 {
if s, ok := stmts[0].(*ast.ExprStmt); ok { if s, ok := stmts[0].(*ast.ExprStmt); ok {
return w.CompileExpr(s.X) return w.CompileExpr(fset, s.X)
} }
} }
errors := new(scanner.ErrorVector) errors := new(scanner.ErrorVector)
cc := &compiler{errors, 0, 0} cc := &compiler{fset, errors, 0, 0}
cb := newCodeBuf() cb := newCodeBuf()
fc := &funcCompiler{ fc := &funcCompiler{
compiler: cc, compiler: cc,
@ -73,12 +73,12 @@ func (w *World) CompileStmtList(stmts []ast.Stmt) (Code, os.Error) {
return &stmtCode{w, fc.get()}, nil return &stmtCode{w, fc.get()}, nil
} }
func (w *World) CompileDeclList(decls []ast.Decl) (Code, os.Error) { func (w *World) CompileDeclList(fset *token.FileSet, decls []ast.Decl) (Code, os.Error) {
stmts := make([]ast.Stmt, len(decls)) stmts := make([]ast.Stmt, len(decls))
for i, d := range decls { for i, d := range decls {
stmts[i] = &ast.DeclStmt{d} stmts[i] = &ast.DeclStmt{d}
} }
return w.CompileStmtList(stmts) return w.CompileStmtList(fset, stmts)
} }
func (s *stmtCode) Type() Type { return nil } func (s *stmtCode) Type() Type { return nil }
@ -95,9 +95,9 @@ type exprCode struct {
eval func(Value, *Thread) eval func(Value, *Thread)
} }
func (w *World) CompileExpr(e ast.Expr) (Code, os.Error) { func (w *World) CompileExpr(fset *token.FileSet, e ast.Expr) (Code, os.Error) {
errors := new(scanner.ErrorVector) errors := new(scanner.ErrorVector)
cc := &compiler{errors, 0, 0} cc := &compiler{fset, errors, 0, 0}
ec := cc.compileExpr(w.scope.block, false, e) ec := cc.compileExpr(w.scope.block, false, e)
if ec == nil { if ec == nil {
@ -135,16 +135,16 @@ func (e *exprCode) Run() (Value, os.Error) {
return v, err return v, err
} }
func (w *World) Compile(text string) (Code, os.Error) { func (w *World) Compile(fset *token.FileSet, text string) (Code, os.Error) {
stmts, err := parser.ParseStmtList("input", text) stmts, err := parser.ParseStmtList(fset, "input", text)
if err == nil { if err == nil {
return w.CompileStmtList(stmts) return w.CompileStmtList(fset, stmts)
} }
// Otherwise try as DeclList. // Otherwise try as DeclList.
decls, err1 := parser.ParseDeclList("input", text) decls, err1 := parser.ParseDeclList(fset, "input", text)
if err1 == nil { if err1 == nil {
return w.CompileDeclList(decls) return w.CompileDeclList(fset, decls)
} }
// Have to pick an error. // Have to pick an error.
@ -162,13 +162,16 @@ func (e *RedefinitionError) String() string {
res := "identifier " + e.Name + " redeclared" res := "identifier " + e.Name + " redeclared"
pos := e.Prev.Pos() pos := e.Prev.Pos()
if pos.IsValid() { if pos.IsValid() {
res += "; previous declaration at " + pos.String() // TODO: fix this - currently this code is not reached by the tests
// need to get a file set (fset) from somewhere
//res += "; previous declaration at " + fset.Position(pos).String()
panic(0)
} }
return res return res
} }
func (w *World) DefineConst(name string, t Type, val Value) os.Error { func (w *World) DefineConst(name string, t Type, val Value) os.Error {
_, prev := w.scope.DefineConst(name, token.Position{}, t, val) _, prev := w.scope.DefineConst(name, token.NoPos, t, val)
if prev != nil { if prev != nil {
return &RedefinitionError{name, prev} return &RedefinitionError{name, prev}
} }
@ -176,7 +179,7 @@ func (w *World) DefineConst(name string, t Type, val Value) os.Error {
} }
func (w *World) DefineVar(name string, t Type, val Value) os.Error { func (w *World) DefineVar(name string, t Type, val Value) os.Error {
v, prev := w.scope.DefineVar(name, token.Position{}, t) v, prev := w.scope.DefineVar(name, token.NoPos, t)
if prev != nil { if prev != nil {
return &RedefinitionError{name, prev} return &RedefinitionError{name, prev}
} }

8
src/pkg/exp/ogle/cmd.go
View file

@ -18,6 +18,7 @@ import (
"strings" "strings"
) )
var fset = token.NewFileSet()
var world *eval.World var world *eval.World
var curProc *Process var curProc *Process
@ -43,7 +44,7 @@ func Main() {
} }
// Try line as code // Try line as code
code, err := world.Compile(string(line)) code, err := world.Compile(fset, string(line))
if err != nil { if err != nil {
scanner.PrintError(os.Stderr, err) scanner.PrintError(os.Stderr, err)
continue continue
@ -63,8 +64,7 @@ func Main() {
func newScanner(input []byte) (*scanner.Scanner, *scanner.ErrorVector) { func newScanner(input []byte) (*scanner.Scanner, *scanner.ErrorVector) {
sc := new(scanner.Scanner) sc := new(scanner.Scanner)
ev := new(scanner.ErrorVector) ev := new(scanner.ErrorVector)
sc.Init("input", input, ev, 0) sc.Init(fset, "input", input, ev, 0)
return sc, ev return sc, ev
} }
@ -101,7 +101,7 @@ func getCmd(line []byte) (*cmd, []byte) {
slit := string(lit) slit := string(lit)
for i := range cmds { for i := range cmds {
if cmds[i].cmd == slit { if cmds[i].cmd == slit {
return &cmds[i], line[pos.Offset+len(lit):] return &cmds[i], line[fset.Position(pos).Offset+len(lit):]
} }
} }
return nil, nil return nil, nil

296
src/pkg/go/ast/ast.go
View file

@ -35,7 +35,7 @@ import (
// All node types implement the Node interface. // All node types implement the Node interface.
type Node interface { type Node interface {
// Pos returns the (beginning) position of the node. // Pos returns the (beginning) position of the node.
Pos() token.Position Pos() token.Pos
} }
@ -65,12 +65,12 @@ type Decl interface {
// A Comment node represents a single //-style or /*-style comment. // A Comment node represents a single //-style or /*-style comment.
type Comment struct { type Comment struct {
Slash token.Position // position of "/" starting the comment Slash token.Pos // position of "/" starting the comment
Text []byte // comment text (excluding '\n' for //-style comments) Text []byte // comment text (excluding '\n' for //-style comments)
} }
func (c *Comment) Pos() token.Position { func (c *Comment) Pos() token.Pos {
return c.Slash return c.Slash
} }
@ -99,7 +99,7 @@ type Field struct {
} }
func (f *Field) Pos() token.Position { func (f *Field) Pos() token.Pos {
if len(f.Names) > 0 { if len(f.Names) > 0 {
return f.Names[0].Pos() return f.Names[0].Pos()
} }
@ -109,9 +109,9 @@ func (f *Field) Pos() token.Position {
// A FieldList represents a list of Fields, enclosed by parentheses or braces. // A FieldList represents a list of Fields, enclosed by parentheses or braces.
type FieldList struct { type FieldList struct {
Opening token.Position // position of opening parenthesis/brace Opening token.Pos // position of opening parenthesis/brace
List []*Field // field list List []*Field // field list
Closing token.Position // position of closing parenthesis/brace Closing token.Pos // position of closing parenthesis/brace
} }
@ -140,29 +140,29 @@ type (
// created. // created.
// //
BadExpr struct { BadExpr struct {
Begin token.Position // beginning position of bad expression Begin token.Pos // beginning position of bad expression
} }
// An Ident node represents an identifier. // An Ident node represents an identifier.
Ident struct { Ident struct {
NamePos token.Position // identifier position NamePos token.Pos // identifier position
Name string // identifier name Name string // identifier name
Obj *Object // denoted object; or nil Obj *Object // denoted object; or nil
} }
// An Ellipsis node stands for the "..." type in a // An Ellipsis node stands for the "..." type in a
// parameter list or the "..." length in an array type. // parameter list or the "..." length in an array type.
// //
Ellipsis struct { Ellipsis struct {
Ellipsis token.Position // position of "..." Ellipsis token.Pos // position of "..."
Elt Expr // ellipsis element type (parameter lists only) Elt Expr // ellipsis element type (parameter lists only)
} }
// A BasicLit node represents a literal of basic type. // A BasicLit node represents a literal of basic type.
BasicLit struct { BasicLit struct {
ValuePos token.Position // literal position ValuePos token.Pos // literal position
Kind token.Token // token.INT, token.FLOAT, token.IMAG, token.CHAR, or token.STRING Kind token.Token // token.INT, token.FLOAT, token.IMAG, token.CHAR, or token.STRING
Value []byte // literal string; e.g. 42, 0x7f, 3.14, 1e-9, 2.4i, 'a', '\x7f', "foo" or `\m\n\o` Value []byte // literal string; e.g. 42, 0x7f, 3.14, 1e-9, 2.4i, 'a', '\x7f', "foo" or `\m\n\o`
} }
// A FuncLit node represents a function literal. // A FuncLit node represents a function literal.
@ -173,17 +173,17 @@ type (
// A CompositeLit node represents a composite literal. // A CompositeLit node represents a composite literal.
CompositeLit struct { CompositeLit struct {
Type Expr // literal type; or nil Type Expr // literal type; or nil
Lbrace token.Position // position of "{" Lbrace token.Pos // position of "{"
Elts []Expr // list of composite elements Elts []Expr // list of composite elements
Rbrace token.Position // position of "}" Rbrace token.Pos // position of "}"
} }
// A ParenExpr node represents a parenthesized expression. // A ParenExpr node represents a parenthesized expression.
ParenExpr struct { ParenExpr struct {
Lparen token.Position // position of "(" Lparen token.Pos // position of "("
X Expr // parenthesized expression X Expr // parenthesized expression
Rparen token.Position // position of ")" Rparen token.Pos // position of ")"
} }
// A SelectorExpr node represents an expression followed by a selector. // A SelectorExpr node represents an expression followed by a selector.
@ -215,36 +215,36 @@ type (
// A CallExpr node represents an expression followed by an argument list. // A CallExpr node represents an expression followed by an argument list.
CallExpr struct { CallExpr struct {
Fun Expr // function expression Fun Expr // function expression
Lparen token.Position // position of "(" Lparen token.Pos // position of "("
Args []Expr // function arguments Args []Expr // function arguments
Ellipsis token.Position // position of "...", if any Ellipsis token.Pos // position of "...", if any
Rparen token.Position // position of ")" Rparen token.Pos // position of ")"
} }
// A StarExpr node represents an expression of the form "*" Expression. // A StarExpr node represents an expression of the form "*" Expression.
// Semantically it could be a unary "*" expression, or a pointer type. // Semantically it could be a unary "*" expression, or a pointer type.
// //
StarExpr struct { StarExpr struct {
Star token.Position // position of "*" Star token.Pos // position of "*"
X Expr // operand X Expr // operand
} }
// A UnaryExpr node represents a unary expression. // A UnaryExpr node represents a unary expression.
// Unary "*" expressions are represented via StarExpr nodes. // Unary "*" expressions are represented via StarExpr nodes.
// //
UnaryExpr struct { UnaryExpr struct {
OpPos token.Position // position of Op OpPos token.Pos // position of Op
Op token.Token // operator Op token.Token // operator
X Expr // operand X Expr // operand
} }
// A BinaryExpr node represents a binary expression. // A BinaryExpr node represents a binary expression.
BinaryExpr struct { BinaryExpr struct {
X Expr // left operand X Expr // left operand
OpPos token.Position // position of Op OpPos token.Pos // position of Op
Op token.Token // operator Op token.Token // operator
Y Expr // right operand Y Expr // right operand
} }
// A KeyValueExpr node represents (key : value) pairs // A KeyValueExpr node represents (key : value) pairs
@ -252,7 +252,7 @@ type (
// //
KeyValueExpr struct { KeyValueExpr struct {
Key Expr Key Expr
Colon token.Position // position of ":" Colon token.Pos // position of ":"
Value Expr Value Expr
} }
) )
@ -276,79 +276,79 @@ const (
type ( type (
// An ArrayType node represents an array or slice type. // An ArrayType node represents an array or slice type.
ArrayType struct { ArrayType struct {
Lbrack token.Position // position of "[" Lbrack token.Pos // position of "["
Len Expr // Ellipsis node for [...]T array types, nil for slice types Len Expr // Ellipsis node for [...]T array types, nil for slice types
Elt Expr // element type Elt Expr // element type
} }
// A StructType node represents a struct type. // A StructType node represents a struct type.
StructType struct { StructType struct {
Struct token.Position // position of "struct" keyword Struct token.Pos // position of "struct" keyword
Fields *FieldList // list of field declarations Fields *FieldList // list of field declarations
Incomplete bool // true if (source) fields are missing in the Fields list Incomplete bool // true if (source) fields are missing in the Fields list
} }
// Pointer types are represented via StarExpr nodes. // Pointer types are represented via StarExpr nodes.
// A FuncType node represents a function type. // A FuncType node represents a function type.
FuncType struct { FuncType struct {
Func token.Position // position of "func" keyword Func token.Pos // position of "func" keyword
Params *FieldList // (incoming) parameters Params *FieldList // (incoming) parameters
Results *FieldList // (outgoing) results Results *FieldList // (outgoing) results
} }
// An InterfaceType node represents an interface type. // An InterfaceType node represents an interface type.
InterfaceType struct { InterfaceType struct {
Interface token.Position // position of "interface" keyword Interface token.Pos // position of "interface" keyword
Methods *FieldList // list of methods Methods *FieldList // list of methods
Incomplete bool // true if (source) methods are missing in the Methods list Incomplete bool // true if (source) methods are missing in the Methods list
} }
// A MapType node represents a map type. // A MapType node represents a map type.
MapType struct { MapType struct {
Map token.Position // position of "map" keyword Map token.Pos // position of "map" keyword
Key Expr Key Expr
Value Expr Value Expr
} }
// A ChanType node represents a channel type. // A ChanType node represents a channel type.
ChanType struct { ChanType struct {
Begin token.Position // position of "chan" keyword or "<-" (whichever comes first) Begin token.Pos // position of "chan" keyword or "<-" (whichever comes first)
Dir ChanDir // channel direction Dir ChanDir // channel direction
Value Expr // value type Value Expr // value type
} }
) )
// Pos() implementations for expression/type nodes. // Pos() implementations for expression/type nodes.
// //
func (x *BadExpr) Pos() token.Position { return x.Begin } func (x *BadExpr) Pos() token.Pos { return x.Begin }
func (x *Ident) Pos() token.Position { return x.NamePos } func (x *Ident) Pos() token.Pos { return x.NamePos }
func (x *Ellipsis) Pos() token.Position { return x.Ellipsis } func (x *Ellipsis) Pos() token.Pos { return x.Ellipsis }
func (x *BasicLit) Pos() token.Position { return x.ValuePos } func (x *BasicLit) Pos() token.Pos { return x.ValuePos }
func (x *FuncLit) Pos() token.Position { return x.Type.Pos() } func (x *FuncLit) Pos() token.Pos { return x.Type.Pos() }
func (x *CompositeLit) Pos() token.Position { func (x *CompositeLit) Pos() token.Pos {
if x.Type != nil { if x.Type != nil {
return x.Type.Pos() return x.Type.Pos()
} }
return x.Lbrace return x.Lbrace
} }
func (x *ParenExpr) Pos() token.Position { return x.Lparen } func (x *ParenExpr) Pos() token.Pos { return x.Lparen }
func (x *SelectorExpr) Pos() token.Position { return x.X.Pos() } func (x *SelectorExpr) Pos() token.Pos { return x.X.Pos() }
func (x *IndexExpr) Pos() token.Position { return x.X.Pos() } func (x *IndexExpr) Pos() token.Pos { return x.X.Pos() }
func (x *SliceExpr) Pos() token.Position { return x.X.Pos() } func (x *SliceExpr) Pos() token.Pos { return x.X.Pos() }
func (x *TypeAssertExpr) Pos() token.Position { return x.X.Pos() } func (x *TypeAssertExpr) Pos() token.Pos { return x.X.Pos() }
func (x *CallExpr) Pos() token.Position { return x.Fun.Pos() } func (x *CallExpr) Pos() token.Pos { return x.Fun.Pos() }
func (x *StarExpr) Pos() token.Position { return x.Star } func (x *StarExpr) Pos() token.Pos { return x.Star }
func (x *UnaryExpr) Pos() token.Position { return x.OpPos } func (x *UnaryExpr) Pos() token.Pos { return x.OpPos }
func (x *BinaryExpr) Pos() token.Position { return x.X.Pos() } func (x *BinaryExpr) Pos() token.Pos { return x.X.Pos() }
func (x *KeyValueExpr) Pos() token.Position { return x.Key.Pos() } func (x *KeyValueExpr) Pos() token.Pos { return x.Key.Pos() }
func (x *ArrayType) Pos() token.Position { return x.Lbrack } func (x *ArrayType) Pos() token.Pos { return x.Lbrack }
func (x *StructType) Pos() token.Position { return x.Struct } func (x *StructType) Pos() token.Pos { return x.Struct }
func (x *FuncType) Pos() token.Position { return x.Func } func (x *FuncType) Pos() token.Pos { return x.Func }
func (x *InterfaceType) Pos() token.Position { return x.Interface } func (x *InterfaceType) Pos() token.Pos { return x.Interface }
func (x *MapType) Pos() token.Position { return x.Map } func (x *MapType) Pos() token.Pos { return x.Map }
func (x *ChanType) Pos() token.Position { return x.Begin } func (x *ChanType) Pos() token.Pos { return x.Begin }
// exprNode() ensures that only expression/type nodes can be // exprNode() ensures that only expression/type nodes can be
@ -382,7 +382,7 @@ func (x *ChanType) exprNode() {}
// ---------------------------------------------------------------------------- // ----------------------------------------------------------------------------
// Convenience functions for Idents // Convenience functions for Idents
var noPos token.Position var noPos token.Pos
// NewIdent creates a new Ident without position. // NewIdent creates a new Ident without position.
// Useful for ASTs generated by code other than the Go parser. // Useful for ASTs generated by code other than the Go parser.
@ -425,7 +425,7 @@ type (
// created. // created.
// //
BadStmt struct { BadStmt struct {
Begin token.Position // beginning position of bad statement Begin token.Pos // beginning position of bad statement
} }
// A DeclStmt node represents a declaration in a statement list. // A DeclStmt node represents a declaration in a statement list.
@ -438,7 +438,7 @@ type (
// of the immediately preceeding semicolon. // of the immediately preceeding semicolon.
// //
EmptyStmt struct { EmptyStmt struct {
Semicolon token.Position // position of preceeding ";" Semicolon token.Pos // position of preceeding ";"
} }
// A LabeledStmt node represents a labeled statement. // A LabeledStmt node represents a labeled statement.
@ -465,26 +465,26 @@ type (
// //
AssignStmt struct { AssignStmt struct {
Lhs []Expr Lhs []Expr
TokPos token.Position // position of Tok TokPos token.Pos // position of Tok
Tok token.Token // assignment token, DEFINE Tok token.Token // assignment token, DEFINE
Rhs []Expr Rhs []Expr
} }
// A GoStmt node represents a go statement. // A GoStmt node represents a go statement.
GoStmt struct { GoStmt struct {
Go token.Position // position of "go" keyword Go token.Pos // position of "go" keyword
Call *CallExpr Call *CallExpr
} }
// A DeferStmt node represents a defer statement. // A DeferStmt node represents a defer statement.
DeferStmt struct { DeferStmt struct {
Defer token.Position // position of "defer" keyword Defer token.Pos // position of "defer" keyword
Call *CallExpr Call *CallExpr
} }
// A ReturnStmt node represents a return statement. // A ReturnStmt node represents a return statement.
ReturnStmt struct { ReturnStmt struct {
Return token.Position // position of "return" keyword Return token.Pos // position of "return" keyword
Results []Expr Results []Expr
} }
@ -492,21 +492,21 @@ type (
// or fallthrough statement. // or fallthrough statement.
// //
BranchStmt struct { BranchStmt struct {
TokPos token.Position // position of Tok TokPos token.Pos // position of Tok
Tok token.Token // keyword token (BREAK, CONTINUE, GOTO, FALLTHROUGH) Tok token.Token // keyword token (BREAK, CONTINUE, GOTO, FALLTHROUGH)
Label *Ident Label *Ident
} }
// A BlockStmt node represents a braced statement list. // A BlockStmt node represents a braced statement list.
BlockStmt struct { BlockStmt struct {
Lbrace token.Position // position of "{" Lbrace token.Pos // position of "{"
List []Stmt List []Stmt
Rbrace token.Position // position of "}" Rbrace token.Pos // position of "}"
} }
// An IfStmt node represents an if statement. // An IfStmt node represents an if statement.
IfStmt struct { IfStmt struct {
If token.Position // position of "if" keyword If token.Pos // position of "if" keyword
Init Stmt Init Stmt
Cond Expr Cond Expr
Body *BlockStmt Body *BlockStmt
@ -515,15 +515,15 @@ type (
// A CaseClause represents a case of an expression switch statement. // A CaseClause represents a case of an expression switch statement.
CaseClause struct { CaseClause struct {
Case token.Position // position of "case" or "default" keyword Case token.Pos // position of "case" or "default" keyword
Values []Expr // nil means default case Values []Expr // nil means default case
Colon token.Position // position of ":" Colon token.Pos // position of ":"
Body []Stmt // statement list; or nil Body []Stmt // statement list; or nil
} }
// A SwitchStmt node represents an expression switch statement. // A SwitchStmt node represents an expression switch statement.
SwitchStmt struct { SwitchStmt struct {
Switch token.Position // position of "switch" keyword Switch token.Pos // position of "switch" keyword
Init Stmt Init Stmt
Tag Expr Tag Expr
Body *BlockStmt // CaseClauses only Body *BlockStmt // CaseClauses only
@ -531,15 +531,15 @@ type (
// A TypeCaseClause represents a case of a type switch statement. // A TypeCaseClause represents a case of a type switch statement.
TypeCaseClause struct { TypeCaseClause struct {
Case token.Position // position of "case" or "default" keyword Case token.Pos // position of "case" or "default" keyword
Types []Expr // nil means default case Types []Expr // nil means default case
Colon token.Position // position of ":" Colon token.Pos // position of ":"
Body []Stmt // statement list; or nil Body []Stmt // statement list; or nil
} }
// An TypeSwitchStmt node represents a type switch statement. // An TypeSwitchStmt node represents a type switch statement.
TypeSwitchStmt struct { TypeSwitchStmt struct {
Switch token.Position // position of "switch" keyword Switch token.Pos // position of "switch" keyword
Init Stmt Init Stmt
Assign Stmt // x := y.(type) Assign Stmt // x := y.(type)
Body *BlockStmt // TypeCaseClauses only Body *BlockStmt // TypeCaseClauses only
@ -547,22 +547,22 @@ type (
// A CommClause node represents a case of a select statement. // A CommClause node represents a case of a select statement.
CommClause struct { CommClause struct {
Case token.Position // position of "case" or "default" keyword Case token.Pos // position of "case" or "default" keyword
Tok token.Token // ASSIGN or DEFINE (valid only if Lhs != nil) Tok token.Token // ASSIGN or DEFINE (valid only if Lhs != nil)
Lhs, Rhs Expr // Rhs == nil means default case Lhs, Rhs Expr // Rhs == nil means default case
Colon token.Position // position of ":" Colon token.Pos // position of ":"
Body []Stmt // statement list; or nil Body []Stmt // statement list; or nil
} }
// An SelectStmt node represents a select statement. // An SelectStmt node represents a select statement.
SelectStmt struct { SelectStmt struct {
Select token.Position // position of "select" keyword Select token.Pos // position of "select" keyword
Body *BlockStmt // CommClauses only Body *BlockStmt // CommClauses only
} }
// A ForStmt represents a for statement. // A ForStmt represents a for statement.
ForStmt struct { ForStmt struct {
For token.Position // position of "for" keyword For token.Pos // position of "for" keyword
Init Stmt Init Stmt
Cond Expr Cond Expr
Post Stmt Post Stmt
@ -571,11 +571,11 @@ type (
// A RangeStmt represents a for statement with a range clause. // A RangeStmt represents a for statement with a range clause.
RangeStmt struct { RangeStmt struct {
For token.Position // position of "for" keyword For token.Pos // position of "for" keyword
Key, Value Expr // Value may be nil Key, Value Expr // Value may be nil
TokPos token.Position // position of Tok TokPos token.Pos // position of Tok
Tok token.Token // ASSIGN, DEFINE Tok token.Token // ASSIGN, DEFINE
X Expr // value to range over X Expr // value to range over
Body *BlockStmt Body *BlockStmt
} }
) )
@ -583,27 +583,27 @@ type (
// Pos() implementations for statement nodes. // Pos() implementations for statement nodes.
// //
func (s *BadStmt) Pos() token.Position { return s.Begin } func (s *BadStmt) Pos() token.Pos { return s.Begin }
func (s *DeclStmt) Pos() token.Position { return s.Decl.Pos() } func (s *DeclStmt) Pos() token.Pos { return s.Decl.Pos() }
func (s *EmptyStmt) Pos() token.Position { return s.Semicolon } func (s *EmptyStmt) Pos() token.Pos { return s.Semicolon }
func (s *LabeledStmt) Pos() token.Position { return s.Label.Pos() } func (s *LabeledStmt) Pos() token.Pos { return s.Label.Pos() }
func (s *ExprStmt) Pos() token.Position { return s.X.Pos() } func (s *ExprStmt) Pos() token.Pos { return s.X.Pos() }
func (s *IncDecStmt) Pos() token.Position { return s.X.Pos() } func (s *IncDecStmt) Pos() token.Pos { return s.X.Pos() }
func (s *AssignStmt) Pos() token.Position { return s.Lhs[0].Pos() } func (s *AssignStmt) Pos() token.Pos { return s.Lhs[0].Pos() }
func (s *GoStmt) Pos() token.Position { return s.Go } func (s *GoStmt) Pos() token.Pos { return s.Go }
func (s *DeferStmt) Pos() token.Position { return s.Defer } func (s *DeferStmt) Pos() token.Pos { return s.Defer }
func (s *ReturnStmt) Pos() token.Position { return s.Return } func (s *ReturnStmt) Pos() token.Pos { return s.Return }
func (s *BranchStmt) Pos() token.Position { return s.TokPos } func (s *BranchStmt) Pos() token.Pos { return s.TokPos }
func (s *BlockStmt) Pos() token.Position { return s.Lbrace } func (s *BlockStmt) Pos() token.Pos { return s.Lbrace }
func (s *IfStmt) Pos() token.Position { return s.If } func (s *IfStmt) Pos() token.Pos { return s.If }
func (s *CaseClause) Pos() token.Position { return s.Case } func (s *CaseClause) Pos() token.Pos { return s.Case }
func (s *SwitchStmt) Pos() token.Position { return s.Switch } func (s *SwitchStmt) Pos() token.Pos { return s.Switch }
func (s *TypeCaseClause) Pos() token.Position { return s.Case } func (s *TypeCaseClause) Pos() token.Pos { return s.Case }
func (s *TypeSwitchStmt) Pos() token.Position { return s.Switch } func (s *TypeSwitchStmt) Pos() token.Pos { return s.Switch }
func (s *CommClause) Pos() token.Position { return s.Case } func (s *CommClause) Pos() token.Pos { return s.Case }
func (s *SelectStmt) Pos() token.Position { return s.Select } func (s *SelectStmt) Pos() token.Pos { return s.Select }
func (s *ForStmt) Pos() token.Position { return s.For } func (s *ForStmt) Pos() token.Pos { return s.For }
func (s *RangeStmt) Pos() token.Position { return s.For } func (s *RangeStmt) Pos() token.Pos { return s.For }
// stmtNode() ensures that only statement nodes can be // stmtNode() ensures that only statement nodes can be
@ -676,14 +676,14 @@ type (
// Pos() implementations for spec nodes. // Pos() implementations for spec nodes.
// //
func (s *ImportSpec) Pos() token.Position { func (s *ImportSpec) Pos() token.Pos {
if s.Name != nil { if s.Name != nil {
return s.Name.Pos() return s.Name.Pos()
} }
return s.Path.Pos() return s.Path.Pos()
} }
func (s *ValueSpec) Pos() token.Position { return s.Names[0].Pos() } func (s *ValueSpec) Pos() token.Pos { return s.Names[0].Pos() }
func (s *TypeSpec) Pos() token.Position { return s.Name.Pos() } func (s *TypeSpec) Pos() token.Pos { return s.Name.Pos() }
// specNode() ensures that only spec nodes can be // specNode() ensures that only spec nodes can be
@ -702,7 +702,7 @@ type (
// created. // created.
// //
BadDecl struct { BadDecl struct {
Begin token.Position // beginning position of bad declaration Begin token.Pos // beginning position of bad declaration
} }
// A GenDecl node (generic declaration node) represents an import, // A GenDecl node (generic declaration node) represents an import,
@ -717,12 +717,12 @@ type (
// token.VAR *ValueSpec // token.VAR *ValueSpec
// //
GenDecl struct { GenDecl struct {
Doc *CommentGroup // associated documentation; or nil Doc *CommentGroup // associated documentation; or nil
TokPos token.Position // position of Tok TokPos token.Pos // position of Tok
Tok token.Token // IMPORT, CONST, TYPE, VAR Tok token.Token // IMPORT, CONST, TYPE, VAR
Lparen token.Position // position of '(', if any Lparen token.Pos // position of '(', if any
Specs []Spec Specs []Spec
Rparen token.Position // position of ')', if any Rparen token.Pos // position of ')', if any
} }
// A FuncDecl node represents a function declaration. // A FuncDecl node represents a function declaration.
@ -738,9 +738,9 @@ type (
// Pos implementations for declaration nodes. // Pos implementations for declaration nodes.
// //
func (d *BadDecl) Pos() token.Position { return d.Begin } func (d *BadDecl) Pos() token.Pos { return d.Begin }
func (d *GenDecl) Pos() token.Position { return d.TokPos } func (d *GenDecl) Pos() token.Pos { return d.TokPos }
func (d *FuncDecl) Pos() token.Position { return d.Type.Pos() } func (d *FuncDecl) Pos() token.Pos { return d.Type.Pos() }
// declNode() ensures that only declaration nodes can be // declNode() ensures that only declaration nodes can be
@ -762,14 +762,14 @@ func (d *FuncDecl) declNode() {}
// //
type File struct { type File struct {
Doc *CommentGroup // associated documentation; or nil Doc *CommentGroup // associated documentation; or nil
Package token.Position // position of "package" keyword Package token.Pos // position of "package" keyword
Name *Ident // package name Name *Ident // package name
Decls []Decl // top-level declarations Decls []Decl // top-level declarations
Comments []*CommentGroup // list of all comments in the source file Comments []*CommentGroup // list of all comments in the source file
} }
func (f *File) Pos() token.Position { return f.Package } func (f *File) Pos() token.Pos { return f.Package }
// A Package node represents a set of source files // A Package node represents a set of source files

33
src/pkg/go/ast/filter.go
View file

@ -307,27 +307,6 @@ const (
var separator = &Comment{noPos, []byte("//")} var separator = &Comment{noPos, []byte("//")}
// lineAfterComment computes the position of the beginning
// of the line immediately following a comment.
func lineAfterComment(c *Comment) token.Position {
pos := c.Pos()
line := pos.Line
text := c.Text
if text[1] == '*' {
/*-style comment - determine endline */
for _, ch := range text {
if ch == '\n' {
line++
}
}
}
pos.Offset += len(text) + 1 // +1 for newline
pos.Line = line + 1 // line after comment
pos.Column = 1 // beginning of line
return pos
}
// MergePackageFiles creates a file AST by merging the ASTs of the // MergePackageFiles creates a file AST by merging the ASTs of the
// files belonging to a package. The mode flags control merging behavior. // files belonging to a package. The mode flags control merging behavior.
// //
@ -351,7 +330,7 @@ func MergePackageFiles(pkg *Package, mode MergeMode) *File {
// a package comment; but it's better to collect extra comments // a package comment; but it's better to collect extra comments
// than drop them on the floor. // than drop them on the floor.
var doc *CommentGroup var doc *CommentGroup
var pos token.Position var pos token.Pos
if ndocs > 0 { if ndocs > 0 {
list := make([]*Comment, ndocs-1) // -1: no separator before first group list := make([]*Comment, ndocs-1) // -1: no separator before first group
i := 0 i := 0
@ -366,11 +345,11 @@ func MergePackageFiles(pkg *Package, mode MergeMode) *File {
list[i] = c list[i] = c
i++ i++
} }
end := lineAfterComment(f.Doc.List[len(f.Doc.List)-1]) if f.Package > pos {
if end.Offset > pos.Offset { // Keep the maximum package clause position as
// Keep the maximum end position as // position for the package clause of the merged
// position for the package clause. // files.
pos = end pos = f.Package
} }
} }
} }

5
src/pkg/go/doc/doc.go
View file

@ -66,7 +66,7 @@ func (doc *docReader) addDoc(comments *ast.CommentGroup) {
n2 := len(comments.List) n2 := len(comments.List)
list := make([]*ast.Comment, n1+1+n2) // + 1 for separator line list := make([]*ast.Comment, n1+1+n2) // + 1 for separator line
copy(list, doc.doc.List) copy(list, doc.doc.List)
list[n1] = &ast.Comment{token.Position{}, []byte("//")} // separator line list[n1] = &ast.Comment{token.NoPos, []byte("//")} // separator line
copy(list[n1+1:], comments.List) copy(list[n1+1:], comments.List)
doc.doc = &ast.CommentGroup{list} doc.doc = &ast.CommentGroup{list}
} }
@ -249,7 +249,6 @@ func (doc *docReader) addDecl(decl ast.Decl) {
doc.addValue(d) doc.addValue(d)
case token.TYPE: case token.TYPE:
// types are handled individually // types are handled individually
var noPos token.Position
for _, spec := range d.Specs { for _, spec := range d.Specs {
// make a (fake) GenDecl node for this TypeSpec // make a (fake) GenDecl node for this TypeSpec
// (we need to do this here - as opposed to just // (we need to do this here - as opposed to just
@ -262,7 +261,7 @@ func (doc *docReader) addDecl(decl ast.Decl) {
// makeTypeDocs below). Simpler data structures, but // makeTypeDocs below). Simpler data structures, but
// would lose GenDecl documentation if the TypeSpec // would lose GenDecl documentation if the TypeSpec
// has documentation as well. // has documentation as well.
doc.addType(&ast.GenDecl{d.Doc, d.Pos(), token.TYPE, noPos, []ast.Spec{spec}, noPos}) doc.addType(&ast.GenDecl{d.Doc, d.Pos(), token.TYPE, token.NoPos, []ast.Spec{spec}, token.NoPos})
// A new GenDecl node is created, no need to nil out d.Doc. // A new GenDecl node is created, no need to nil out d.Doc.
} }
} }

41
src/pkg/go/parser/interface.go
View file

@ -57,18 +57,18 @@ func (p *parser) parseEOF() os.Error {
// ParseExpr parses a Go expression and returns the corresponding // ParseExpr parses a Go expression and returns the corresponding
// AST node. The filename and src arguments have the same interpretation // AST node. The fset, filename, and src arguments have the same interpretation
// as for ParseFile. If there is an error, the result expression // as for ParseFile. If there is an error, the result expression
// may be nil or contain a partial AST. // may be nil or contain a partial AST.
// //
func ParseExpr(filename string, src interface{}) (ast.Expr, os.Error) { func ParseExpr(fset *token.FileSet, filename string, src interface{}) (ast.Expr, os.Error) {
data, err := readSource(filename, src) data, err := readSource(filename, src)
if err != nil { if err != nil {
return nil, err return nil, err
} }
var p parser var p parser
p.init(filename, data, 0) p.init(fset, filename, data, 0)
x := p.parseExpr() x := p.parseExpr()
if p.tok == token.SEMICOLON { if p.tok == token.SEMICOLON {
p.next() // consume automatically inserted semicolon, if any p.next() // consume automatically inserted semicolon, if any
@ -78,39 +78,41 @@ func ParseExpr(filename string, src interface{}) (ast.Expr, os.Error) {
// ParseStmtList parses a list of Go statements and returns the list // ParseStmtList parses a list of Go statements and returns the list
// of corresponding AST nodes. The filename and src arguments have the same // of corresponding AST nodes. The fset, filename, and src arguments have the same
// interpretation as for ParseFile. If there is an error, the node // interpretation as for ParseFile. If there is an error, the node
// list may be nil or contain partial ASTs. // list may be nil or contain partial ASTs.
// //
func ParseStmtList(filename string, src interface{}) ([]ast.Stmt, os.Error) { func ParseStmtList(fset *token.FileSet, filename string, src interface{}) ([]ast.Stmt, os.Error) {
data, err := readSource(filename, src) data, err := readSource(filename, src)
if err != nil { if err != nil {
return nil, err return nil, err
} }
var p parser var p parser
p.init(filename, data, 0) p.init(fset, filename, data, 0)
return p.parseStmtList(), p.parseEOF() return p.parseStmtList(), p.parseEOF()
} }
// ParseDeclList parses a list of Go declarations and returns the list // ParseDeclList parses a list of Go declarations and returns the list
// of corresponding AST nodes. The filename and src arguments have the same // of corresponding AST nodes. The fset, filename, and src arguments have the same
// interpretation as for ParseFile. If there is an error, the node // interpretation as for ParseFile. If there is an error, the node
// list may be nil or contain partial ASTs. // list may be nil or contain partial ASTs.
// //
func ParseDeclList(filename string, src interface{}) ([]ast.Decl, os.Error) { func ParseDeclList(fset *token.FileSet, filename string, src interface{}) ([]ast.Decl, os.Error) {
data, err := readSource(filename, src) data, err := readSource(filename, src)
if err != nil { if err != nil {
return nil, err return nil, err
} }
var p parser var p parser
p.init(filename, data, 0) p.init(fset, filename, data, 0)
return p.parseDeclList(), p.parseEOF() return p.parseDeclList(), p.parseEOF()
} }
// TODO(gri) Change ParseFile to Parse and improve documentation (issue 1311).
// ParseFile parses a Go source file and returns a File node. // ParseFile parses a Go source file and returns a File node.
// //
// If src != nil, ParseFile parses the file source from src. src may // If src != nil, ParseFile parses the file source from src. src may
@ -121,7 +123,8 @@ func ParseDeclList(filename string, src interface{}) ([]ast.Decl, os.Error) {
// If src == nil, ParseFile parses the file specified by filename. // If src == nil, ParseFile parses the file specified by filename.
// //
// The mode parameter controls the amount of source text parsed and other // The mode parameter controls the amount of source text parsed and other
// optional parser functionality. // optional parser functionality. Position information is recorded in the
// file set fset.
// //
// If the source couldn't be read, the returned AST is nil and the error // If the source couldn't be read, the returned AST is nil and the error
// indicates the specific failure. If the source was read but syntax // indicates the specific failure. If the source was read but syntax
@ -129,30 +132,31 @@ func ParseDeclList(filename string, src interface{}) ([]ast.Decl, os.Error) {
// representing the fragments of erroneous source code). Multiple errors // representing the fragments of erroneous source code). Multiple errors
// are returned via a scanner.ErrorList which is sorted by file position. // are returned via a scanner.ErrorList which is sorted by file position.
// //
func ParseFile(filename string, src interface{}, mode uint) (*ast.File, os.Error) { func ParseFile(fset *token.FileSet, filename string, src interface{}, mode uint) (*ast.File, os.Error) {
data, err := readSource(filename, src) data, err := readSource(filename, src)
if err != nil { if err != nil {
return nil, err return nil, err
} }
var p parser var p parser
p.init(filename, data, mode) p.init(fset, filename, data, mode)
return p.parseFile(), p.GetError(scanner.NoMultiples) // parseFile() reads to EOF return p.parseFile(), p.GetError(scanner.NoMultiples) // parseFile() reads to EOF
} }
// ParseFiles calls ParseFile for each file in the filenames list and returns // ParseFiles calls ParseFile for each file in the filenames list and returns
// a map of package name -> package AST with all the packages found. The mode // a map of package name -> package AST with all the packages found. The mode
// bits are passed to ParseFile unchanged. // bits are passed to ParseFile unchanged. Position information is recorded
// in the file set fset.
// //
// Files with parse errors are ignored. In this case the map of packages may // Files with parse errors are ignored. In this case the map of packages may
// be incomplete (missing packages and/or incomplete packages) and the first // be incomplete (missing packages and/or incomplete packages) and the first
// error encountered is returned. // error encountered is returned.
// //
func ParseFiles(filenames []string, mode uint) (pkgs map[string]*ast.Package, first os.Error) { func ParseFiles(fset *token.FileSet, filenames []string, mode uint) (pkgs map[string]*ast.Package, first os.Error) {
pkgs = make(map[string]*ast.Package) pkgs = make(map[string]*ast.Package)
for _, filename := range filenames { for _, filename := range filenames {
if src, err := ParseFile(filename, nil, mode); err == nil { if src, err := ParseFile(fset, filename, nil, mode); err == nil {
name := src.Name.Name name := src.Name.Name
pkg, found := pkgs[name] pkg, found := pkgs[name]
if !found { if !found {
@ -171,13 +175,14 @@ func ParseFiles(filenames []string, mode uint) (pkgs map[string]*ast.Package, fi
// ParseDir calls ParseFile for the files in the directory specified by path and // ParseDir calls ParseFile for the files in the directory specified by path and
// returns a map of package name -> package AST with all the packages found. If // returns a map of package name -> package AST with all the packages found. If
// filter != nil, only the files with os.FileInfo entries passing through the filter // filter != nil, only the files with os.FileInfo entries passing through the filter
// are considered. The mode bits are passed to ParseFile unchanged. // are considered. The mode bits are passed to ParseFile unchanged. Position
// information is recorded in the file set fset.
// //
// If the directory couldn't be read, a nil map and the respective error are // If the directory couldn't be read, a nil map and the respective error are
// returned. If a parse error occured, a non-nil but incomplete map and the // returned. If a parse error occured, a non-nil but incomplete map and the
// error are returned. // error are returned.
// //
func ParseDir(path string, filter func(*os.FileInfo) bool, mode uint) (map[string]*ast.Package, os.Error) { func ParseDir(fset *token.FileSet, path string, filter func(*os.FileInfo) bool, mode uint) (map[string]*ast.Package, os.Error) {
fd, err := os.Open(path, os.O_RDONLY, 0) fd, err := os.Open(path, os.O_RDONLY, 0)
if err != nil { if err != nil {
return nil, err return nil, err
@ -200,5 +205,5 @@ func ParseDir(path string, filter func(*os.FileInfo) bool, mode uint) (map[strin
} }
filenames = filenames[0:n] filenames = filenames[0:n]
return ParseFiles(filenames, mode) return ParseFiles(fset, filenames, mode)
} }

67
src/pkg/go/parser/parser.go
View file

@ -35,6 +35,7 @@ const (
// The parser structure holds the parser's internal state. // The parser structure holds the parser's internal state.
type parser struct { type parser struct {
file *token.File
scanner.ErrorVector scanner.ErrorVector
scanner scanner.Scanner scanner scanner.Scanner
@ -49,9 +50,9 @@ type parser struct {
lineComment *ast.CommentGroup // the last line comment lineComment *ast.CommentGroup // the last line comment
// Next token // Next token
pos token.Position // token position pos token.Pos // token position
tok token.Token // one token look-ahead tok token.Token // one token look-ahead
lit []byte // token literal lit []byte // token literal
// Non-syntactic parser control // Non-syntactic parser control
exprLev int // < 0: in control clause, >= 0: in expression exprLev int // < 0: in control clause, >= 0: in expression
@ -68,8 +69,8 @@ func scannerMode(mode uint) uint {
} }
func (p *parser) init(filename string, src []byte, mode uint) { func (p *parser) init(fset *token.FileSet, filename string, src []byte, mode uint) {
p.scanner.Init(filename, src, p, scannerMode(mode)) p.file = p.scanner.Init(fset, filename, src, p, scannerMode(mode))
p.mode = mode p.mode = mode
p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently) p.trace = mode&Trace != 0 // for convenience (p.trace is used frequently)
p.next() p.next()
@ -83,7 +84,8 @@ func (p *parser) printTrace(a ...interface{}) {
const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " + const dots = ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " +
". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . " ". . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . "
const n = uint(len(dots)) const n = uint(len(dots))
fmt.Printf("%5d:%3d: ", p.pos.Line, p.pos.Column) pos := p.file.Position(p.pos)
fmt.Printf("%5d:%3d: ", pos.Line, pos.Column)
i := 2 * p.indent i := 2 * p.indent
for ; i > n; i -= n { for ; i > n; i -= n {
fmt.Print(dots) fmt.Print(dots)
@ -111,9 +113,9 @@ func un(p *parser) {
func (p *parser) next0() { func (p *parser) next0() {
// Because of one-token look-ahead, print the previous token // Because of one-token look-ahead, print the previous token
// when tracing as it provides a more readable output. The // when tracing as it provides a more readable output. The
// very first token (p.pos.Line == 0) is not initialized (it // very first token (!p.pos.IsValid()) is not initialized
// is token.ILLEGAL), so don't print it . // (it is token.ILLEGAL), so don't print it .
if p.trace && p.pos.Line > 0 { if p.trace && p.pos.IsValid() {
s := p.tok.String() s := p.tok.String()
switch { switch {
case p.tok.IsLiteral(): case p.tok.IsLiteral():
@ -132,7 +134,7 @@ func (p *parser) next0() {
func (p *parser) consumeComment() (comment *ast.Comment, endline int) { func (p *parser) consumeComment() (comment *ast.Comment, endline int) {
// /*-style comments may end on a different line than where they start. // /*-style comments may end on a different line than where they start.
// Scan the comment for '\n' chars and adjust endline accordingly. // Scan the comment for '\n' chars and adjust endline accordingly.
endline = p.pos.Line endline = p.file.Line(p.pos)
if p.lit[1] == '*' { if p.lit[1] == '*' {
for _, b := range p.lit { for _, b := range p.lit {
if b == '\n' { if b == '\n' {
@ -155,8 +157,8 @@ func (p *parser) consumeComment() (comment *ast.Comment, endline int) {
// //
func (p *parser) consumeCommentGroup() (comments *ast.CommentGroup, endline int) { func (p *parser) consumeCommentGroup() (comments *ast.CommentGroup, endline int) {
var list []*ast.Comment var list []*ast.Comment
endline = p.pos.Line endline = p.file.Line(p.pos)
for p.tok == token.COMMENT && endline+1 >= p.pos.Line { for p.tok == token.COMMENT && endline+1 >= p.file.Line(p.pos) {
var comment *ast.Comment var comment *ast.Comment
comment, endline = p.consumeComment() comment, endline = p.consumeComment()
list = append(list, comment) list = append(list, comment)
@ -188,18 +190,18 @@ func (p *parser) consumeCommentGroup() (comments *ast.CommentGroup, endline int)
func (p *parser) next() { func (p *parser) next() {
p.leadComment = nil p.leadComment = nil
p.lineComment = nil p.lineComment = nil
line := p.pos.Line // current line line := p.file.Line(p.pos) // current line
p.next0() p.next0()
if p.tok == token.COMMENT { if p.tok == token.COMMENT {
var comment *ast.CommentGroup var comment *ast.CommentGroup
var endline int var endline int
if p.pos.Line == line { if p.file.Line(p.pos) == line {
// The comment is on same line as previous token; it // The comment is on same line as previous token; it
// cannot be a lead comment but may be a line comment. // cannot be a lead comment but may be a line comment.
comment, endline = p.consumeCommentGroup() comment, endline = p.consumeCommentGroup()
if p.pos.Line != endline { if p.file.Line(p.pos) != endline {
// The next token is on a different line, thus // The next token is on a different line, thus
// the last comment group is a line comment. // the last comment group is a line comment.
p.lineComment = comment p.lineComment = comment
@ -212,7 +214,7 @@ func (p *parser) next() {
comment, endline = p.consumeCommentGroup() comment, endline = p.consumeCommentGroup()
} }
if endline+1 == p.pos.Line { if endline+1 == p.file.Line(p.pos) {
// The next token is following on the line immediately after the // The next token is following on the line immediately after the
// comment group, thus the last comment group is a lead comment. // comment group, thus the last comment group is a lead comment.
p.leadComment = comment p.leadComment = comment
@ -221,9 +223,14 @@ func (p *parser) next() {
} }
func (p *parser) errorExpected(pos token.Position, msg string) { func (p *parser) error(pos token.Pos, msg string) {
p.Error(p.file.Position(pos), msg)
}
func (p *parser) errorExpected(pos token.Pos, msg string) {
msg = "expected " + msg msg = "expected " + msg
if pos.Offset == p.pos.Offset { if pos == p.pos {
// the error happened at the current position; // the error happened at the current position;
// make the error message more specific // make the error message more specific
if p.tok == token.SEMICOLON && p.lit[0] == '\n' { if p.tok == token.SEMICOLON && p.lit[0] == '\n' {
@ -235,11 +242,11 @@ func (p *parser) errorExpected(pos token.Position, msg string) {
} }
} }
} }
p.Error(pos, msg) p.error(pos, msg)
} }
func (p *parser) expect(tok token.Token) token.Position { func (p *parser) expect(tok token.Token) token.Pos {
pos := p.pos pos := p.pos
if p.tok != tok { if p.tok != tok {
p.errorExpected(pos, "'"+tok.String()+"'") p.errorExpected(pos, "'"+tok.String()+"'")
@ -461,11 +468,11 @@ func (p *parser) tryVarType(isParam bool) ast.Expr {
p.next() p.next()
typ := p.tryType() // don't use parseType so we can provide better error message typ := p.tryType() // don't use parseType so we can provide better error message
if typ == nil { if typ == nil {
p.Error(pos, "'...' parameter is missing type") p.error(pos, "'...' parameter is missing type")
typ = &ast.BadExpr{pos} typ = &ast.BadExpr{pos}
} }
if p.tok != token.RPAREN { if p.tok != token.RPAREN {
p.Error(pos, "can use '...' with last parameter type only") p.error(pos, "can use '...' with last parameter type only")
} }
return &ast.Ellipsis{pos, typ} return &ast.Ellipsis{pos, typ}
} }
@ -618,7 +625,7 @@ func (p *parser) parseMethodSpec() *ast.Field {
// method // method
idents = []*ast.Ident{ident} idents = []*ast.Ident{ident}
params, results := p.parseSignature() params, results := p.parseSignature()
typ = &ast.FuncType{noPos, params, results} typ = &ast.FuncType{token.NoPos, params, results}
} else { } else {
// embedded interface // embedded interface
typ = x typ = x
@ -888,7 +895,7 @@ func (p *parser) parseCallOrConversion(fun ast.Expr) *ast.CallExpr {
lparen := p.expect(token.LPAREN) lparen := p.expect(token.LPAREN)
p.exprLev++ p.exprLev++
var list []ast.Expr var list []ast.Expr
var ellipsis token.Position var ellipsis token.Pos
for p.tok != token.RPAREN && p.tok != token.EOF && !ellipsis.IsValid() { for p.tok != token.RPAREN && p.tok != token.EOF && !ellipsis.IsValid() {
list = append(list, p.parseExpr()) list = append(list, p.parseExpr())
if p.tok == token.ELLIPSIS { if p.tok == token.ELLIPSIS {
@ -1063,7 +1070,7 @@ func (p *parser) checkExprOrType(x ast.Expr) ast.Expr {
} }
case *ast.ArrayType: case *ast.ArrayType:
if len, isEllipsis := t.Len.(*ast.Ellipsis); isEllipsis { if len, isEllipsis := t.Len.(*ast.Ellipsis); isEllipsis {
p.Error(len.Pos(), "expected array length, found '...'") p.error(len.Pos(), "expected array length, found '...'")
x = &ast.BadExpr{x.Pos()} x = &ast.BadExpr{x.Pos()}
} }
} }
@ -1189,7 +1196,7 @@ func (p *parser) parseSimpleStmt(labelOk bool) ast.Stmt {
return &ast.LabeledStmt{label, p.parseStmt()} return &ast.LabeledStmt{label, p.parseStmt()}
} }
} }
p.Error(x[0].Pos(), "illegal label declaration") p.error(x[0].Pos(), "illegal label declaration")
return &ast.BadStmt{x[0].Pos()} return &ast.BadStmt{x[0].Pos()}
case case
@ -1205,7 +1212,7 @@ func (p *parser) parseSimpleStmt(labelOk bool) ast.Stmt {
} }
if len(x) > 1 { if len(x) > 1 {
p.Error(x[0].Pos(), "only one expression allowed") p.error(x[0].Pos(), "only one expression allowed")
// continue with first expression // continue with first expression
} }
@ -1303,7 +1310,7 @@ func (p *parser) makeExpr(s ast.Stmt) ast.Expr {
if es, isExpr := s.(*ast.ExprStmt); isExpr { if es, isExpr := s.(*ast.ExprStmt); isExpr {
return p.checkExpr(es.X) return p.checkExpr(es.X)
} }
p.Error(s.Pos(), "expected condition, found simple statement") p.error(s.Pos(), "expected condition, found simple statement")
return &ast.BadExpr{s.Pos()} return &ast.BadExpr{s.Pos()}
} }
@ -1718,7 +1725,7 @@ func (p *parser) parseGenDecl(keyword token.Token, f parseSpecFunction) *ast.Gen
doc := p.leadComment doc := p.leadComment
pos := p.expect(keyword) pos := p.expect(keyword)
var lparen, rparen token.Position var lparen, rparen token.Pos
var list []ast.Spec var list []ast.Spec
if p.tok == token.LPAREN { if p.tok == token.LPAREN {
lparen = p.pos lparen = p.pos
@ -1747,7 +1754,7 @@ func (p *parser) parseReceiver() *ast.FieldList {
// must have exactly one receiver // must have exactly one receiver
if par.NumFields() != 1 { if par.NumFields() != 1 {
p.errorExpected(pos, "exactly one receiver") p.errorExpected(pos, "exactly one receiver")
par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{noPos}}} par.List = []*ast.Field{&ast.Field{Type: &ast.BadExpr{token.NoPos}}}
return par return par
} }

11
src/pkg/go/parser/parser_test.go
View file

@ -5,11 +5,14 @@
package parser package parser
import ( import (
"go/token"
"os" "os"
"testing" "testing"
) )
var fset = token.NewFileSet()
var illegalInputs = []interface{}{ var illegalInputs = []interface{}{
nil, nil,
3.14, 3.14,
@ -20,7 +23,7 @@ var illegalInputs = []interface{}{
func TestParseIllegalInputs(t *testing.T) { func TestParseIllegalInputs(t *testing.T) {
for _, src := range illegalInputs { for _, src := range illegalInputs {
_, err := ParseFile("", src, 0) _, err := ParseFile(fset, "", src, 0)
if err == nil { if err == nil {
t.Errorf("ParseFile(%v) should have failed", src) t.Errorf("ParseFile(%v) should have failed", src)
} }
@ -48,7 +51,7 @@ var validPrograms = []interface{}{
func TestParseValidPrograms(t *testing.T) { func TestParseValidPrograms(t *testing.T) {
for _, src := range validPrograms { for _, src := range validPrograms {
_, err := ParseFile("", src, 0) _, err := ParseFile(fset, "", src, 0)
if err != nil { if err != nil {
t.Errorf("ParseFile(%q): %v", src, err) t.Errorf("ParseFile(%q): %v", src, err)
} }
@ -64,7 +67,7 @@ var validFiles = []string{
func TestParse3(t *testing.T) { func TestParse3(t *testing.T) {
for _, filename := range validFiles { for _, filename := range validFiles {
_, err := ParseFile(filename, nil, 0) _, err := ParseFile(fset, filename, nil, 0)
if err != nil { if err != nil {
t.Errorf("ParseFile(%s): %v", filename, err) t.Errorf("ParseFile(%s): %v", filename, err)
} }
@ -89,7 +92,7 @@ func dirFilter(f *os.FileInfo) bool { return nameFilter(f.Name) }
func TestParse4(t *testing.T) { func TestParse4(t *testing.T) {
path := "." path := "."
pkgs, err := ParseDir(path, dirFilter, 0) pkgs, err := ParseDir(fset, path, dirFilter, 0)
if err != nil { if err != nil {
t.Fatalf("ParseDir(%s): %v", path, err) t.Fatalf("ParseDir(%s): %v", path, err)
} }

67
src/pkg/go/printer/nodes.go
View file

@ -72,7 +72,7 @@ func (p *printer) setComment(g *ast.CommentGroup) {
// for some reason there are pending comments; this // for some reason there are pending comments; this
// should never happen - handle gracefully and flush // should never happen - handle gracefully and flush
// all comments up to g, ignore anything after that // all comments up to g, ignore anything after that
p.flush(g.List[0].Pos(), token.ILLEGAL) p.flush(p.fset.Position(g.List[0].Pos()), token.ILLEGAL)
} }
p.comments[0] = g p.comments[0] = g
p.cindex = 0 p.cindex = 0
@ -104,7 +104,7 @@ func (p *printer) identList(list []*ast.Ident, indent bool, multiLine *bool) {
if !indent { if !indent {
mode |= noIndent mode |= noIndent
} }
p.exprList(noPos, xlist, 1, mode, multiLine, noPos) p.exprList(token.NoPos, xlist, 1, mode, multiLine, token.NoPos)
} }
@ -127,7 +127,7 @@ func (p *printer) keySize(pair *ast.KeyValueExpr) int {
// TODO(gri) Consider rewriting this to be independent of []ast.Expr // TODO(gri) Consider rewriting this to be independent of []ast.Expr
// so that we can use the algorithm for any kind of list // so that we can use the algorithm for any kind of list
// (e.g., pass list via a channel over which to range). // (e.g., pass list via a channel over which to range).
func (p *printer) exprList(prev token.Position, list []ast.Expr, depth int, mode exprListMode, multiLine *bool, next token.Position) { func (p *printer) exprList(prev0 token.Pos, list []ast.Expr, depth int, mode exprListMode, multiLine *bool, next0 token.Pos) {
if len(list) == 0 { if len(list) == 0 {
return return
} }
@ -136,13 +136,15 @@ func (p *printer) exprList(prev token.Position, list []ast.Expr, depth int, mode
p.print(blank) p.print(blank)
} }
line := list[0].Pos().Line prev := p.fset.Position(prev0)
next := p.fset.Position(next0)
line := p.fset.Position(list[0].Pos()).Line
endLine := next.Line endLine := next.Line
if endLine == 0 { if endLine == 0 {
// TODO(gri): endLine may be incorrect as it is really the beginning // TODO(gri): endLine may be incorrect as it is really the beginning
// of the last list entry. There may be only one, very long // of the last list entry. There may be only one, very long
// entry in which case line == endLine. // entry in which case line == endLine.
endLine = list[len(list)-1].Pos().Line endLine = p.fset.Position(list[len(list)-1].Pos()).Line
} }
if prev.IsValid() && prev.Line == line && line == endLine { if prev.IsValid() && prev.Line == line && line == endLine {
@ -199,7 +201,7 @@ func (p *printer) exprList(prev token.Position, list []ast.Expr, depth int, mode
// print all list elements // print all list elements
for i, x := range list { for i, x := range list {
prevLine := line prevLine := line
line = x.Pos().Line line = p.fset.Position(x.Pos()).Line
// determine if the next linebreak, if any, needs to use formfeed: // determine if the next linebreak, if any, needs to use formfeed:
// in general, use the entire node size to make the decision; for // in general, use the entire node size to make the decision; for
@ -303,9 +305,9 @@ func (p *printer) parameters(fields *ast.FieldList, multiLine *bool) {
if i > 0 { if i > 0 {
p.print(token.COMMA) p.print(token.COMMA)
if len(par.Names) > 0 { if len(par.Names) > 0 {
line = par.Names[0].Pos().Line line = p.fset.Position(par.Names[0].Pos()).Line
} else { } else {
line = par.Type.Pos().Line line = p.fset.Position(par.Type.Pos()).Line
} }
if 0 < prevLine && prevLine < line && p.linebreak(line, 0, ignore, true) { if 0 < prevLine && prevLine < line && p.linebreak(line, 0, ignore, true) {
*multiLine = true *multiLine = true
@ -318,7 +320,7 @@ func (p *printer) parameters(fields *ast.FieldList, multiLine *bool) {
p.print(blank) p.print(blank)
} }
p.expr(par.Type, multiLine) p.expr(par.Type, multiLine)
prevLine = par.Type.Pos().Line prevLine = p.fset.Position(par.Type.Pos()).Line
} }
} }
p.print(fields.Closing, token.RPAREN) p.print(fields.Closing, token.RPAREN)
@ -375,7 +377,7 @@ func (p *printer) isOneLineFieldList(list []*ast.Field) bool {
func (p *printer) setLineComment(text string) { func (p *printer) setLineComment(text string) {
p.setComment(&ast.CommentGroup{[]*ast.Comment{&ast.Comment{noPos, []byte(text)}}}) p.setComment(&ast.CommentGroup{[]*ast.Comment{&ast.Comment{token.NoPos, []byte(text)}}})
} }
@ -389,7 +391,7 @@ func (p *printer) fieldList(fields *ast.FieldList, isIncomplete bool, ctxt exprC
list := fields.List list := fields.List
rbrace := fields.Closing rbrace := fields.Closing
if !isIncomplete && !p.commentBefore(rbrace) { if !isIncomplete && !p.commentBefore(p.fset.Position(rbrace)) {
// possibly a one-line struct/interface // possibly a one-line struct/interface
if len(list) == 0 { if len(list) == 0 {
// no blank between keyword and {} in this case // no blank between keyword and {} in this case
@ -427,7 +429,7 @@ func (p *printer) fieldList(fields *ast.FieldList, isIncomplete bool, ctxt exprC
var ml bool var ml bool
for i, f := range list { for i, f := range list {
if i > 0 { if i > 0 {
p.linebreak(f.Pos().Line, 1, ignore, ml) p.linebreak(p.fset.Position(f.Pos()).Line, 1, ignore, ml)
} }
ml = false ml = false
extraTabs := 0 extraTabs := 0
@ -462,7 +464,7 @@ func (p *printer) fieldList(fields *ast.FieldList, isIncomplete bool, ctxt exprC
if len(list) > 0 { if len(list) > 0 {
p.print(formfeed) p.print(formfeed)
} }
p.flush(rbrace, token.RBRACE) // make sure we don't loose the last line comment p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't loose the last line comment
p.setLineComment("// contains unexported fields") p.setLineComment("// contains unexported fields")
} }
@ -471,7 +473,7 @@ func (p *printer) fieldList(fields *ast.FieldList, isIncomplete bool, ctxt exprC
var ml bool var ml bool
for i, f := range list { for i, f := range list {
if i > 0 { if i > 0 {
p.linebreak(f.Pos().Line, 1, ignore, ml) p.linebreak(p.fset.Position(f.Pos()).Line, 1, ignore, ml)
} }
ml = false ml = false
p.setComment(f.Doc) p.setComment(f.Doc)
@ -489,7 +491,7 @@ func (p *printer) fieldList(fields *ast.FieldList, isIncomplete bool, ctxt exprC
if len(list) > 0 { if len(list) > 0 {
p.print(formfeed) p.print(formfeed)
} }
p.flush(rbrace, token.RBRACE) // make sure we don't loose the last line comment p.flush(p.fset.Position(rbrace), token.RBRACE) // make sure we don't loose the last line comment
p.setLineComment("// contains unexported methods") p.setLineComment("// contains unexported methods")
} }
@ -660,7 +662,7 @@ func (p *printer) binaryExpr(x *ast.BinaryExpr, prec1, cutoff, depth int, multiL
p.print(blank) p.print(blank)
} }
xline := p.pos.Line // before the operator (it may be on the next line!) xline := p.pos.Line // before the operator (it may be on the next line!)
yline := x.Y.Pos().Line yline := p.fset.Position(x.Y.Pos()).Line
p.print(x.OpPos, x.Op) p.print(x.OpPos, x.Op)
if xline != yline && xline > 0 && yline > 0 { if xline != yline && xline > 0 && yline > 0 {
// at least one line break, but respect an extra empty line // at least one line break, but respect an extra empty line
@ -805,7 +807,7 @@ func (p *printer) expr1(expr ast.Expr, prec1, depth int, ctxt exprContext, multi
case *ast.FuncLit: case *ast.FuncLit:
p.expr(x.Type, multiLine) p.expr(x.Type, multiLine)
p.funcBody(x.Body, distance(x.Type.Pos(), p.pos), true, multiLine) p.funcBody(x.Body, p.distance(x.Type.Pos(), p.pos), true, multiLine)
case *ast.ParenExpr: case *ast.ParenExpr:
if _, hasParens := x.X.(*ast.ParenExpr); hasParens { if _, hasParens := x.X.(*ast.ParenExpr); hasParens {
@ -820,7 +822,7 @@ func (p *printer) expr1(expr ast.Expr, prec1, depth int, ctxt exprContext, multi
case *ast.SelectorExpr: case *ast.SelectorExpr:
parts := selectorExprList(expr) parts := selectorExprList(expr)
p.exprList(noPos, parts, depth, periodSep, multiLine, noPos) p.exprList(token.NoPos, parts, depth, periodSep, multiLine, token.NoPos)
case *ast.TypeAssertExpr: case *ast.TypeAssertExpr:
p.expr1(x.X, token.HighestPrec, depth, 0, multiLine) p.expr1(x.X, token.HighestPrec, depth, 0, multiLine)
@ -957,7 +959,7 @@ func (p *printer) stmtList(list []ast.Stmt, _indent int, nextIsRBrace bool) {
for i, s := range list { for i, s := range list {
// _indent == 0 only for lists of switch/select case clauses; // _indent == 0 only for lists of switch/select case clauses;
// in those cases each clause is a new section // in those cases each clause is a new section
p.linebreak(s.Pos().Line, 1, ignore, i == 0 || _indent == 0 || multiLine) p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, i == 0 || _indent == 0 || multiLine)
multiLine = false multiLine = false
p.stmt(s, nextIsRBrace && i == len(list)-1, &multiLine) p.stmt(s, nextIsRBrace && i == len(list)-1, &multiLine)
} }
@ -971,7 +973,7 @@ func (p *printer) stmtList(list []ast.Stmt, _indent int, nextIsRBrace bool) {
func (p *printer) block(s *ast.BlockStmt, indent int) { func (p *printer) block(s *ast.BlockStmt, indent int) {
p.print(s.Pos(), token.LBRACE) p.print(s.Pos(), token.LBRACE)
p.stmtList(s.List, indent, true) p.stmtList(s.List, indent, true)
p.linebreak(s.Rbrace.Line, 1, ignore, true) p.linebreak(p.fset.Position(s.Rbrace).Line, 1, ignore, true)
p.print(s.Rbrace, token.RBRACE) p.print(s.Rbrace, token.RBRACE)
} }
@ -1076,7 +1078,7 @@ func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool, multiLine *bool) {
break break
} }
} else { } else {
p.linebreak(s.Stmt.Pos().Line, 1, ignore, true) p.linebreak(p.fset.Position(s.Stmt.Pos()).Line, 1, ignore, true)
} }
p.stmt(s.Stmt, nextIsRBrace, multiLine) p.stmt(s.Stmt, nextIsRBrace, multiLine)
@ -1096,7 +1098,7 @@ func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool, multiLine *bool) {
} }
p.exprList(s.Pos(), s.Lhs, depth, commaSep, multiLine, s.TokPos) p.exprList(s.Pos(), s.Lhs, depth, commaSep, multiLine, s.TokPos)
p.print(blank, s.TokPos, s.Tok) p.print(blank, s.TokPos, s.Tok)
p.exprList(s.TokPos, s.Rhs, depth, blankStart|commaSep, multiLine, noPos) p.exprList(s.TokPos, s.Rhs, depth, blankStart|commaSep, multiLine, token.NoPos)
case *ast.GoStmt: case *ast.GoStmt:
p.print(token.GO, blank) p.print(token.GO, blank)
@ -1109,7 +1111,7 @@ func (p *printer) stmt(stmt ast.Stmt, nextIsRBrace bool, multiLine *bool) {
case *ast.ReturnStmt: case *ast.ReturnStmt:
p.print(token.RETURN) p.print(token.RETURN)
if s.Results != nil { if s.Results != nil {
p.exprList(s.Pos(), s.Results, 1, blankStart|commaSep, multiLine, noPos) p.exprList(s.Pos(), s.Results, 1, blankStart|commaSep, multiLine, token.NoPos)
} }
case *ast.BranchStmt: case *ast.BranchStmt:
@ -1254,7 +1256,7 @@ func (p *printer) spec(spec ast.Spec, n int, doIndent bool, multiLine *bool) {
} }
if s.Values != nil { if s.Values != nil {
p.print(blank, token.ASSIGN) p.print(blank, token.ASSIGN)
p.exprList(noPos, s.Values, 1, blankStart|commaSep, multiLine, noPos) p.exprList(token.NoPos, s.Values, 1, blankStart|commaSep, multiLine, token.NoPos)
} }
p.setComment(s.Comment) p.setComment(s.Comment)
@ -1267,7 +1269,7 @@ func (p *printer) spec(spec ast.Spec, n int, doIndent bool, multiLine *bool) {
} }
if s.Values != nil { if s.Values != nil {
p.print(vtab, token.ASSIGN) p.print(vtab, token.ASSIGN)
p.exprList(noPos, s.Values, 1, blankStart|commaSep, multiLine, noPos) p.exprList(token.NoPos, s.Values, 1, blankStart|commaSep, multiLine, token.NoPos)
extraTabs-- extraTabs--
} }
if s.Comment != nil { if s.Comment != nil {
@ -1308,7 +1310,7 @@ func (p *printer) genDecl(d *ast.GenDecl, multiLine *bool) {
var ml bool var ml bool
for i, s := range d.Specs { for i, s := range d.Specs {
if i > 0 { if i > 0 {
p.linebreak(s.Pos().Line, 1, ignore, ml) p.linebreak(p.fset.Position(s.Pos()).Line, 1, ignore, ml)
} }
ml = false ml = false
p.spec(s, len(d.Specs), false, &ml) p.spec(s, len(d.Specs), false, &ml)
@ -1337,7 +1339,7 @@ func (p *printer) nodeSize(n ast.Node, maxSize int) (size int) {
// in RawFormat // in RawFormat
cfg := Config{Mode: RawFormat} cfg := Config{Mode: RawFormat}
var buf bytes.Buffer var buf bytes.Buffer
if _, err := cfg.Fprint(&buf, n); err != nil { if _, err := cfg.Fprint(&buf, p.fset, n); err != nil {
return return
} }
if buf.Len() <= maxSize { if buf.Len() <= maxSize {
@ -1355,11 +1357,11 @@ func (p *printer) nodeSize(n ast.Node, maxSize int) (size int) {
func (p *printer) isOneLineFunc(b *ast.BlockStmt, headerSize int) bool { func (p *printer) isOneLineFunc(b *ast.BlockStmt, headerSize int) bool {
pos1 := b.Pos() pos1 := b.Pos()
pos2 := b.Rbrace pos2 := b.Rbrace
if pos1.IsValid() && pos2.IsValid() && pos1.Line != pos2.Line { if pos1.IsValid() && pos2.IsValid() && p.fset.Position(pos1).Line != p.fset.Position(pos2).Line {
// opening and closing brace are on different lines - don't make it a one-liner // opening and closing brace are on different lines - don't make it a one-liner
return false return false
} }
if len(b.List) > 5 || p.commentBefore(pos2) { if len(b.List) > 5 || p.commentBefore(p.fset.Position(pos2)) {
// too many statements or there is a comment inside - don't make it a one-liner // too many statements or there is a comment inside - don't make it a one-liner
return false return false
} }
@ -1416,7 +1418,8 @@ func (p *printer) funcBody(b *ast.BlockStmt, headerSize int, isLit bool, multiLi
// distance returns the column difference between from and to if both // distance returns the column difference between from and to if both
// are on the same line; if they are on different lines (or unknown) // are on the same line; if they are on different lines (or unknown)
// the result is infinity. // the result is infinity.
func distance(from, to token.Position) int { func (p *printer) distance(from0 token.Pos, to token.Position) int {
from := p.fset.Position(from0)
if from.IsValid() && to.IsValid() && from.Line == to.Line { if from.IsValid() && to.IsValid() && from.Line == to.Line {
return to.Column - from.Column return to.Column - from.Column
} }
@ -1434,7 +1437,7 @@ func (p *printer) funcDecl(d *ast.FuncDecl, multiLine *bool) {
} }
p.expr(d.Name, multiLine) p.expr(d.Name, multiLine)
p.signature(d.Type.Params, d.Type.Results, multiLine) p.signature(d.Type.Params, d.Type.Results, multiLine)
p.funcBody(d.Body, distance(d.Pos(), p.pos), false, multiLine) p.funcBody(d.Body, p.distance(d.Pos(), p.pos), false, multiLine)
} }
@ -1484,7 +1487,7 @@ func (p *printer) file(src *ast.File) {
if prev != tok { if prev != tok {
min = 2 min = 2
} }
p.linebreak(d.Pos().Line, min, ignore, false) p.linebreak(p.fset.Position(d.Pos()).Line, min, ignore, false)
p.decl(d, ignoreMultiLine) p.decl(d, ignoreMultiLine)
} }
} }

27
src/pkg/go/printer/printer.go
View file

@ -62,6 +62,7 @@ type printer struct {
// Configuration (does not change after initialization) // Configuration (does not change after initialization)
output io.Writer output io.Writer
Config Config
fset *token.FileSet
errors chan os.Error errors chan os.Error
// Current state // Current state
@ -94,9 +95,10 @@ type printer struct {
} }
func (p *printer) init(output io.Writer, cfg *Config) { func (p *printer) init(output io.Writer, cfg *Config, fset *token.FileSet) {
p.output = output p.output = output
p.Config = *cfg p.Config = *cfg
p.fset = fset
p.errors = make(chan os.Error) p.errors = make(chan os.Error)
p.buffer = make([]whiteSpace, 0, 16) // whitespace sequences are short p.buffer = make([]whiteSpace, 0, 16) // whitespace sequences are short
} }
@ -596,7 +598,7 @@ func (p *printer) writeComment(comment *ast.Comment) {
// shortcut common case of //-style comments // shortcut common case of //-style comments
if text[1] == '/' { if text[1] == '/' {
p.writeCommentLine(comment, comment.Pos(), text) p.writeCommentLine(comment, p.fset.Position(comment.Pos()), text)
return return
} }
@ -608,7 +610,7 @@ func (p *printer) writeComment(comment *ast.Comment) {
// write comment lines, separated by formfeed, // write comment lines, separated by formfeed,
// without a line break after the last line // without a line break after the last line
linebreak := formfeeds[0:1] linebreak := formfeeds[0:1]
pos := comment.Pos() pos := p.fset.Position(comment.Pos())
for i, line := range lines { for i, line := range lines {
if i > 0 { if i > 0 {
p.write(linebreak) p.write(linebreak)
@ -669,14 +671,14 @@ func (p *printer) intersperseComments(next token.Position, tok token.Token) (dro
var last *ast.Comment var last *ast.Comment
for ; p.commentBefore(next); p.cindex++ { for ; p.commentBefore(next); p.cindex++ {
for _, c := range p.comments[p.cindex].List { for _, c := range p.comments[p.cindex].List {
p.writeCommentPrefix(c.Pos(), next, last == nil, tok.IsKeyword()) p.writeCommentPrefix(p.fset.Position(c.Pos()), next, last == nil, tok.IsKeyword())
p.writeComment(c) p.writeComment(c)
last = c last = c
} }
} }
if last != nil { if last != nil {
if last.Text[1] == '*' && last.Pos().Line == next.Line { if last.Text[1] == '*' && p.fset.Position(last.Pos()).Line == next.Line {
// the last comment is a /*-style comment and the next item // the last comment is a /*-style comment and the next item
// follows on the same line: separate with an extra blank // follows on the same line: separate with an extra blank
p.write([]byte{' '}) p.write([]byte{' '})
@ -842,9 +844,9 @@ func (p *printer) print(args ...interface{}) {
data = []byte(s) data = []byte(s)
} }
tok = x tok = x
case token.Position: case token.Pos:
if x.IsValid() { if x.IsValid() {
next = x // accurate position of next item next = p.fset.Position(x) // accurate position of next item
} }
tok = p.lastTok tok = p.lastTok
default: default:
@ -873,7 +875,7 @@ func (p *printer) print(args ...interface{}) {
// before the next position in the source code. // before the next position in the source code.
// //
func (p *printer) commentBefore(next token.Position) bool { func (p *printer) commentBefore(next token.Position) bool {
return p.cindex < len(p.comments) && p.comments[p.cindex].List[0].Pos().Offset < next.Offset return p.cindex < len(p.comments) && p.fset.Position(p.comments[p.cindex].List[0].Pos()).Offset < next.Offset
} }
@ -1026,10 +1028,11 @@ type Config struct {
// Fprint "pretty-prints" an AST node to output and returns the number // Fprint "pretty-prints" an AST node to output and returns the number
// of bytes written and an error (if any) for a given configuration cfg. // of bytes written and an error (if any) for a given configuration cfg.
// Position information is interpreted relative to the file set fset.
// The node type must be *ast.File, or assignment-compatible to ast.Expr, // The node type must be *ast.File, or assignment-compatible to ast.Expr,
// ast.Decl, ast.Spec, or ast.Stmt. // ast.Decl, ast.Spec, or ast.Stmt.
// //
func (cfg *Config) Fprint(output io.Writer, node interface{}) (int, os.Error) { func (cfg *Config) Fprint(output io.Writer, fset *token.FileSet, node interface{}) (int, os.Error) {
// redirect output through a trimmer to eliminate trailing whitespace // redirect output through a trimmer to eliminate trailing whitespace
// (Input to a tabwriter must be untrimmed since trailing tabs provide // (Input to a tabwriter must be untrimmed since trailing tabs provide
// formatting information. The tabwriter could provide trimming // formatting information. The tabwriter could provide trimming
@ -1061,7 +1064,7 @@ func (cfg *Config) Fprint(output io.Writer, node interface{}) (int, os.Error) {
// setup printer and print node // setup printer and print node
var p printer var p printer
p.init(output, cfg) p.init(output, cfg, fset)
go func() { go func() {
switch n := node.(type) { switch n := node.(type) {
case ast.Expr: case ast.Expr:
@ -1111,7 +1114,7 @@ func (cfg *Config) Fprint(output io.Writer, node interface{}) (int, os.Error) {
// Fprint "pretty-prints" an AST node to output. // Fprint "pretty-prints" an AST node to output.
// It calls Config.Fprint with default settings. // It calls Config.Fprint with default settings.
// //
func Fprint(output io.Writer, node interface{}) os.Error { func Fprint(output io.Writer, fset *token.FileSet, node interface{}) os.Error {
_, err := (&Config{Tabwidth: 8}).Fprint(output, node) // don't care about number of bytes written _, err := (&Config{Tabwidth: 8}).Fprint(output, fset, node) // don't care about number of bytes written
return err return err
} }

8
src/pkg/go/printer/printer_test.go
View file

@ -10,6 +10,7 @@ import (
"io/ioutil" "io/ioutil"
"go/ast" "go/ast"
"go/parser" "go/parser"
"go/token"
"path" "path"
"testing" "testing"
) )
@ -24,6 +25,9 @@ const (
var update = flag.Bool("update", false, "update golden files") var update = flag.Bool("update", false, "update golden files")
var fset = token.NewFileSet()
func lineString(text []byte, i int) string { func lineString(text []byte, i int) string {
i0 := i i0 := i
for i < len(text) && text[i] != '\n' { for i < len(text) && text[i] != '\n' {
@ -43,7 +47,7 @@ const (
func check(t *testing.T, source, golden string, mode checkMode) { func check(t *testing.T, source, golden string, mode checkMode) {
// parse source // parse source
prog, err := parser.ParseFile(source, nil, parser.ParseComments) prog, err := parser.ParseFile(fset, source, nil, parser.ParseComments)
if err != nil { if err != nil {
t.Error(err) t.Error(err)
return return
@ -63,7 +67,7 @@ func check(t *testing.T, source, golden string, mode checkMode) {
// format source // format source
var buf bytes.Buffer var buf bytes.Buffer
if _, err := cfg.Fprint(&buf, prog); err != nil { if _, err := cfg.Fprint(&buf, fset, prog); err != nil {
t.Error(err) t.Error(err)
} }
res := buf.Bytes() res := buf.Bytes()

125
src/pkg/go/scanner/scanner.go
View file

@ -24,18 +24,16 @@ import (
// //
type Scanner struct { type Scanner struct {
// immutable state // immutable state
file *token.File // source file handle
src []byte // source src []byte // source
err ErrorHandler // error reporting; or nil err ErrorHandler // error reporting; or nil
mode uint // scanning mode mode uint // scanning mode
// scanning state // scanning state
filename string // current filename; may change via //line filename:line comment
line int // current line
column int // current column
ch int // current character ch int // current character
offset int // character offset offset int // character offset
rdOffset int // reading offset (position after current character) rdOffset int // reading offset (position after current character)
lineOffset int // current line offset
insertSemi bool // insert a semicolon before next newline insertSemi bool // insert a semicolon before next newline
// public state - ok to modify // public state - ok to modify
@ -47,22 +45,21 @@ type Scanner struct {
// S.ch < 0 means end-of-file. // S.ch < 0 means end-of-file.
// //
func (S *Scanner) next() { func (S *Scanner) next() {
S.column++
if S.rdOffset < len(S.src) { if S.rdOffset < len(S.src) {
S.offset = S.rdOffset S.offset = S.rdOffset
if S.ch == '\n' { if S.ch == '\n' {
S.line++ S.lineOffset = S.offset
S.column = 1 S.file.AddLine(S.offset)
} }
r, w := int(S.src[S.rdOffset]), 1 r, w := int(S.src[S.rdOffset]), 1
switch { switch {
case r == 0: case r == 0:
S.error("illegal character NUL") S.error(S.offset, "illegal character NUL")
case r >= 0x80: case r >= 0x80:
// not ASCII // not ASCII
r, w = utf8.DecodeRune(S.src[S.rdOffset:]) r, w = utf8.DecodeRune(S.src[S.rdOffset:])
if r == utf8.RuneError && w == 1 { if r == utf8.RuneError && w == 1 {
S.error("illegal UTF-8 encoding") S.error(S.offset, "illegal UTF-8 encoding")
} }
} }
S.rdOffset += w S.rdOffset += w
@ -70,7 +67,8 @@ func (S *Scanner) next() {
} else { } else {
S.offset = len(S.src) S.offset = len(S.src)
if S.ch == '\n' { if S.ch == '\n' {
S.column = 1 S.lineOffset = S.offset
S.file.AddLine(S.offset)
} }
S.ch = -1 // eof S.ch = -1 // eof
} }
@ -86,32 +84,37 @@ const (
InsertSemis // automatically insert semicolons InsertSemis // automatically insert semicolons
) )
// TODO(gri) Would it be better to simply provide *token.File to Init
// instead of fset, and filename, and then return the file?
// It could cause an error/panic if the provided file.Size()
// doesn't match len(src).
// Init prepares the scanner S to tokenize the text src. Calls to Scan // Init prepares the scanner S to tokenize the text src. It sets the
// will use the error handler err if they encounter a syntax error and // scanner at the beginning of the source text, adds a new file with
// err is not nil. Also, for each error encountered, the Scanner field // the given filename to the file set fset, and returns that file.
// ErrorCount is incremented by one. The filename parameter is used as
// filename in the token.Position returned by Scan for each token. The
// mode parameter determines how comments and illegal characters are
// handled.
// //
func (S *Scanner) Init(filename string, src []byte, err ErrorHandler, mode uint) { // Calls to Scan will use the error handler err if they encounter a
// syntax error and err is not nil. Also, for each error encountered,
// the Scanner field ErrorCount is incremented by one. The mode parameter
// determines how comments, illegal characters, and semicolons are handled.
//
func (S *Scanner) Init(fset *token.FileSet, filename string, src []byte, err ErrorHandler, mode uint) *token.File {
// Explicitly initialize all fields since a scanner may be reused. // Explicitly initialize all fields since a scanner may be reused.
S.file = fset.AddFile(filename, fset.Base(), len(src))
S.src = src S.src = src
S.err = err S.err = err
S.mode = mode S.mode = mode
S.filename = filename
S.line = 1
S.column = 0
S.ch = ' ' S.ch = ' '
S.offset = 0 S.offset = 0
S.rdOffset = 0 S.rdOffset = 0
S.lineOffset = 0
S.insertSemi = false S.insertSemi = false
S.ErrorCount = 0 S.ErrorCount = 0
S.next() S.next()
return S.file
} }
@ -145,14 +148,9 @@ func charString(ch int) string {
} }
func (S *Scanner) error(msg string) { func (S *Scanner) error(offs int, msg string) {
S.errorAt(token.Position{S.filename, S.offset, S.line, S.column}, msg)
}
func (S *Scanner) errorAt(pos token.Position, msg string) {
if S.err != nil { if S.err != nil {
S.err.Error(pos, msg) S.err.Error(S.file.Position(S.file.Pos(offs)), msg)
} }
S.ErrorCount++ S.ErrorCount++
} }
@ -167,8 +165,7 @@ func (S *Scanner) interpretLineComment(text []byte) {
if line, err := strconv.Atoi(string(text[i+1:])); err == nil && line > 0 { if line, err := strconv.Atoi(string(text[i+1:])); err == nil && line > 0 {
// valid //line filename:line comment; // valid //line filename:line comment;
// update scanner position // update scanner position
S.filename = string(text[len(prefix):i]) S.file.AddLineInfo(S.lineOffset, string(text[len(prefix):i]), line-1) // -1 since comment applies to next line
S.line = line - 1 // -1 since the '\n' has not been consumed yet
} }
} }
} }
@ -178,8 +175,6 @@ func (S *Scanner) interpretLineComment(text []byte) {
func (S *Scanner) scanComment() { func (S *Scanner) scanComment() {
// initial '/' already consumed; S.ch == '/' || S.ch == '*' // initial '/' already consumed; S.ch == '/' || S.ch == '*'
offs := S.offset - 1 // position of initial '/' offs := S.offset - 1 // position of initial '/'
col := S.column - 1
pos := token.Position{S.filename, S.offset - 1, S.line, S.column - 1}
if S.ch == '/' { if S.ch == '/' {
//-style comment //-style comment
@ -187,7 +182,7 @@ func (S *Scanner) scanComment() {
for S.ch != '\n' && S.ch >= 0 { for S.ch != '\n' && S.ch >= 0 {
S.next() S.next()
} }
if col == 1 { if offs == S.lineOffset {
// comment starts at the beginning of the current line // comment starts at the beginning of the current line
S.interpretLineComment(S.src[offs:S.offset]) S.interpretLineComment(S.src[offs:S.offset])
} }
@ -205,24 +200,20 @@ func (S *Scanner) scanComment() {
} }
} }
S.errorAt(pos, "comment not terminated") S.error(offs, "comment not terminated")
} }
func (S *Scanner) findLineEnd() bool { func (S *Scanner) findLineEnd() bool {
// initial '/' already consumed // initial '/' already consumed
defer func(line, col, offs int) { defer func(offs int) {
// reset scanner state to where it was upon calling findLineEnd // reset scanner state to where it was upon calling findLineEnd
// (we don't scan //line comments and ignore errors thus
// S.filename and S.ErrorCount don't change)
S.line = line
S.column = col
S.ch = '/' S.ch = '/'
S.offset = offs S.offset = offs
S.rdOffset = offs + 1 S.rdOffset = offs + 1
S.next() // consume initial '/' again S.next() // consume initial '/' again
}(S.line, S.column-1, S.offset-1) }(S.offset - 1)
// read ahead until a newline, EOF, or non-comment token is found // read ahead until a newline, EOF, or non-comment token is found
for S.ch == '/' || S.ch == '*' { for S.ch == '/' || S.ch == '*' {
@ -309,7 +300,7 @@ func (S *Scanner) scanNumber(seenDecimalPoint bool) token.Token {
if S.ch == '0' { if S.ch == '0' {
// int or float // int or float
pos := token.Position{S.filename, S.offset, S.line, S.column} offs := S.offset
S.next() S.next()
if S.ch == 'x' || S.ch == 'X' { if S.ch == 'x' || S.ch == 'X' {
// hexadecimal int // hexadecimal int
@ -329,7 +320,7 @@ func (S *Scanner) scanNumber(seenDecimalPoint bool) token.Token {
} }
// octal int // octal int
if seenDecimalDigit { if seenDecimalDigit {
S.errorAt(pos, "illegal octal number") S.error(offs, "illegal octal number")
} }
} }
goto exit goto exit
@ -366,7 +357,7 @@ exit:
func (S *Scanner) scanEscape(quote int) { func (S *Scanner) scanEscape(quote int) {
pos := token.Position{S.filename, S.offset, S.line, S.column} offs := S.offset
var i, base, max uint32 var i, base, max uint32
switch S.ch { switch S.ch {
@ -386,7 +377,7 @@ func (S *Scanner) scanEscape(quote int) {
i, base, max = 8, 16, unicode.MaxRune i, base, max = 8, 16, unicode.MaxRune
default: default:
S.next() // always make progress S.next() // always make progress
S.errorAt(pos, "unknown escape sequence") S.error(offs, "unknown escape sequence")
return return
} }
@ -394,7 +385,7 @@ func (S *Scanner) scanEscape(quote int) {
for ; i > 0 && S.ch != quote && S.ch >= 0; i-- { for ; i > 0 && S.ch != quote && S.ch >= 0; i-- {
d := uint32(digitVal(S.ch)) d := uint32(digitVal(S.ch))
if d >= base { if d >= base {
S.error("illegal character in escape sequence") S.error(S.offset, "illegal character in escape sequence")
break break
} }
x = x*base + d x = x*base + d
@ -405,14 +396,14 @@ func (S *Scanner) scanEscape(quote int) {
S.next() S.next()
} }
if x > max || 0xd800 <= x && x < 0xe000 { if x > max || 0xd800 <= x && x < 0xe000 {
S.errorAt(pos, "escape sequence is invalid Unicode code point") S.error(offs, "escape sequence is invalid Unicode code point")
} }
} }
func (S *Scanner) scanChar() { func (S *Scanner) scanChar() {
// '\'' opening already consumed // '\'' opening already consumed
pos := token.Position{S.filename, S.offset - 1, S.line, S.column - 1} offs := S.offset - 1
n := 0 n := 0
for S.ch != '\'' { for S.ch != '\'' {
@ -420,7 +411,7 @@ func (S *Scanner) scanChar() {
n++ n++
S.next() S.next()
if ch == '\n' || ch < 0 { if ch == '\n' || ch < 0 {
S.errorAt(pos, "character literal not terminated") S.error(offs, "character literal not terminated")
n = 1 n = 1
break break
} }
@ -432,20 +423,20 @@ func (S *Scanner) scanChar() {
S.next() S.next()
if n != 1 { if n != 1 {
S.errorAt(pos, "illegal character literal") S.error(offs, "illegal character literal")
} }
} }
func (S *Scanner) scanString() { func (S *Scanner) scanString() {
// '"' opening already consumed // '"' opening already consumed
pos := token.Position{S.filename, S.offset - 1, S.line, S.column - 1} offs := S.offset - 1
for S.ch != '"' { for S.ch != '"' {
ch := S.ch ch := S.ch
S.next() S.next()
if ch == '\n' || ch < 0 { if ch == '\n' || ch < 0 {
S.errorAt(pos, "string not terminated") S.error(offs, "string not terminated")
break break
} }
if ch == '\\' { if ch == '\\' {
@ -459,13 +450,13 @@ func (S *Scanner) scanString() {
func (S *Scanner) scanRawString() { func (S *Scanner) scanRawString() {
// '`' opening already consumed // '`' opening already consumed
pos := token.Position{S.filename, S.offset - 1, S.line, S.column - 1} offs := S.offset - 1
for S.ch != '`' { for S.ch != '`' {
ch := S.ch ch := S.ch
S.next() S.next()
if ch < 0 { if ch < 0 {
S.errorAt(pos, "string not terminated") S.error(offs, "string not terminated")
break break
} }
} }
@ -544,14 +535,18 @@ var newline = []byte{'\n'}
// must check the scanner's ErrorCount or the number of calls // must check the scanner's ErrorCount or the number of calls
// of the error handler, if there was one installed. // of the error handler, if there was one installed.
// //
func (S *Scanner) Scan() (pos token.Position, tok token.Token, lit []byte) { // Scan adds line information to the file added to the file
// set with Init. Token positions are relative to that file
// and thus relative to the file set.
//
func (S *Scanner) Scan() (token.Pos, token.Token, []byte) {
scanAgain: scanAgain:
S.skipWhitespace() S.skipWhitespace()
// current token start // current token start
insertSemi := false insertSemi := false
pos, tok = token.Position{S.filename, S.offset, S.line, S.column}, token.ILLEGAL
offs := S.offset offs := S.offset
tok := token.ILLEGAL
// determine token value // determine token value
switch ch := S.ch; { switch ch := S.ch; {
@ -570,7 +565,7 @@ scanAgain:
case -1: case -1:
if S.insertSemi { if S.insertSemi {
S.insertSemi = false // EOF consumed S.insertSemi = false // EOF consumed
return pos, token.SEMICOLON, newline return S.file.Pos(offs), token.SEMICOLON, newline
} }
tok = token.EOF tok = token.EOF
case '\n': case '\n':
@ -578,7 +573,7 @@ scanAgain:
// set in the first place and exited early // set in the first place and exited early
// from S.skipWhitespace() // from S.skipWhitespace()
S.insertSemi = false // newline consumed S.insertSemi = false // newline consumed
return pos, token.SEMICOLON, newline return S.file.Pos(offs), token.SEMICOLON, newline
case '"': case '"':
insertSemi = true insertSemi = true
tok = token.STRING tok = token.STRING
@ -640,17 +635,13 @@ scanAgain:
case '/': case '/':
if S.ch == '/' || S.ch == '*' { if S.ch == '/' || S.ch == '*' {
// comment // comment
line := S.line
col := S.column - 1 // beginning of comment
if S.insertSemi && S.findLineEnd() { if S.insertSemi && S.findLineEnd() {
// reset position to the beginning of the comment // reset position to the beginning of the comment
S.line = line
S.column = col
S.ch = '/' S.ch = '/'
S.offset = offs S.offset = offs
S.rdOffset = offs + 1 S.rdOffset = offs + 1
S.insertSemi = false // newline consumed S.insertSemi = false // newline consumed
return pos, token.SEMICOLON, newline return S.file.Pos(offs), token.SEMICOLON, newline
} }
S.scanComment() S.scanComment()
if S.mode&ScanComments == 0 { if S.mode&ScanComments == 0 {
@ -690,7 +681,7 @@ scanAgain:
tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR) tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LOR)
default: default:
if S.mode&AllowIllegalChars == 0 { if S.mode&AllowIllegalChars == 0 {
S.errorAt(pos, "illegal character "+charString(ch)) S.error(offs, "illegal character "+charString(ch))
} }
insertSemi = S.insertSemi // preserve insertSemi info insertSemi = S.insertSemi // preserve insertSemi info
} }
@ -699,7 +690,7 @@ scanAgain:
if S.mode&InsertSemis != 0 { if S.mode&InsertSemis != 0 {
S.insertSemi = insertSemi S.insertSemi = insertSemi
} }
return pos, tok, S.src[offs:S.offset] return S.file.Pos(offs), tok, S.src[offs:S.offset]
} }
@ -709,9 +700,9 @@ scanAgain:
// false (usually when the token value is token.EOF). The result is the number // false (usually when the token value is token.EOF). The result is the number
// of errors encountered. // of errors encountered.
// //
func Tokenize(filename string, src []byte, err ErrorHandler, mode uint, f func(pos token.Position, tok token.Token, lit []byte) bool) int { func Tokenize(set *token.FileSet, filename string, src []byte, err ErrorHandler, mode uint, f func(pos token.Pos, tok token.Token, lit []byte) bool) int {
var s Scanner var s Scanner
s.Init(filename, src, err, mode) s.Init(set, filename, src, err, mode)
for f(s.Scan()) { for f(s.Scan()) {
// action happens in f // action happens in f
} }

47
src/pkg/go/scanner/scanner_test.go
View file

@ -11,6 +11,9 @@ import (
) )
var fset = token.NewFileSet()
const /* class */ ( const /* class */ (
special = iota special = iota
literal literal
@ -196,7 +199,8 @@ func newlineCount(s string) int {
} }
func checkPos(t *testing.T, lit string, pos, expected token.Position) { func checkPos(t *testing.T, lit string, p token.Pos, expected token.Position) {
pos := fset.Position(p)
if pos.Filename != expected.Filename { if pos.Filename != expected.Filename {
t.Errorf("bad filename for %q: got %s, expected %s", lit, pos.Filename, expected.Filename) t.Errorf("bad filename for %q: got %s, expected %s", lit, pos.Filename, expected.Filename)
} }
@ -219,14 +223,14 @@ func TestScan(t *testing.T) {
for _, e := range tokens { for _, e := range tokens {
src += e.lit + whitespace src += e.lit + whitespace
} }
src_linecount := newlineCount(src) src_linecount := newlineCount(src) + 1
whitespace_linecount := newlineCount(whitespace) whitespace_linecount := newlineCount(whitespace)
// verify scan // verify scan
index := 0 index := 0
epos := token.Position{"", 0, 1, 1} // expected position epos := token.Position{"", 0, 1, 1} // expected position
nerrors := Tokenize("", []byte(src), &testErrorHandler{t}, ScanComments, nerrors := Tokenize(fset, "", []byte(src), &testErrorHandler{t}, ScanComments,
func(pos token.Position, tok token.Token, litb []byte) bool { func(pos token.Pos, tok token.Token, litb []byte) bool {
e := elt{token.EOF, "", special} e := elt{token.EOF, "", special}
if index < len(tokens) { if index < len(tokens) {
e = tokens[index] e = tokens[index]
@ -265,7 +269,7 @@ func TestScan(t *testing.T) {
func checkSemi(t *testing.T, line string, mode uint) { func checkSemi(t *testing.T, line string, mode uint) {
var S Scanner var S Scanner
S.Init("TestSemis", []byte(line), nil, mode) file := S.Init(fset, "TestSemis", []byte(line), nil, mode)
pos, tok, lit := S.Scan() pos, tok, lit := S.Scan()
for tok != token.EOF { for tok != token.EOF {
if tok == token.ILLEGAL { if tok == token.ILLEGAL {
@ -276,7 +280,7 @@ func checkSemi(t *testing.T, line string, mode uint) {
semiLit = ";" semiLit = ";"
} }
// next token must be a semicolon // next token must be a semicolon
semiPos := pos semiPos := file.Position(pos)
semiPos.Offset++ semiPos.Offset++
semiPos.Column++ semiPos.Column++
pos, tok, lit = S.Scan() pos, tok, lit = S.Scan()
@ -468,10 +472,11 @@ func TestLineComments(t *testing.T) {
// verify scan // verify scan
var S Scanner var S Scanner
S.Init("TestLineComments", []byte(src), nil, 0) file := S.Init(fset, "TestLineComments", []byte(src), nil, 0)
for _, s := range segments { for _, s := range segments {
pos, _, lit := S.Scan() p, _, lit := S.Scan()
checkPos(t, string(lit), pos, token.Position{s.filename, pos.Offset, s.line, pos.Column}) pos := file.Position(p)
checkPos(t, string(lit), p, token.Position{s.filename, pos.Offset, s.line, pos.Column})
} }
if S.ErrorCount != 0 { if S.ErrorCount != 0 {
@ -485,7 +490,11 @@ func TestInit(t *testing.T) {
var s Scanner var s Scanner
// 1st init // 1st init
s.Init("", []byte("if true { }"), nil, 0) src1 := "if true { }"
f1 := s.Init(fset, "", []byte(src1), nil, 0)
if f1.Size() != len(src1) {
t.Errorf("bad file size: got %d, expected %d", f1.Size(), len(src1))
}
s.Scan() // if s.Scan() // if
s.Scan() // true s.Scan() // true
_, tok, _ := s.Scan() // { _, tok, _ := s.Scan() // {
@ -494,7 +503,11 @@ func TestInit(t *testing.T) {
} }
// 2nd init // 2nd init
s.Init("", []byte("go true { ]"), nil, 0) src2 := "go true { ]"
f2 := s.Init(fset, "", []byte(src2), nil, 0)
if f2.Size() != len(src2) {
t.Errorf("bad file size: got %d, expected %d", f2.Size(), len(src2))
}
_, tok, _ = s.Scan() // go _, tok, _ = s.Scan() // go
if tok != token.GO { if tok != token.GO {
t.Errorf("bad token: got %s, expected %s", tok.String(), token.GO) t.Errorf("bad token: got %s, expected %s", tok.String(), token.GO)
@ -510,11 +523,11 @@ func TestIllegalChars(t *testing.T) {
var s Scanner var s Scanner
const src = "*?*$*@*" const src = "*?*$*@*"
s.Init("", []byte(src), &testErrorHandler{t}, AllowIllegalChars) file := s.Init(fset, "", []byte(src), &testErrorHandler{t}, AllowIllegalChars)
for offs, ch := range src { for offs, ch := range src {
pos, tok, lit := s.Scan() pos, tok, lit := s.Scan()
if pos.Offset != offs { if poffs := file.Offset(pos); poffs != offs {
t.Errorf("bad position for %s: got %d, expected %d", string(lit), pos.Offset, offs) t.Errorf("bad position for %s: got %d, expected %d", string(lit), poffs, offs)
} }
if tok == token.ILLEGAL && string(lit) != string(ch) { if tok == token.ILLEGAL && string(lit) != string(ch) {
t.Errorf("bad token: got %s, expected %s", string(lit), string(ch)) t.Errorf("bad token: got %s, expected %s", string(lit), string(ch))
@ -538,8 +551,8 @@ func TestStdErrorHander(t *testing.T) {
"@ @ @" // original file, line 1 again "@ @ @" // original file, line 1 again
v := new(ErrorVector) v := new(ErrorVector)
nerrors := Tokenize("File1", []byte(src), v, 0, nerrors := Tokenize(fset, "File1", []byte(src), v, 0,
func(pos token.Position, tok token.Token, litb []byte) bool { func(pos token.Pos, tok token.Token, litb []byte) bool {
return tok != token.EOF return tok != token.EOF
}) })
@ -584,7 +597,7 @@ func (h *errorCollector) Error(pos token.Position, msg string) {
func checkError(t *testing.T, src string, tok token.Token, pos int, err string) { func checkError(t *testing.T, src string, tok token.Token, pos int, err string) {
var s Scanner var s Scanner
var h errorCollector var h errorCollector
s.Init("", []byte(src), &h, ScanComments) s.Init(fset, "", []byte(src), &h, ScanComments)
_, tok0, _ := s.Scan() _, tok0, _ := s.Scan()
_, tok1, _ := s.Scan() _, tok1, _ := s.Scan()
if tok0 != tok { if tok0 != tok {

31
src/pkg/go/token/position.go
View file

@ -116,6 +116,10 @@ func (s *FileSet) file(p Pos) *File {
// Position converts a Pos in the fileset into a general Position. // Position converts a Pos in the fileset into a general Position.
func (s *FileSet) Position(p Pos) (pos Position) { func (s *FileSet) Position(p Pos) (pos Position) {
if p != NoPos { if p != NoPos {
// TODO(gri) consider optimizing the case where p
// is in the last file addded, or perhaps
// looked at - will eliminate one level
// of search
s.mutex.RLock() s.mutex.RLock()
if f := s.file(p); f != nil { if f := s.file(p); f != nil {
offset := int(p) - f.base offset := int(p) - f.base
@ -242,7 +246,7 @@ func (f *File) Pos(offset int) Pos {
// Offset returns the offset for the given file position p; // Offset returns the offset for the given file position p;
// p must be a Pos value in that file. // p must be a valid Pos value in that file.
// f.Offset(f.Pos(offset)) == offset. // f.Offset(f.Pos(offset)) == offset.
// //
func (f *File) Offset(p Pos) int { func (f *File) Offset(p Pos) int {
@ -253,14 +257,27 @@ func (f *File) Offset(p Pos) int {
} }
// Position returns the Position value for the given file offset; // Line returns the line number for the given file position p;
// the offset must be <= f.Size(). // p must be a Pos value in that file or NoPos.
// //
func (f *File) Position(offset int) Position { func (f *File) Line(p Pos) int {
if offset > f.size { // TODO(gri) this can be implemented much more efficiently
panic("illegal file offset") return f.Position(p).Line
}
// Position returns the Position value for the given file position p;
// p must be a Pos value in that file or NoPos.
//
func (f *File) Position(p Pos) (pos Position) {
if p != NoPos {
if int(p) < f.base || int(p) > f.base+f.size {
panic("illegal Pos value")
}
// TODO(gri) compute Position directly instead of going via the fset!
pos = f.set.Position(p)
} }
return f.set.Position(Pos(offset + f.base)) return
} }

4
src/pkg/go/token/position_test.go
View file

@ -71,7 +71,7 @@ func verifyPositions(t *testing.T, fset *FileSet, f *File, lines []int) {
} }
line, col := linecol(lines, offs) line, col := linecol(lines, offs)
msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p) msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
checkPos(t, msg, f.Position(offs), Position{f.Name(), offs, line, col}) checkPos(t, msg, f.Position(f.Pos(offs)), Position{f.Name(), offs, line, col})
checkPos(t, msg, fset.Position(p), Position{f.Name(), offs, line, col}) checkPos(t, msg, fset.Position(p), Position{f.Name(), offs, line, col})
} }
} }
@ -131,7 +131,7 @@ func TestLineInfo(t *testing.T) {
p := f.Pos(offs) p := f.Pos(offs)
_, col := linecol(lines, offs) _, col := linecol(lines, offs)
msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p) msg := fmt.Sprintf("%s (offs = %d, p = %d)", f.Name(), offs, p)
checkPos(t, msg, f.Position(offs), Position{"bar", offs, 42, col}) checkPos(t, msg, f.Position(f.Pos(offs)), Position{"bar", offs, 42, col})
checkPos(t, msg, fset.Position(p), Position{"bar", offs, 42, col}) checkPos(t, msg, fset.Position(p), Position{"bar", offs, 42, col})
} }
} }

2
src/pkg/go/typechecker/scope.go
View file

@ -26,7 +26,7 @@ func (tc *typechecker) closeScope() {
// objPos computes the source position of the declaration of an object name. // objPos computes the source position of the declaration of an object name.
// Only required for error reporting, so doesn't have to be fast. // Only required for error reporting, so doesn't have to be fast.
func objPos(obj *ast.Object) (pos token.Position) { func objPos(obj *ast.Object) (pos token.Pos) {
switch d := obj.Decl.(type) { switch d := obj.Decl.(type) {
case *ast.Field: case *ast.Field:
for _, n := range d.Names { for _, n := range d.Names {

15
src/pkg/go/typechecker/typechecker.go
View file

@ -2,6 +2,7 @@
// Use of this source code is governed by a BSD-style // Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file. // license that can be found in the LICENSE file.
// INCOMPLETE PACKAGE.
// This package implements typechecking of a Go AST. // This package implements typechecking of a Go AST.
// The result of the typecheck is an augmented AST // The result of the typecheck is an augmented AST
// with object and type information for each identifier. // with object and type information for each identifier.
@ -37,8 +38,9 @@ type Importer func(path string) ([]byte, os.Error)
// If errors are reported, the AST may be incompletely augmented (fields // If errors are reported, the AST may be incompletely augmented (fields
// may be nil) or contain incomplete object, type, or scope information. // may be nil) or contain incomplete object, type, or scope information.
// //
func CheckPackage(pkg *ast.Package, importer Importer) os.Error { func CheckPackage(fset *token.FileSet, pkg *ast.Package, importer Importer) os.Error {
var tc typechecker var tc typechecker
tc.fset = fset
tc.importer = importer tc.importer = importer
tc.checkPackage(pkg) tc.checkPackage(pkg)
return tc.GetError(scanner.Sorted) return tc.GetError(scanner.Sorted)
@ -49,10 +51,10 @@ func CheckPackage(pkg *ast.Package, importer Importer) os.Error {
// CheckPackage. If the complete package consists of more than just // CheckPackage. If the complete package consists of more than just
// one file, the file may not typecheck without errors. // one file, the file may not typecheck without errors.
// //
func CheckFile(file *ast.File, importer Importer) os.Error { func CheckFile(fset *token.FileSet, file *ast.File, importer Importer) os.Error {
// create a single-file dummy package // create a single-file dummy package
pkg := &ast.Package{file.Name.Name, nil, map[string]*ast.File{file.Name.NamePos.Filename: file}} pkg := &ast.Package{file.Name.Name, nil, map[string]*ast.File{fset.Position(file.Name.NamePos).Filename: file}}
return CheckPackage(pkg, importer) return CheckPackage(fset, pkg, importer)
} }
@ -60,6 +62,7 @@ func CheckFile(file *ast.File, importer Importer) os.Error {
// Typechecker state // Typechecker state
type typechecker struct { type typechecker struct {
fset *token.FileSet
scanner.ErrorVector scanner.ErrorVector
importer Importer importer Importer
topScope *ast.Scope // current top-most scope topScope *ast.Scope // current top-most scope
@ -68,8 +71,8 @@ type typechecker struct {
} }
func (tc *typechecker) Errorf(pos token.Position, format string, args ...interface{}) { func (tc *typechecker) Errorf(pos token.Pos, format string, args ...interface{}) {
tc.Error(pos, fmt.Sprintf(format, args...)) tc.Error(tc.fset.Position(pos), fmt.Sprintf(format, args...))
} }

12
src/pkg/go/typechecker/typechecker_test.go
View file

@ -44,6 +44,8 @@ import (
const testDir = "./testdata" // location of test packages const testDir = "./testdata" // location of test packages
var fset = token.NewFileSet()
var ( var (
pkgPat = flag.String("pkg", ".*", "regular expression to select test packages by package name") pkgPat = flag.String("pkg", ".*", "regular expression to select test packages by package name")
trace = flag.Bool("trace", false, "print package names") trace = flag.Bool("trace", false, "print package names")
@ -66,8 +68,8 @@ func expectedErrors(t *testing.T, pkg *ast.Package) (list scanner.ErrorList) {
} }
var s scanner.Scanner var s scanner.Scanner
s.Init(filename, src, nil, scanner.ScanComments) s.Init(fset, filename, src, nil, scanner.ScanComments)
var prev token.Position // position of last non-comment token var prev token.Pos // position of last non-comment token
loop: loop:
for { for {
pos, tok, lit := s.Scan() pos, tok, lit := s.Scan()
@ -77,7 +79,7 @@ func expectedErrors(t *testing.T, pkg *ast.Package) (list scanner.ErrorList) {
case token.COMMENT: case token.COMMENT:
s := errRx.FindSubmatch(lit) s := errRx.FindSubmatch(lit)
if len(s) == 2 { if len(s) == 2 {
list = append(list, &scanner.Error{prev, string(s[1])}) list = append(list, &scanner.Error{fset.Position(prev), string(s[1])})
} }
default: default:
prev = pos prev = pos
@ -125,7 +127,7 @@ func TestTypeCheck(t *testing.T) {
t.Fatalf("illegal flag value %q: %s", *pkgPat, err) t.Fatalf("illegal flag value %q: %s", *pkgPat, err)
} }
pkgs, err := parser.ParseDir(testDir, testFilter, 0) pkgs, err := parser.ParseDir(fset, testDir, testFilter, 0)
if err != nil { if err != nil {
scanner.PrintError(os.Stderr, err) scanner.PrintError(os.Stderr, err)
t.Fatalf("packages in %s contain syntax errors", testDir) t.Fatalf("packages in %s contain syntax errors", testDir)
@ -141,7 +143,7 @@ func TestTypeCheck(t *testing.T) {
} }
xlist := expectedErrors(t, pkg) xlist := expectedErrors(t, pkg)
err := CheckPackage(pkg, nil) err := CheckPackage(fset, pkg, nil)
if err != nil { if err != nil {
if elist, ok := err.(scanner.ErrorList); ok { if elist, ok := err.(scanner.ErrorList); ok {
// verify that errors match // verify that errors match

4
test/fixedbugs/bug206.go
View file

@ -10,14 +10,14 @@ import "go/ast";
func g(list []ast.Expr) { func g(list []ast.Expr) {
n := len(list)-1; n := len(list)-1;
println(list[n].Pos().Line); println(list[n].Pos());
} }
// f is the same as g except that the expression assigned to n is inlined. // f is the same as g except that the expression assigned to n is inlined.
func f(list []ast.Expr) { func f(list []ast.Expr) {
// n := len(list)-1; // n := len(list)-1;
println(list[len(list)-1 /* n */].Pos().Line); println(list[len(list)-1 /* n */].Pos());
} }