cmd/dist: generate files for package runtime

goc2c moves here.
parallel builds like old makefiles (-j4).
add clean command.
add banner command.
implement Go version check.
real argument parsing (same as 6g etc)

Windows changes will be a separate CL.

R=golang-dev, bradfitz, iant
CC=golang-dev
https://golang.org/cl/5622058

src/cmd/dist/a.h

@@ -37,10 +37,13 @@ enum {
 
 // buf.c
 bool	bequal(Buf *s, Buf *t);
+void	bsubst(Buf *b, char *x, char *y);
 void	bfree(Buf *b);
 void	bgrow(Buf *b, int n);
 void	binit(Buf *b);
+char*	bpathf(Buf *b, char *fmt, ...);
 char*	bprintf(Buf *b, char *fmt, ...);
+void	bwritef(Buf *b, char *fmt, ...);
 void	breset(Buf *b);
 char*	bstr(Buf *b);
 char*	btake(Buf *b);
@@ -62,23 +65,41 @@ void	splitfields(Vec*, char*);
 extern char *default_goroot;
 extern char *goarch;
 extern char *gobin;
+extern char *gochar;
 extern char *gohostarch;
 extern char *gohostos;
 extern char *goos;
 extern char *goroot;
+extern char *goversion;
 extern char *workdir;
 extern char *slash;
 
+int	find(char*, char**, int);
 void	init(void);
+void	cmdbanner(int, char**);
 void	cmdbootstrap(int, char**);
+void	cmdclean(int, char**);
 void	cmdenv(int, char**);
 void	cmdinstall(int, char**);
+void	cmdversion(int, char**);
 
 // buildgc.c
 void	gcopnames(char*, char*);
 void	mkenam(char*, char*);
 
+// buildruntime.c
+void	mkzasm(char*, char*);
+void	mkzgoarch(char*, char*);
+void	mkzgoos(char*, char*);
+void	mkzruntimedefs(char*, char*);
+void	mkzversion(char*, char*);
+
+// goc2c.c
+void	goc2c(char*, char*);
+
 // main.c
+extern int vflag;
+void	usage(void);
 void	xmain(int argc, char **argv);
 
 // portability layer (plan9.c, unix.c, windows.c)
@@ -94,6 +115,8 @@ Time	mtime(char*);
 void	readfile(Buf*, char*);
 void	run(Buf *b, char *dir, int mode, char *cmd, ...);
 void	runv(Buf *b, char *dir, int mode, Vec *argv);
+void	bgrunv(char *dir, int mode, Vec *argv);
+void	bgwait(void);
 bool	streq(char*, char*);
 void	writefile(Buf*, char*);
 void	xatexit(void (*f)(void));
@@ -118,7 +141,6 @@ void	xremoveall(char *p);
 void	xsetenv(char*, char*);
 int	xstrcmp(char*, char*);
 char*	xstrdup(char *p);
-int	xstreq(char*, char*);
 int	xstrlen(char*);
 char*	xstrrchr(char*, int);
 char*	xstrstr(char*, char*);

src/cmd/dist/arg.h

@@ -0,0 +1,50 @@
+/*
+Derived from Inferno include/kern.h.
+
+http://code.google.com/p/inferno-os/source/browse/include/kern.h
+
+	Copyright © 1994-1999 Lucent Technologies Inc.  All rights reserved.
+	Revisions Copyright © 2000-2007 Vita Nuova Holdings Limited (www.vitanuova.com).  All rights reserved.
+	Portions Copyright © 2009 The Go Authors.  All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+
+/* command line */
+extern char	*argv0;
+#define	ARGBEGIN	for((argv0?0:(argv0=(*argv))),argv++,argc--;\
+			    argv[0] && argv[0][0]=='-' && argv[0][1];\
+			    argc--, argv++) {\
+				char *_args, *_argt;\
+				char _argc;\
+				_args = &argv[0][1];\
+				if(_args[0]=='-' && _args[1]==0){\
+					argc--; argv++; break;\
+				}\
+				_argc = 0;\
+				while((_argc = *_args++) != 0)\
+				switch(_argc)
+#define	ARGEND		_argt=0;USED(_argt);USED(_argc);USED(_args);}USED(argv);USED(argc);
+#define	ARGF()		(_argt=_args, _args="",\
+				(*_argt? _argt: argv[1]? (argc--, *++argv): 0))
+#define	EARGF(x)	(_argt=_args, _args="",\
+				(*_argt? _argt: argv[1]? (argc--, *++argv): ((x), fatal("usage"), (char*)0)))
+
+#define	ARGC()		_argc
+

src/cmd/dist/buf.c

@@ -99,6 +99,32 @@ bequal(Buf *s, Buf *t)
 	return s->len == t->len && xmemcmp(s->p, t->p, s->len) == 0;
 }
 
+// bsubst rewites b to replace all occurrences of x with y.
+void
+bsubst(Buf *b, char *x, char *y)
+{
+	char *p;
+	int nx, ny, pos;
+
+	nx = xstrlen(x);
+	ny = xstrlen(y);
+
+	pos = 0;
+	for(;;) {
+		p = xstrstr(bstr(b)+pos, x);
+		if(p == nil)
+			break;
+		if(nx != ny) {
+			if(nx < ny)
+				bgrow(b, ny-nx);
+			xmemmove(p+ny, p+nx, (b->p+b->len)-(p+nx));
+		}
+		xmemmove(p, y, ny);
+		pos = p+ny - b->p;
+		b->len += ny - nx;
+	}
+}
+
 // The invariant with the vectors is that v->p[0:v->len] is allocated
 // strings that are owned by the vector.  The data beyond v->len may
 // be garbage.
@@ -214,7 +240,7 @@ vuniq(Vec *v)
 void
 splitlines(Vec *v, char *p)
 {
-	int i, c;
+	int i;
 	char *start;
 	
 	vreset(v);

src/cmd/dist/buildruntime.c

@@ -0,0 +1,346 @@
+// Copyright 2012 The Go Authors.  All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "a.h"
+#include <stdio.h>
+
+/*
+ * Helpers for building pkg/runtime.
+ */
+
+// mkzversion writes zversion.go:
+//
+//	package runtime
+//	const defaultGoroot = <goroot>
+//	const theVersion = <version>
+//
+void
+mkzversion(char *dir, char *file)
+{
+	Buf b, out;
+	
+	binit(&b);
+	binit(&out);
+	
+	bwritestr(&out, bprintf(&b,
+		"// auto generated by go tool dist\n"
+		"\n"
+		"package runtime\n"
+		"\n"
+		"const defaultGoroot = `%s`\n"
+		"const theVersion = `%s`\n", goroot, goversion));
+
+	writefile(&out, file);
+	
+	bfree(&b);
+	bfree(&out);
+}
+
+// mkzgoarch writes zgoarch_$GOARCH.go:
+//
+//	package runtime
+//	const theGoarch = <goarch>
+//
+void
+mkzgoarch(char *dir, char *file)
+{
+	Buf b, out;
+	
+	binit(&b);
+	binit(&out);
+	
+	bwritestr(&out, bprintf(&b,
+		"// auto generated by go tool dist\n"
+		"\n"
+		"package runtime\n"
+		"\n"
+		"const theGoarch = `%s`\n", goarch));
+
+	writefile(&out, file);
+	
+	bfree(&b);
+	bfree(&out);
+}
+
+// mkzgoos writes zgoos_$GOOS.go:
+//
+//	package runtime
+//	const theGoos = <goos>
+//
+void
+mkzgoos(char *dir, char *file)
+{
+	Buf b, out;
+	
+	binit(&b);
+	binit(&out);
+	
+	bwritestr(&out, bprintf(&b,
+		"// auto generated by go tool dist\n"
+		"\n"
+		"package runtime\n"
+		"\n"
+		"const theGoos = `%s`\n", goos));
+
+	writefile(&out, file);
+	
+	bfree(&b);
+	bfree(&out);
+}
+
+static struct {
+	char *goarch;
+	char *goos;
+	char *hdr;
+} zasmhdr[] = {
+	{"386", "windows",
+		"#define	get_tls(r)	MOVL 0x14(FS), r\n"
+		"#define	g(r)	0(r)\n"
+		"#define	m(r)	4(r)\n"
+	},
+	{"386", "plan9",
+		"#define	get_tls(r)	MOVL _tos(SB), r \n"
+		"#define	g(r)	-8(r)\n"
+		"#define	m(r)	-4(r)\n"
+	},
+	{"386", "linux",
+		"// On Linux systems, what we call 0(GS) and 4(GS) for g and m\n"
+		"// turn into %gs:-8 and %gs:-4 (using gcc syntax to denote\n"
+		"// what the machine sees as opposed to 8l input).\n"
+		"// 8l rewrites 0(GS) and 4(GS) into these.\n"
+		"//\n"
+		"// On Linux Xen, it is not allowed to use %gs:-8 and %gs:-4\n"
+		"// directly.  Instead, we have to store %gs:0 into a temporary\n"
+		"// register and then use -8(%reg) and -4(%reg).  This kind\n"
+		"// of addressing is correct even when not running Xen.\n"
+		"//\n"
+		"// 8l can rewrite MOVL 0(GS), CX into the appropriate pair\n"
+		"// of mov instructions, using CX as the intermediate register\n"
+		"// (safe because CX is about to be written to anyway).\n"
+		"// But 8l cannot handle other instructions, like storing into 0(GS),\n"
+		"// which is where these macros come into play.\n"
+		"// get_tls sets up the temporary and then g and r use it.\n"
+		"//\n"
+		"// The final wrinkle is that get_tls needs to read from %gs:0,\n"
+		"// but in 8l input it's called 8(GS), because 8l is going to\n"
+		"// subtract 8 from all the offsets, as described above.\n"
+		"#define	get_tls(r)	MOVL 8(GS), r\n"
+		"#define	g(r)	-8(r)\n"
+		"#define	m(r)	-4(r)\n"
+	},
+	{"386", "",
+		"#define	get_tls(r)\n"
+		"#define	g(r)	0(GS)\n"
+		"#define	m(r)	4(GS)\n"
+	},
+	
+	{"amd64", "windows",
+		"#define	get_tls(r) MOVQ 0x28(GS), r\n"
+		"#define	g(r) 0(r)\n"
+		"#define	m(r) 8(r)\n"
+	},
+	{"amd64", "",
+		"// The offsets 0 and 8 are known to:\n"
+		"//	../../cmd/6l/pass.c:/D_GS\n"
+		"//	cgo/gcc_linux_amd64.c:/^threadentry\n"
+		"//	cgo/gcc_darwin_amd64.c:/^threadentry\n"
+		"//\n"
+		"#define	get_tls(r)\n"
+		"#define	g(r) 0(GS)\n"
+		"#define	m(r) 8(GS)\n"
+	},
+	
+	{"arm", "",
+	"#define	g	R10\n"
+	"#define	m	R9\n"
+	"#define	LR	R14\n"
+	},
+};
+
+// mkzasm writes zasm_$GOOS_$GOARCH.h,
+// which contains struct offsets for use by
+// assembly files.  It also writes a copy to the work space
+// under the name zasm_GOOS_GOARCH.h (no expansion).
+// 
+void
+mkzasm(char *dir, char *file)
+{
+	int i, n;
+	char *aggr, *p;
+	Buf in, b, out;
+	Vec argv, lines, fields;
+
+	binit(&in);
+	binit(&b);
+	binit(&out);
+	vinit(&argv);
+	vinit(&lines);
+	vinit(&fields);
+	
+	bwritestr(&out, "// auto generated by go tool dist\n\n");
+	for(i=0; i<nelem(zasmhdr); i++) {
+		if(hasprefix(goarch, zasmhdr[i].goarch) && hasprefix(goos, zasmhdr[i].goos)) {
+			bwritestr(&out, zasmhdr[i].hdr);
+			goto ok;
+		}
+	}
+	fatal("unknown $GOOS/$GOARCH in mkzasm");
+ok:
+
+	// Run 6c -DGOOS_goos -DGOARCH_goarch -Iworkdir -a proc.c
+	// to get acid [sic] output.
+	vreset(&argv);
+	vadd(&argv, bpathf(&b, "%s/bin/tool/%sc", goroot, gochar));
+	vadd(&argv, bprintf(&b, "-DGOOS_%s", goos));
+	vadd(&argv, bprintf(&b, "-DGOARCH_%s", goarch));
+	vadd(&argv, bprintf(&b, "-I%s", workdir));
+	vadd(&argv, "-a");
+	vadd(&argv, "proc.c");
+	runv(&in, dir, CheckExit, &argv);
+	
+	// Convert input like
+	//	aggr G
+	//	{
+	//		Gobuf 24 sched;
+	//		'Y' 48 stack0;
+	//	}
+	// into output like
+	//	#define g_sched 24
+	//	#define g_stack0 48
+	//
+	aggr = nil;
+	splitlines(&lines, bstr(&in));
+	for(i=0; i<lines.len; i++) {
+		splitfields(&fields, lines.p[i]);
+		if(fields.len == 2 && streq(fields.p[0], "aggr")) {
+			if(streq(fields.p[1], "G"))
+				aggr = "g";
+			else if(streq(fields.p[1], "M"))
+				aggr = "m";
+			else if(streq(fields.p[1], "Gobuf"))
+				aggr = "gobuf";
+			else if(streq(fields.p[1], "WinCall"))
+				aggr = "wincall";
+		}
+		if(hasprefix(lines.p[i], "}"))
+			aggr = nil;
+		if(aggr && hasprefix(lines.p[i], "\t") && fields.len >= 2) {
+			n = fields.len;
+			p = fields.p[n-1];
+			if(p[xstrlen(p)-1] == ';')
+				p[xstrlen(p)-1] = '\0';
+			bwritestr(&out, bprintf(&b, "#define %s_%s %s\n", aggr, fields.p[n-1], fields.p[n-2]));
+		}
+	}
+	
+	// Write both to file and to workdir/zasm_GOOS_GOARCH.h.
+	writefile(&out, file);
+	writefile(&out, bprintf(&b, "%s/zasm_GOOS_GOARCH.h", workdir));
+
+	bfree(&in);
+	bfree(&b);
+	bfree(&out);
+	vfree(&argv);
+	vfree(&lines);
+	vfree(&fields);
+}
+
+static char *runtimedefs[] = {
+	"proc.c",
+	"iface.c",
+	"hashmap.c",
+	"chan.c",
+};
+
+// mkzruntimedefs writes zruntime_defs_$GOOS_$GOARCH.h,
+// which contains Go struct definitions equivalent to the C ones.
+// Mostly we just write the output of 6c -q to the file.
+// However, we run it on multiple files, so we have to delete
+// the duplicated definitions, and we don't care about the funcs
+// and consts, so we delete those too.
+// 
+void
+mkzruntimedefs(char *dir, char *file)
+{
+	int i, skip;
+	char *p;
+	Buf in, b, out;
+	Vec argv, lines, fields, seen;
+	
+	binit(&in);
+	binit(&b);
+	binit(&out);
+	vinit(&argv);
+	vinit(&lines);
+	vinit(&fields);
+	vinit(&seen);
+	
+	bwritestr(&out, "// auto generated by go tool dist\n"
+		"\n"
+		"package runtime\n"
+		"import \"unsafe\"\n"
+		"var _ unsafe.Pointer\n"
+		"\n"
+	);
+
+	
+	// Run 6c -DGOOS_goos -DGOARCH_goarch -Iworkdir -q
+	// on each of the runtimedefs C files.
+	vadd(&argv, bpathf(&b, "%s/bin/tool/%sc", goroot, gochar));
+	vadd(&argv, bprintf(&b, "-DGOOS_%s", goos));
+	vadd(&argv, bprintf(&b, "-DGOARCH_%s", goarch));
+	vadd(&argv, bprintf(&b, "-I%s", workdir));
+	vadd(&argv, "-q");
+	vadd(&argv, "");
+	p = argv.p[argv.len-1];
+	for(i=0; i<nelem(runtimedefs); i++) {
+		argv.p[argv.len-1] = runtimedefs[i];
+		runv(&b, dir, CheckExit, &argv);
+		bwriteb(&in, &b);
+	}
+	argv.p[argv.len-1] = p;
+		
+	// Process the aggregate output.
+	skip = 0;
+	splitlines(&lines, bstr(&in));
+	for(i=0; i<lines.len; i++) {
+		p = lines.p[i];
+		// Drop comment, func, and const lines.
+		if(hasprefix(p, "//") || hasprefix(p, "const") || hasprefix(p, "func"))
+			continue;
+		
+		// Note beginning of type or var decl, which can be multiline.
+		// Remove duplicates.  The linear check of seen here makes the
+		// whole processing quadratic in aggregate, but there are only
+		// about 100 declarations, so this is okay (and simple).
+		if(hasprefix(p, "type ") || hasprefix(p, "var ")) {
+			splitfields(&fields, p);
+			if(fields.len < 2)
+				continue;
+			if(find(fields.p[1], seen.p, seen.len) >= 0) {
+				if(streq(fields.p[fields.len-1], "{"))
+					skip = 1;  // skip until }
+				continue;
+			}
+			vadd(&seen, fields.p[1]);
+		}
+		if(skip) {
+			if(hasprefix(p, "}"))
+				skip = 0;
+			continue;
+		}
+		
+		bwritestr(&out, p);
+	}
+	
+	writefile(&out, file);
+
+	bfree(&in);
+	bfree(&b);
+	bfree(&out);
+	vfree(&argv);
+	vfree(&lines);
+	vfree(&fields);
+	vfree(&seen);
+}

src/cmd/dist/goc2c.c

@@ -0,0 +1,727 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+#include "a.h"
+
+/*
+ * Translate a .goc file into a .c file.  A .goc file is a combination
+ * of a limited form of Go with C.
+ */
+
+/*
+	package PACKAGENAME
+	{# line}
+	func NAME([NAME TYPE { , NAME TYPE }]) [(NAME TYPE { , NAME TYPE })] \{
+	  C code with proper brace nesting
+	\}
+*/
+
+/*
+ * We generate C code which implements the function such that it can
+ * be called from Go and executes the C code.
+ */
+
+static char *input;
+static Buf *output;
+#define EOF -1
+
+static int
+xgetchar(void)
+{
+	int c;
+	
+	c = *input;
+	if(c == 0)
+		return EOF;
+	input++;
+	return c;
+}
+
+static void
+xungetc(void)
+{
+	input--;
+}
+
+static void
+xputchar(char c)
+{
+	bwrite(output, &c, 1);
+}
+
+static int
+xisspace(int c)
+{
+	return c == ' ' || c == '\t' || c == '\r' || c == '\n';
+}
+
+/* Whether we're emitting for gcc */
+static int gcc;
+
+/* File and line number */
+static const char *file;
+static unsigned int lineno = 1;
+
+/* List of names and types.  */
+struct params {
+	struct params *next;
+	char *name;
+	char *type;
+};
+
+/* index into type_table */
+enum {
+	Bool,
+	Float,
+	Int,
+	Uint,
+	Uintptr,
+	String,
+	Slice,
+	Eface,
+};
+
+static struct {
+	char *name;
+	int size;
+} type_table[] = {
+	/* variable sized first, for easy replacement */
+	/* order matches enum above */
+	/* default is 32-bit architecture sizes */
+	{"bool",		1},
+	{"float",	4},
+	{"int",		4},
+	{"uint",		4},
+	{"uintptr",	4},
+	{"String",	8},
+	{"Slice",	12},
+	{"Eface",	8},
+
+	/* fixed size */
+	{"float32",	4},
+	{"float64",	8},
+	{"byte",		1},
+	{"int8",		1},
+	{"uint8",	1},
+	{"int16",	2},
+	{"uint16",	2},
+	{"int32",	4},
+	{"uint32",	4},
+	{"int64",	8},
+	{"uint64",	8},
+
+	{nil},
+};
+
+/* Fixed structure alignment (non-gcc only) */
+int structround = 4;
+
+/* Unexpected EOF.  */
+static void
+bad_eof(void)
+{
+	fatal("%s:%ud: unexpected EOF\n", file, lineno);
+}
+
+/* Free a list of parameters.  */
+static void
+free_params(struct params *p)
+{
+	while (p != nil) {
+		struct params *next;
+
+		next = p->next;
+		xfree(p->name);
+		xfree(p->type);
+		xfree(p);
+		p = next;
+	}
+}
+
+/* Read a character, tracking lineno.  */
+static int
+getchar_update_lineno(void)
+{
+	int c;
+
+	c = xgetchar();
+	if (c == '\n')
+		++lineno;
+	return c;
+}
+
+/* Read a character, giving an error on EOF, tracking lineno.  */
+static int
+getchar_no_eof(void)
+{
+	int c;
+
+	c = getchar_update_lineno();
+	if (c == EOF)
+		bad_eof();
+	return c;
+}
+
+/* Read a character, skipping comments.  */
+static int
+getchar_skipping_comments(void)
+{
+	int c;
+
+	while (1) {
+		c = getchar_update_lineno();
+		if (c != '/')
+			return c;
+
+		c = xgetchar();
+		if (c == '/') {
+			do {
+				c = getchar_update_lineno();
+			} while (c != EOF && c != '\n');
+			return c;
+		} else if (c == '*') {
+			while (1) {
+				c = getchar_update_lineno();
+				if (c == EOF)
+					return EOF;
+				if (c == '*') {
+					do {
+						c = getchar_update_lineno();
+					} while (c == '*');
+					if (c == '/')
+						break;
+				}
+			}
+		} else {
+			xungetc();
+			return '/';
+		}
+	}
+}
+
+/*
+ * Read and return a token.  Tokens are string or character literals
+ * or else delimited by whitespace or by [(),{}].
+ * The latter are all returned as single characters.
+ */
+static char *
+read_token(void)
+{
+	int c, q;
+	char *buf;
+	unsigned int alc, off;
+	char* delims = "(),{}";
+
+	while (1) {
+		c = getchar_skipping_comments();
+		if (c == EOF)
+			return nil;
+		if (!xisspace(c))
+			break;
+	}
+	alc = 16;
+	buf = xmalloc(alc + 1);
+	off = 0;
+	if(c == '"' || c == '\'') {
+		q = c;
+		buf[off] = c;
+		++off;
+		while (1) {
+			if (off+2 >= alc) { // room for c and maybe next char
+				alc *= 2;
+				buf = xrealloc(buf, alc + 1);
+			}
+			c = getchar_no_eof();
+			buf[off] = c;
+			++off;
+			if(c == q)
+				break;
+			if(c == '\\') {
+				buf[off] = getchar_no_eof();
+				++off;
+			}
+		}
+	} else if (xstrrchr(delims, c) != nil) {
+		buf[off] = c;
+		++off;
+	} else {
+		while (1) {
+			if (off >= alc) {
+				alc *= 2;
+				buf = xrealloc(buf, alc + 1);
+			}
+			buf[off] = c;
+			++off;
+			c = getchar_skipping_comments();
+			if (c == EOF)
+				break;
+			if (xisspace(c) || xstrrchr(delims, c) != nil) {
+				if (c == '\n')
+					lineno--;
+				xungetc();
+				break;
+			}
+		}
+	}
+	buf[off] = '\0';
+	return buf;
+}
+
+/* Read a token, giving an error on EOF.  */
+static char *
+read_token_no_eof(void)
+{
+	char *token = read_token();
+	if (token == nil)
+		bad_eof();
+	return token;
+}
+
+/* Read the package clause, and return the package name.  */
+static char *
+read_package(void)
+{
+	char *token;
+
+	token = read_token_no_eof();
+	if (token == nil)
+		fatal("%s:%ud: no token\n", file, lineno);
+	if (!streq(token, "package")) {
+		fatal("%s:%ud: expected \"package\", got \"%s\"\n",
+			file, lineno, token);
+	}
+	return read_token_no_eof();
+}
+
+/* Read and copy preprocessor lines.  */
+static void
+read_preprocessor_lines(void)
+{
+	while (1) {
+		int c;
+
+		do {
+			c = getchar_skipping_comments();
+		} while (xisspace(c));
+		if (c != '#') {
+			xungetc();
+			break;
+		}
+		xputchar(c);
+		do {
+			c = getchar_update_lineno();
+			xputchar(c);
+		} while (c != '\n');
+	}
+}
+
+/*
+ * Read a type in Go syntax and return a type in C syntax.  We only
+ * permit basic types and pointers.
+ */
+static char *
+read_type(void)
+{
+	char *p, *op, *q;
+	int pointer_count;
+	unsigned int len;
+
+	p = read_token_no_eof();
+	if (*p != '*')
+		return p;
+	op = p;
+	pointer_count = 0;
+	while (*p == '*') {
+		++pointer_count;
+		++p;
+	}
+	len = xstrlen(p);
+	q = xmalloc(len + pointer_count + 1);
+	xmemmove(q, p, len);
+	while (pointer_count > 0) {
+		q[len] = '*';
+		++len;
+		--pointer_count;
+	}
+	q[len] = '\0';
+	xfree(op);
+	return q;
+}
+
+/* Return the size of the given type. */
+static int
+type_size(char *p)
+{
+	int i;
+
+	if(p[xstrlen(p)-1] == '*')
+		return type_table[Uintptr].size;
+
+	for(i=0; type_table[i].name; i++)
+		if(streq(type_table[i].name, p))
+			return type_table[i].size;
+	fatal("%s:%ud: unknown type %s\n", file, lineno, p);
+	return 0;
+}
+
+/*
+ * Read a list of parameters.  Each parameter is a name and a type.
+ * The list ends with a ')'.  We have already read the '('.
+ */
+static struct params *
+read_params(int *poffset)
+{
+	char *token;
+	struct params *ret, **pp, *p;
+	int offset, size, rnd;
+
+	ret = nil;
+	pp = &ret;
+	token = read_token_no_eof();
+	offset = 0;
+	if (!streq(token, ")")) {
+		while (1) {
+			p = xmalloc(sizeof(struct params));
+			p->name = token;
+			p->type = read_type();
+			p->next = nil;
+			*pp = p;
+			pp = &p->next;
+
+			size = type_size(p->type);
+			rnd = size;
+			if(rnd > structround)
+				rnd = structround;
+			if(offset%rnd)
+				offset += rnd - offset%rnd;
+			offset += size;
+
+			token = read_token_no_eof();
+			if (!streq(token, ","))
+				break;
+			token = read_token_no_eof();
+		}
+	}
+	if (!streq(token, ")")) {
+		fatal("%s:%ud: expected '('\n",
+			file, lineno);
+	}
+	if (poffset != nil)
+		*poffset = offset;
+	return ret;
+}
+
+/*
+ * Read a function header.  This reads up to and including the initial
+ * '{' character.  Returns 1 if it read a header, 0 at EOF.
+ */
+static int
+read_func_header(char **name, struct params **params, int *paramwid, struct params **rets)
+{
+	int lastline;
+	char *token;
+
+	lastline = -1;
+	while (1) {
+		token = read_token();
+		if (token == nil)
+			return 0;
+		if (streq(token, "func")) {
+			if(lastline != -1)
+				bwritef(output, "\n");
+			break;
+		}
+		if (lastline != lineno) {
+			if (lastline == lineno-1)
+				bwritef(output, "\n");
+			else
+				bwritef(output, "\n#line %d \"%s\"\n", lineno, file);
+			lastline = lineno;
+		}
+		bwritef(output, "%s ", token);
+	}
+
+	*name = read_token_no_eof();
+
+	token = read_token();
+	if (token == nil || !streq(token, "(")) {
+		fatal("%s:%ud: expected \"(\"\n",
+			file, lineno);
+	}
+	*params = read_params(paramwid);
+
+	token = read_token();
+	if (token == nil || !streq(token, "("))
+		*rets = nil;
+	else {
+		*rets = read_params(nil);
+		token = read_token();
+	}
+	if (token == nil || !streq(token, "{")) {
+		fatal("%s:%ud: expected \"{\"\n",
+			file, lineno);
+	}
+	return 1;
+}
+
+/* Write out parameters.  */
+static void
+write_params(struct params *params, int *first)
+{
+	struct params *p;
+
+	for (p = params; p != nil; p = p->next) {
+		if (*first)
+			*first = 0;
+		else
+			bwritef(output, ", ");
+		bwritef(output, "%s %s", p->type, p->name);
+	}
+}
+
+/* Write a 6g function header.  */
+static void
+write_6g_func_header(char *package, char *name, struct params *params,
+		     int paramwid, struct params *rets)
+{
+	int first, n;
+
+	bwritef(output, "void\n%s·%s(", package, name);
+	first = 1;
+	write_params(params, &first);
+
+	/* insert padding to align output struct */
+	if(rets != nil && paramwid%structround != 0) {
+		n = structround - paramwid%structround;
+		if(n & 1)
+			bwritef(output, ", uint8");
+		if(n & 2)
+			bwritef(output, ", uint16");
+		if(n & 4)
+			bwritef(output, ", uint32");
+	}
+
+	write_params(rets, &first);
+	bwritef(output, ")\n{\n");
+}
+
+/* Write a 6g function trailer.  */
+static void
+write_6g_func_trailer(struct params *rets)
+{
+	struct params *p;
+
+	for (p = rets; p != nil; p = p->next)
+		bwritef(output, "\tFLUSH(&%s);\n", p->name);
+	bwritef(output, "}\n");
+}
+
+/* Define the gcc function return type if necessary.  */
+static void
+define_gcc_return_type(char *package, char *name, struct params *rets)
+{
+	struct params *p;
+
+	if (rets == nil || rets->next == nil)
+		return;
+	bwritef(output, "struct %s_%s_ret {\n", package, name);
+	for (p = rets; p != nil; p = p->next)
+		bwritef(output, "  %s %s;\n", p->type, p->name);
+	bwritef(output, "};\n");
+}
+
+/* Write out the gcc function return type.  */
+static void
+write_gcc_return_type(char *package, char *name, struct params *rets)
+{
+	if (rets == nil)
+		bwritef(output, "void");
+	else if (rets->next == nil)
+		bwritef(output, "%s", rets->type);
+	else
+		bwritef(output, "struct %s_%s_ret", package, name);
+}
+
+/* Write out a gcc function header.  */
+static void
+write_gcc_func_header(char *package, char *name, struct params *params,
+		      struct params *rets)
+{
+	int first;
+	struct params *p;
+
+	define_gcc_return_type(package, name, rets);
+	write_gcc_return_type(package, name, rets);
+	bwritef(output, " %s_%s(", package, name);
+	first = 1;
+	write_params(params, &first);
+	bwritef(output, ") asm (\"%s.%s\");\n", package, name);
+	write_gcc_return_type(package, name, rets);
+	bwritef(output, " %s_%s(", package, name);
+	first = 1;
+	write_params(params, &first);
+	bwritef(output, ")\n{\n");
+	for (p = rets; p != nil; p = p->next)
+		bwritef(output, "  %s %s;\n", p->type, p->name);
+}
+
+/* Write out a gcc function trailer.  */
+static void
+write_gcc_func_trailer(char *package, char *name, struct params *rets)
+{
+	if (rets == nil)
+		;
+	else if (rets->next == nil)
+		bwritef(output, "return %s;\n", rets->name);
+	else {
+		struct params *p;
+
+		bwritef(output, "  {\n    struct %s_%s_ret __ret;\n", package, name);
+		for (p = rets; p != nil; p = p->next)
+			bwritef(output, "    __ret.%s = %s;\n", p->name, p->name);
+		bwritef(output, "    return __ret;\n  }\n");
+	}
+	bwritef(output, "}\n");
+}
+
+/* Write out a function header.  */
+static void
+write_func_header(char *package, char *name,
+		  struct params *params, int paramwid,
+		  struct params *rets)
+{
+	if (gcc)
+		write_gcc_func_header(package, name, params, rets);
+	else
+		write_6g_func_header(package, name, params, paramwid, rets);
+	bwritef(output, "#line %d \"%s\"\n", lineno, file);
+}
+
+/* Write out a function trailer.  */
+static void
+write_func_trailer(char *package, char *name,
+		   struct params *rets)
+{
+	if (gcc)
+		write_gcc_func_trailer(package, name, rets);
+	else
+		write_6g_func_trailer(rets);
+}
+
+/*
+ * Read and write the body of the function, ending in an unnested }
+ * (which is read but not written).
+ */
+static void
+copy_body(void)
+{
+	int nesting = 0;
+	while (1) {
+		int c;
+
+		c = getchar_no_eof();
+		if (c == '}' && nesting == 0)
+			return;
+		xputchar(c);
+		switch (c) {
+		default:
+			break;
+		case '{':
+			++nesting;
+			break;
+		case '}':
+			--nesting;
+			break;
+		case '/':
+			c = getchar_update_lineno();
+			xputchar(c);
+			if (c == '/') {
+				do {
+					c = getchar_no_eof();
+					xputchar(c);
+				} while (c != '\n');
+			} else if (c == '*') {
+				while (1) {
+					c = getchar_no_eof();
+					xputchar(c);
+					if (c == '*') {
+						do {
+							c = getchar_no_eof();
+							xputchar(c);
+						} while (c == '*');
+						if (c == '/')
+							break;
+					}
+				}
+			}
+			break;
+		case '"':
+		case '\'':
+			{
+				int delim = c;
+				do {
+					c = getchar_no_eof();
+					xputchar(c);
+					if (c == '\\') {
+						c = getchar_no_eof();
+						xputchar(c);
+						c = '\0';
+					}
+				} while (c != delim);
+			}
+			break;
+		}
+	}
+}
+
+/* Process the entire file.  */
+static void
+process_file(void)
+{
+	char *package, *name;
+	struct params *params, *rets;
+	int paramwid;
+
+	package = read_package();
+	read_preprocessor_lines();
+	while (read_func_header(&name, &params, &paramwid, &rets)) {
+		write_func_header(package, name, params, paramwid, rets);
+		copy_body();
+		write_func_trailer(package, name, rets);
+		xfree(name);
+		free_params(params);
+		free_params(rets);
+	}
+	xfree(package);
+}
+
+void
+goc2c(char *goc, char *c)
+{
+	Buf in, out;
+	
+	binit(&in);
+	binit(&out);
+	
+	file = goc;
+	readfile(&in, goc);
+
+	// TODO: set gcc=1 when using gcc
+
+	if(!gcc && streq(goarch, "amd64")) {
+		type_table[Uintptr].size = 8;
+		type_table[String].size = 16;
+		type_table[Slice].size = 8+4+4;
+		type_table[Eface].size = 8+8;
+		structround = 8;
+	}
+
+	bprintf(&out, "// auto generated by go tool dist\n\n");
+	input = bstr(&in);
+	output = &out;
+
+	process_file();
+	
+	writefile(&out, c);
+}

src/cmd/dist/main.c

@@ -4,14 +4,20 @@
 
 #include "a.h"
 
+int vflag;
+char *argv0;
+
 // cmdtab records the available commands.
 static struct {
 	char *name;
 	void (*f)(int, char**);
 } cmdtab[] = {
+	{"banner", cmdbanner},
 	{"bootstrap", cmdbootstrap},
+	{"clean", cmdclean},
 	{"env", cmdenv},
 	{"install", cmdinstall},
+	{"version", cmdversion},
 };
 
 // The OS-specific main calls into the portable code here.
@@ -20,12 +26,8 @@ xmain(int argc, char **argv)
 {
 	int i;
 
-	if(argc <= 1) {
+	if(argc <= 1)
-		xprintf("go tool dist commands:\n");
+		usage();
-		for(i=0; i<nelem(cmdtab); i++)
-			xprintf("\t%s\n", cmdtab[i].name);
-		xexit(1);
-	}
 	
 	for(i=0; i<nelem(cmdtab); i++) {
 		if(streq(cmdtab[i].name, argv[1])) {
@@ -34,5 +36,6 @@ xmain(int argc, char **argv)
 		}
 	}
 
-	fatal("unknown command %s", argv[1]);
+	xprintf("unknown command %s\n", argv[1]);
+	usage();
 }

src/cmd/dist/unix.c

@@ -40,6 +40,36 @@ bprintf(Buf *b, char *fmt, ...)
 	return bstr(b);
 }
 
+// bpathf is the same as bprintf (on windows it turns / into \ after the printf).
+// It returns a pointer to the NUL-terminated buffer contents.
+char*
+bpathf(Buf *b, char *fmt, ...)
+{
+	va_list arg;
+	char buf[4096];
+	
+	breset(b);
+	va_start(arg, fmt);
+	vsnprintf(buf, sizeof buf, fmt, arg);
+	va_end(arg);
+	bwritestr(b, buf);
+	return bstr(b);
+}
+
+// bwritef is like bprintf but does not reset the buffer
+// and does not return the NUL-terminated string.
+void
+bwritef(Buf *b, char *fmt, ...)
+{
+	va_list arg;
+	char buf[4096];
+	
+	va_start(arg, fmt);
+	vsnprintf(buf, sizeof buf, fmt, arg);
+	va_end(arg);
+	bwritestr(b, buf);
+}
+
 // breadfrom appends to b all the data that can be read from fd.
 static void
 breadfrom(Buf *b, int fd)
@@ -69,6 +99,8 @@ xgetenv(Buf *b, char *name)
 		bwritestr(b, p);
 }
 
+static void genrun(Buf *b, char *dir, int mode, Vec *argv, int bg);
+
 // run runs the command named by cmd.
 // If b is not nil, run replaces b with the output of the command.
 // If dir is not nil, run runs the command in that directory.
@@ -92,15 +124,43 @@ run(Buf *b, char *dir, int mode, char *cmd, ...)
 	vfree(&argv);
 }
 
-
 // runv is like run but takes a vector.
 void
 runv(Buf *b, char *dir, int mode, Vec *argv)
 {
-	int i, p[2], pid, status;
+	genrun(b, dir, mode, argv, 1);
+}
+
+// bgrunv is like run but runs the command in the background.
+// bgwait waits for pending bgrunv to finish.
+void
+bgrunv(char *dir, int mode, Vec *argv)
+{
+	genrun(nil, dir, mode, argv, 0);
+}
+
+#define MAXBG 4 /* maximum number of jobs to run at once */
+
+static struct {
+	int pid;
+	int mode;
+	char *cmd;
+} bg[MAXBG];
+static int nbg;
+
+static void bgwait1(void);
+
+// genrun is the generic run implementation.
+static void
+genrun(Buf *b, char *dir, int mode, Vec *argv, int wait)
+{
+	int i, p[2], pid;
 	Buf cmd;
 	char *q;
 
+	while(nbg >= nelem(bg))
+		bgwait1();
+
 	// Generate a copy of the command to show in a log.
 	// Substitute $WORK for the work directory.
 	binit(&cmd);
@@ -114,8 +174,8 @@ runv(Buf *b, char *dir, int mode, Vec *argv)
 		}
 		bwritestr(&cmd, q);
 	}
-	printf("%s\n", bstr(&cmd));
+	if(vflag > 1)
-	bfree(&cmd);
+		xprintf("%s\n", bstr(&cmd));
 
 	if(b != nil) {
 		breset(b);
@@ -143,6 +203,7 @@ runv(Buf *b, char *dir, int mode, Vec *argv)
 		}
 		vadd(argv, nil);
 		execvp(argv->p[0], argv->p);
+		fprintf(stderr, "%s\n", bstr(&cmd));
 		fprintf(stderr, "exec %s: %s\n", argv->p[0], strerror(errno));
 		_exit(1);
 	}
@@ -151,18 +212,55 @@ runv(Buf *b, char *dir, int mode, Vec *argv)
 		breadfrom(b, p[0]);
 		close(p[0]);
 	}
-wait:
+
+	if(nbg < 0)
+		fatal("bad bookkeeping");
+	bg[nbg].pid = pid;
+	bg[nbg].mode = mode;
+	bg[nbg].cmd = btake(&cmd);
+	nbg++;
+	
+	if(wait)
+		bgwait();
+
+	bfree(&cmd);
+}
+
+// bgwait1 waits for a single background job.
+static void
+bgwait1(void)
+{
+	int i, pid, status, mode;
+	char *cmd;
+
 	errno = 0;
-	if(waitpid(pid, &status, 0) != pid) {
+	while((pid = wait(&status)) < 0) {
-		if(errno == EINTR)
+		if(errno != EINTR)
-			goto wait;
+			fatal("waitpid: %s", strerror(errno));
-		fatal("waitpid: %s", strerror(errno));
 	}
-	if(mode==CheckExit && (!WIFEXITED(status) || WEXITSTATUS(status) != 0)) {
+	for(i=0; i<nbg; i++)
-		if(b != nil)
+		if(bg[i].pid == pid)
-			fwrite(b->p, b->len, 1, stderr);
+			goto ok;
-		fatal("%s failed", argv->p[0]);
+	fatal("waitpid: unexpected pid");
+
+ok:
+	cmd = bg[i].cmd;
+	mode = bg[i].mode;
+	bg[i].pid = 0;
+	bg[i] = bg[--nbg];
+	
+	if(mode == CheckExit && (!WIFEXITED(status) || WEXITSTATUS(status) != 0)) {
+		fatal("FAILED: %s", cmd);
 	}
+	xfree(cmd);
+}
+
+// bgwait waits for all the background jobs.
+void
+bgwait(void)
+{
+	while(nbg > 0)
+		bgwait1();
 }
 
 // xgetwd replaces b with the current directory.
@@ -288,6 +386,8 @@ xmkdirall(char *p)
 void
 xremove(char *p)
 {
+	if(vflag > 1)
+		xprintf("rm %s\n", p);
 	unlink(p);
 }
 
@@ -308,8 +408,12 @@ xremoveall(char *p)
 			bprintf(&b, "%s/%s", p, dir.p[i]);
 			xremoveall(bstr(&b));
 		}
+		if(vflag > 1)
+			xprintf("rm %s\n", p);
 		rmdir(p);
 	} else {
+		if(vflag > 1)
+			xprintf("rm %s\n", p);
 		unlink(p);
 	}
 	
@@ -526,9 +630,9 @@ main(int argc, char **argv)
 
 	binit(&b);
 	p = argv[0];
-	if(hassuffix(p, "bin/go-tool/dist")) {
+	if(hassuffix(p, "bin/tool/dist")) {
 		default_goroot = xstrdup(p);
-		default_goroot[strlen(p)-strlen("bin/go-tool/dist")] = '\0';
+		default_goroot[strlen(p)-strlen("bin/tool/dist")] = '\0';
 	}
 	
 	slash = "/";