cmd/compile: make non-concurrent compiles deterministic again

Spreading function compilation across multiple goroutines results in
non-deterministic output. This is how cmd/compile has historically
behaved for concurrent builds, but is troublesome for non-concurrent
builds, particularly because it interferes with "toolstash -cmp".

I spent some time trying to think of a simple, unified algorithm that
can concurrently schedule work but gracefully degrades to a
deterministic build for single-worker builds, but I couldn't come up
with any. The simplest idea I found was to simply abstract away the
operation of scheduling work so that we can have alternative
deterministic vs concurrent modes.

Change-Id: I08afa00527ce1844432412f4f8553781c4e323df
Reviewed-on: https://go-review.googlesource.com/c/go/+/318229
Trust: Matthew Dempsky <mdempsky@google.com>
Run-TryBot: Matthew Dempsky <mdempsky@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Cuong Manh Le <cuong.manhle.vn@gmail.com>

src/cmd/compile/internal/gc/compile.go

@@ -119,38 +119,51 @@ func compileFunctions() {
 		})
 	}
 
-	// We queue up a goroutine per function that needs to be
+	// By default, we perform work right away on the current goroutine
-	// compiled, but require them to grab an available worker ID
+	// as the solo worker.
-	// before doing any substantial work to limit parallelism.
+	queue := func(work func(int)) {
-	workerIDs := make(chan int, base.Flag.LowerC)
+		work(0)
-	for i := 0; i < base.Flag.LowerC; i++ {
+	}
+
+	if nWorkers := base.Flag.LowerC; nWorkers > 1 {
+		// For concurrent builds, we create a goroutine per task, but
+		// require them to hold a unique worker ID while performing work
+		// to limit parallelism.
+		workerIDs := make(chan int, nWorkers)
+		for i := 0; i < nWorkers; i++ {
 			workerIDs <- i
 		}
 
-	var wg sync.WaitGroup
+		queue = func(work func(int)) {
-	var asyncCompile func(*ir.Func)
-	asyncCompile = func(fn *ir.Func) {
-		wg.Add(1)
 			go func() {
 				worker := <-workerIDs
-			ssagen.Compile(fn, worker)
+				work(worker)
 				workerIDs <- worker
-
-			// Done compiling fn. Schedule it's closures for compilation.
-			for _, closure := range fn.Closures {
-				asyncCompile(closure)
-			}
-			wg.Done()
 			}()
 		}
+	}
+
+	var wg sync.WaitGroup
+	var compile func([]*ir.Func)
+	compile = func(fns []*ir.Func) {
+		wg.Add(len(fns))
+		for _, fn := range fns {
+			fn := fn
+			queue(func(worker int) {
+				ssagen.Compile(fn, worker)
+				compile(fn.Closures)
+				wg.Done()
+			})
+		}
+	}
 
 	types.CalcSizeDisabled = true // not safe to calculate sizes concurrently
 	base.Ctxt.InParallel = true
-	for _, fn := range compilequeue {
+
-		asyncCompile(fn)
+	compile(compilequeue)
-	}
 	compilequeue = nil
 	wg.Wait()
+
 	base.Ctxt.InParallel = false
 	types.CalcSizeDisabled = false
 }