cmd/compile: don't crash when exporting self-recursive interfaces

For #16369.

Change-Id: I4c9f5a66b95558adcc1bcface164b9b2b4382d2f
Reviewed-on: https://go-review.googlesource.com/24979
Reviewed-by: Alan Donovan <adonovan@google.com>

src/cmd/compile/internal/gc/bexport.go

@@ -197,6 +197,9 @@ type exporter struct {
 	written int // bytes written
 	indent  int // for p.trace
 	trace   bool
+
+	// work-around for issue #16369 only
+	nesting int // amount of "nesting" of interface types
 }
 
 // export writes the exportlist for localpkg to out and returns the number of bytes written.
@@ -790,11 +793,39 @@ func (p *exporter) typ(t *Type) {
 
 	case TINTER:
 		p.tag(interfaceTag)
-
 		// gc doesn't separate between embedded interfaces
 		// and methods declared explicitly with an interface
 		p.int(0) // no embedded interfaces
+
+		// Because the compiler flattens interfaces containing
+		// embedded interfaces, it is possible to create interface
+		// types that recur through an unnamed type.
+		// If trackAllTypes is disabled, such recursion is not
+		// detected, leading to a stack overflow during export
+		// (issue #16369).
+		// As a crude work-around we terminate deep recursion
+		// through interface types with an empty interface and
+		// report an error.
+		// This will catch endless recursion, but is unlikely
+		// to trigger for valid, deeply nested types given the
+		// high threshold.
+		// It would be ok to continue without reporting an error
+		// since the export format is valid. But a subsequent
+		// import would import an incorrect type. The textual
+		// exporter does not report an error but importing the
+		// resulting package will lead to a syntax error during
+		// import.
+		// TODO(gri) remove this once we have a permanent fix
+		// for the issue.
+		if p.nesting > 100 {
+			p.int(0) // 0 methods to indicate empty interface
+			yyerrorl(t.Lineno, "cannot export unnamed recursive interface")
+			break
+		}
+
+		p.nesting++
 		p.methodList(t)
+		p.nesting--
 
 	case TMAP:
 		p.tag(mapTag)

src/go/types/stdlib_test.go

@@ -156,6 +156,7 @@ func TestStdFixed(t *testing.T) {
 		"issue7746.go",  // large constants - consumes too much memory
 		"issue11362.go", // canonical import path check
 		"issue15002.go", // uses Mmap; testTestDir should consult build tags
+		"issue16369.go", // go/types handles this correctly - not an issue
 	)
 }
 

test/fixedbugs/issue16369.go

@@ -0,0 +1,13 @@
+// errorcheck
+
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+type T interface {
+	M(interface {
+		T
+	}) // ERROR "cannot export unnamed recursive interface"
+}