encoding/json: limit max nesting depth

Limit the maximum nesting depth when parsing to protect against stack
overflow, permitted by https://tools.ietf.org/html/rfc7159#section-9

A nesting depth limit of 10,000 was chosen to be a conservative
balance between avoiding stack overflow and avoiding impacting
legitimate JSON documents.

10,000 is less than 1% of the experimental stack depth limit
with the default stack size:
* On 64-bit systems, the default stack limit is 1GB,
  which allows ~2,800,000 frames of recursive parsing
* On 32-bit systems, the default stack limit is 250MB,
  which allows ~1,100,000 frames of recursive parsing

Fixes #31789

Change-Id: I4f5a90e89dcb4ab1a957ad9d02e1fa0efafaccf6
Reviewed-on: https://go-review.googlesource.com/c/go/+/199837
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>

src/encoding/json/decode_test.go

@@ -2431,3 +2431,99 @@ func TestUnmarshalMapWithTextUnmarshalerStringKey(t *testing.T) {
 		t.Errorf(`Key "foo" is not existed in map: %v`, p)
 	}
 }
+
+func TestUnmarshalMaxDepth(t *testing.T) {
+	testcases := []struct {
+		name        string
+		data        string
+		errMaxDepth bool
+	}{
+		{
+			name:        "ArrayUnderMaxNestingDepth",
+			data:        `{"a":` + strings.Repeat(`[`, 10000-1) + strings.Repeat(`]`, 10000-1) + `}`,
+			errMaxDepth: false,
+		},
+		{
+			name:        "ArrayOverMaxNestingDepth",
+			data:        `{"a":` + strings.Repeat(`[`, 10000) + strings.Repeat(`]`, 10000) + `}`,
+			errMaxDepth: true,
+		},
+		{
+			name:        "ArrayOverStackDepth",
+			data:        `{"a":` + strings.Repeat(`[`, 3000000) + strings.Repeat(`]`, 3000000) + `}`,
+			errMaxDepth: true,
+		},
+		{
+			name:        "ObjectUnderMaxNestingDepth",
+			data:        `{"a":` + strings.Repeat(`{"a":`, 10000-1) + `0` + strings.Repeat(`}`, 10000-1) + `}`,
+			errMaxDepth: false,
+		},
+		{
+			name:        "ObjectOverMaxNestingDepth",
+			data:        `{"a":` + strings.Repeat(`{"a":`, 10000) + `0` + strings.Repeat(`}`, 10000) + `}`,
+			errMaxDepth: true,
+		},
+		{
+			name:        "ObjectOverStackDepth",
+			data:        `{"a":` + strings.Repeat(`{"a":`, 3000000) + `0` + strings.Repeat(`}`, 3000000) + `}`,
+			errMaxDepth: true,
+		},
+	}
+
+	targets := []struct {
+		name     string
+		newValue func() interface{}
+	}{
+		{
+			name: "unstructured",
+			newValue: func() interface{} {
+				var v interface{}
+				return &v
+			},
+		},
+		{
+			name: "typed named field",
+			newValue: func() interface{} {
+				v := struct {
+					A interface{} `json:"a"`
+				}{}
+				return &v
+			},
+		},
+		{
+			name: "typed missing field",
+			newValue: func() interface{} {
+				v := struct {
+					B interface{} `json:"b"`
+				}{}
+				return &v
+			},
+		},
+		{
+			name: "custom unmarshaler",
+			newValue: func() interface{} {
+				v := unmarshaler{}
+				return &v
+			},
+		},
+	}
+
+	for _, tc := range testcases {
+		for _, target := range targets {
+			t.Run(target.name+"-"+tc.name, func(t *testing.T) {
+				err := Unmarshal([]byte(tc.data), target.newValue())
+				if !tc.errMaxDepth {
+					if err != nil {
+						t.Errorf("unexpected error: %v", err)
+					}
+				} else {
+					if err == nil {
+						t.Errorf("expected error containing 'exceeded max depth', got none")
+					} else if !strings.Contains(err.Error(), "exceeded max depth") {
+						t.Errorf("expected error containing 'exceeded max depth', got: %v", err)
+					}
+				}
+			})
+		}
+	}
+}