encoding/json: fix truncated Token error regression in goexperiment.jsonv2

The jsontext.Decoder.ReadToken method reports a non-EOF error,
if the token stream is truncated and does not form a valid JSON value.
In contrast, the v1 json.Decoder.Token method would report EOF
so long as the input was a prefix of some valid JSON value.
Modify json.Decoder.Token to preserve historical behavior.

This only modifies code that is compiled in under goexperiment.jsonv2.

Updates #69449
Fixes #74750

Change-Id: Ifd281c46f118f0e748076013fefc7659f77c56ed
Reviewed-on: https://go-review.googlesource.com/c/go/+/689516
Reviewed-by: Damien Neil <dneil@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Auto-Submit: Joseph Tsai <joetsai@digital-static.net>
Reviewed-by: Michael Knyszek <mknyszek@google.com>

src/encoding/json/stream_test.go

@@ -522,3 +522,38 @@ func TestHTTPDecoding(t *testing.T) {
 		t.Errorf("Decode error:\n\tgot:  %v\n\twant: io.EOF", err)
 	}
 }
+
+func TestTokenTruncation(t *testing.T) {
+	tests := []struct {
+		in  string
+		err error
+	}{
+		{in: ``, err: io.EOF},
+		{in: `{`, err: io.EOF},
+		{in: `{"`, err: io.ErrUnexpectedEOF},
+		{in: `{"k"`, err: io.EOF},
+		{in: `{"k":`, err: io.EOF},
+		{in: `{"k",`, err: &SyntaxError{"invalid character ',' after object key", int64(len(`{"k"`))}},
+		{in: `{"k"}`, err: &SyntaxError{"invalid character '}' after object key", int64(len(`{"k"`))}},
+		{in: ` [0`, err: io.EOF},
+		{in: `[0.`, err: io.ErrUnexpectedEOF},
+		{in: `[0. `, err: &SyntaxError{"invalid character ' ' after decimal point in numeric literal", int64(len(`[0.`))}},
+		{in: `[0,`, err: io.EOF},
+		{in: `[0:`, err: &SyntaxError{"invalid character ':' after array element", int64(len(`[0`))}},
+		{in: `n`, err: io.ErrUnexpectedEOF},
+		{in: `nul`, err: io.ErrUnexpectedEOF},
+		{in: `fal `, err: &SyntaxError{"invalid character ' ' in literal false (expecting 's')", int64(len(`fal `))}},
+		{in: `false`, err: io.EOF},
+	}
+	for _, tt := range tests {
+		d := NewDecoder(strings.NewReader(tt.in))
+		for i := 0; true; i++ {
+			if _, err := d.Token(); err != nil {
+				if !reflect.DeepEqual(err, tt.err) {
+					t.Errorf("`%s`: %d.Token error = %#v, want %v", tt.in, i, err, tt.err)
+				}
+				break
+			}
+		}
+	}
+}

src/encoding/json/v2_stream.go

@@ -8,6 +8,7 @@ package json
 
 import (
 	"bytes"
+	"errors"
 	"io"
 
 	"encoding/json/jsontext"
@@ -193,6 +194,16 @@ func (d Delim) String() string {
 func (dec *Decoder) Token() (Token, error) {
 	tok, err := dec.dec.ReadToken()
 	if err != nil {
+		// Historically, v1 would report just [io.EOF] if
+		// the stream is a prefix of a valid JSON value.
+		// It reports an unwrapped [io.ErrUnexpectedEOF] if
+		// truncated within a JSON token such as a literal, number, or string.
+		if errors.Is(err, io.ErrUnexpectedEOF) {
+			if len(bytes.Trim(dec.dec.UnreadBuffer(), " \r\n\t,:")) == 0 {
+				return nil, io.EOF
+			}
+			return nil, io.ErrUnexpectedEOF
+		}
 		return nil, transformSyntacticError(err)
 	}
 	switch k := tok.Kind(); k {

src/encoding/json/v2_stream_test.go

@@ -502,3 +502,38 @@ func TestHTTPDecoding(t *testing.T) {
 		t.Errorf("Decode error:\n\tgot:  %v\n\twant: io.EOF", err)
 	}
 }
+
+func TestTokenTruncation(t *testing.T) {
+	tests := []struct {
+		in  string
+		err error
+	}{
+		{in: ``, err: io.EOF},
+		{in: `{`, err: io.EOF},
+		{in: `{"`, err: io.ErrUnexpectedEOF},
+		{in: `{"k"`, err: io.EOF},
+		{in: `{"k":`, err: io.EOF},
+		{in: `{"k",`, err: &SyntaxError{"invalid character ',' after object key", int64(len(`{"k"`))}},
+		{in: `{"k"}`, err: &SyntaxError{"invalid character '}' after object key", int64(len(`{"k"`))}},
+		{in: ` [0`, err: io.EOF},
+		{in: `[0.`, err: io.ErrUnexpectedEOF},
+		{in: `[0. `, err: &SyntaxError{"invalid character ' ' in numeric literal", int64(len(`[0.`))}},
+		{in: `[0,`, err: io.EOF},
+		{in: `[0:`, err: &SyntaxError{"invalid character ':' after array element", int64(len(`[0`))}},
+		{in: `n`, err: io.ErrUnexpectedEOF},
+		{in: `nul`, err: io.ErrUnexpectedEOF},
+		{in: `fal `, err: &SyntaxError{"invalid character ' ' in literal false (expecting 's')", int64(len(`fal`))}},
+		{in: `false`, err: io.EOF},
+	}
+	for _, tt := range tests {
+		d := NewDecoder(strings.NewReader(tt.in))
+		for i := 0; true; i++ {
+			if _, err := d.Token(); err != nil {
+				if !reflect.DeepEqual(err, tt.err) {
+					t.Errorf("`%s`: %d.Token error = %#v, want %v", tt.in, i, err, tt.err)
+				}
+				break
+			}
+		}
+	}
+}