Stowage/go
2
0
Fork 0
mirror of https://github.com/golang/go.git synced 2025-12-08 06:10:04 +00:00
Code Issues Projects Releases Packages Wiki Activity

encoding/json: fix truncated Token error regression in goexperiment.jsonv2

Browse source 
The jsontext.Decoder.ReadToken method reports a non-EOF error,
if the token stream is truncated and does not form a valid JSON value.
In contrast, the v1 json.Decoder.Token method would report EOF
so long as the input was a prefix of some valid JSON value.
Modify json.Decoder.Token to preserve historical behavior.

This only modifies code that is compiled in under goexperiment.jsonv2.

Updates #69449
Fixes #74750

Change-Id: Ifd281c46f118f0e748076013fefc7659f77c56ed
Reviewed-on: https://go-review.googlesource.com/c/go/+/689516
Reviewed-by: Damien Neil <dneil@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Auto-Submit: Joseph Tsai <joetsai@digital-static.net>
Reviewed-by: Michael Knyszek <mknyszek@google.com>
This commit is contained in:
Joe Tsai 2025-07-24 17:10:54 -07:00 • committed by Gopher Robot
parent ebdbfccd98
commit c76c3abc54
3 changed files with 81 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

35
src/encoding/json/stream_test.go
View file

@ -522,3 +522,38 @@ func TestHTTPDecoding(t *testing.T) {
t.Errorf("Decode error:\n\tgot: %v\n\twant: io.EOF", err)
}
}
func TestTokenTruncation(t *testing.T) {
tests := []struct {
in string
err error
}{
{in: ``, err: io.EOF},
{in: `{`, err: io.EOF},
{in: `{"`, err: io.ErrUnexpectedEOF},
{in: `{"k"`, err: io.EOF},
{in: `{"k":`, err: io.EOF},
{in: `{"k",`, err: &SyntaxError{"invalid character ',' after object key", int64(len(`{"k"`))}},
{in: `{"k"}`, err: &SyntaxError{"invalid character '}' after object key", int64(len(`{"k"`))}},
{in: ` [0`, err: io.EOF},
{in: `[0.`, err: io.ErrUnexpectedEOF},
{in: `[0. `, err: &SyntaxError{"invalid character ' ' after decimal point in numeric literal", int64(len(`[0.`))}},
{in: `[0,`, err: io.EOF},
{in: `[0:`, err: &SyntaxError{"invalid character ':' after array element", int64(len(`[0`))}},
{in: `n`, err: io.ErrUnexpectedEOF},
{in: `nul`, err: io.ErrUnexpectedEOF},
{in: `fal `, err: &SyntaxError{"invalid character ' ' in literal false (expecting 's')", int64(len(`fal `))}},
{in: `false`, err: io.EOF},
}
for _, tt := range tests {
d := NewDecoder(strings.NewReader(tt.in))
for i := 0; true; i++ {
if _, err := d.Token(); err != nil {
if !reflect.DeepEqual(err, tt.err) {
t.Errorf("`%s`: %d.Token error = %#v, want %v", tt.in, i, err, tt.err)
}
break
}
}
}
}

11
src/encoding/json/v2_stream.go
View file

@ -8,6 +8,7 @@ package json
import (
"bytes"
"errors"
"io"
"encoding/json/jsontext"
@ -193,6 +194,16 @@ func (d Delim) String() string {
func (dec *Decoder) Token() (Token, error) {
tok, err := dec.dec.ReadToken()
if err != nil {
// Historically, v1 would report just [io.EOF] if
// the stream is a prefix of a valid JSON value.
// It reports an unwrapped [io.ErrUnexpectedEOF] if
// truncated within a JSON token such as a literal, number, or string.
if errors.Is(err, io.ErrUnexpectedEOF) {
if len(bytes.Trim(dec.dec.UnreadBuffer(), " \r\n\t,:")) == 0 {
return nil, io.EOF
}
return nil, io.ErrUnexpectedEOF
}
return nil, transformSyntacticError(err)
}
switch k := tok.Kind(); k {

35
src/encoding/json/v2_stream_test.go
View file

@ -502,3 +502,38 @@ func TestHTTPDecoding(t *testing.T) {
t.Errorf("Decode error:\n\tgot: %v\n\twant: io.EOF", err)
}
}
func TestTokenTruncation(t *testing.T) {
tests := []struct {
in string
err error
}{
{in: ``, err: io.EOF},
{in: `{`, err: io.EOF},
{in: `{"`, err: io.ErrUnexpectedEOF},
{in: `{"k"`, err: io.EOF},
{in: `{"k":`, err: io.EOF},
{in: `{"k",`, err: &SyntaxError{"invalid character ',' after object key", int64(len(`{"k"`))}},
{in: `{"k"}`, err: &SyntaxError{"invalid character '}' after object key", int64(len(`{"k"`))}},
{in: ` [0`, err: io.EOF},
{in: `[0.`, err: io.ErrUnexpectedEOF},
{in: `[0. `, err: &SyntaxError{"invalid character ' ' in numeric literal", int64(len(`[0.`))}},
{in: `[0,`, err: io.EOF},
{in: `[0:`, err: &SyntaxError{"invalid character ':' after array element", int64(len(`[0`))}},
{in: `n`, err: io.ErrUnexpectedEOF},
{in: `nul`, err: io.ErrUnexpectedEOF},
{in: `fal `, err: &SyntaxError{"invalid character ' ' in literal false (expecting 's')", int64(len(`fal`))}},
{in: `false`, err: io.EOF},
}
for _, tt := range tests {
d := NewDecoder(strings.NewReader(tt.in))
for i := 0; true; i++ {
if _, err := d.Token(); err != nil {
if !reflect.DeepEqual(err, tt.err) {
t.Errorf("`%s`: %d.Token error = %#v, want %v", tt.in, i, err, tt.err)
}
break
}
}
}
}