2010-04-21 16:40:53 -07:00
// Copyright 2010 The Go Authors. All rights reserved.
2009-11-30 13:55:09 -08:00
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package json
import (
2010-04-21 16:40:53 -07:00
"bytes"
2013-08-14 14:56:07 -04:00
"encoding"
2016-03-08 12:41:35 -08:00
"errors"
2011-12-15 10:02:47 -08:00
"fmt"
2012-09-10 23:31:40 -04:00
"image"
2016-04-13 16:51:25 -07:00
"math"
2016-10-12 16:54:02 -04:00
"math/big"
2015-07-14 19:31:44 -04:00
"net"
2009-12-15 15:35:38 -08:00
"reflect"
2016-04-13 16:51:25 -07:00
"strconv"
2010-04-21 16:40:53 -07:00
"strings"
2009-12-15 15:35:38 -08:00
"testing"
2013-02-14 14:46:15 -05:00
"time"
2009-11-30 13:55:09 -08:00
)
2010-05-11 14:38:55 -07:00
type T struct {
X string
Y int
2011-09-15 08:09:43 +10:00
Z int ` json:"-" `
2010-05-11 14:38:55 -07:00
}
2012-05-01 11:37:44 +10:00
type U struct {
Alphabet string ` json:"alpha" `
}
2012-06-25 17:36:09 -04:00
type V struct {
2021-12-01 12:15:45 -05:00
F1 any
2012-06-25 17:36:09 -04:00
F2 int32
F3 Number
2016-01-18 16:26:05 +01:00
F4 * VOuter
}
type VOuter struct {
V V
2012-06-25 17:36:09 -04:00
}
2018-09-03 11:20:23 +01:00
type W struct {
S SS
}
2018-10-28 01:44:09 +07:00
type P struct {
PP PP
}
type PP struct {
encoding/json: fix performance regression in the decoder
In golang.org/cl/145218, a feature was added where the JSON decoder
would keep track of the entire path to a field when reporting an
UnmarshalTypeError.
However, we all failed to check if this affected the benchmarks - myself
included, as a reviewer. Below are the numbers comparing the CL's parent
with itself, once it was merged:
name old time/op new time/op delta
CodeDecoder-8 12.9ms ± 1% 28.2ms ± 2% +119.33% (p=0.002 n=6+6)
name old speed new speed delta
CodeDecoder-8 151MB/s ± 1% 69MB/s ± 3% -54.40% (p=0.002 n=6+6)
name old alloc/op new alloc/op delta
CodeDecoder-8 2.74MB ± 0% 109.39MB ± 0% +3891.83% (p=0.002 n=6+6)
name old allocs/op new allocs/op delta
CodeDecoder-8 77.5k ± 0% 168.5k ± 0% +117.30% (p=0.004 n=6+5)
The reason why the decoder got twice as slow is because it now allocated
~40x as many objects, which puts a lot of pressure on the garbage
collector.
The reason is that the CL concatenated strings every time a nested field
was decoded. In other words, practically every field generated garbage
when decoded. This is hugely wasteful, especially considering that the
vast majority of JSON decoding inputs won't return UnmarshalTypeError.
Instead, use a stack of fields, and make sure to always use the same
backing array, to ensure we only need to grow the slice to the maximum
depth once.
The original CL also introduced a bug. The field stack string wasn't
reset to its original state when reaching "d.opcode == scanEndObject",
so the last field in a decoded struct could leak. For example, an added
test decodes a list of structs, and encoding/json before this CL would
fail:
got: cannot unmarshal string into Go struct field T.Ts.Y.Y.Y of type int
want: cannot unmarshal string into Go struct field T.Ts.Y of type int
To fix that, simply reset the stack after decoding every field, even if
it's the last.
Below is the original performance versus this CL. There's a tiny
performance hit, probably due to the append for every decoded field, but
at least we're back to the usual ~150MB/s.
name old time/op new time/op delta
CodeDecoder-8 12.9ms ± 1% 13.0ms ± 1% +1.25% (p=0.009 n=6+6)
name old speed new speed delta
CodeDecoder-8 151MB/s ± 1% 149MB/s ± 1% -1.24% (p=0.009 n=6+6)
name old alloc/op new alloc/op delta
CodeDecoder-8 2.74MB ± 0% 2.74MB ± 0% +0.00% (p=0.002 n=6+6)
name old allocs/op new allocs/op delta
CodeDecoder-8 77.5k ± 0% 77.5k ± 0% +0.00% (p=0.002 n=6+6)
Finally, make all of these benchmarks report allocs by default. The
decoder ones are pretty sensitive to generated garbage, so ReportAllocs
would have made the performance regression more obvious.
Change-Id: I67b50f86b2e72f55539429450c67bfb1a9464b67
Reviewed-on: https://go-review.googlesource.com/c/go/+/167978
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
2019-03-17 22:45:30 +00:00
T T
Ts [ ] T
2018-10-28 01:44:09 +07:00
}
2018-09-03 11:20:23 +01:00
type SS string
func ( * SS ) UnmarshalJSON ( data [ ] byte ) error {
2023-07-31 15:18:12 -07:00
return & UnmarshalTypeError { Value : "number" , Type : reflect . TypeFor [ SS ] ( ) }
2018-09-03 11:20:23 +01:00
}
2013-04-09 15:00:21 -07:00
// ifaceNumAsFloat64/ifaceNumAsNumber are used to test unmarshaling with and
2012-06-25 17:36:09 -04:00
// without UseNumber
2021-12-01 12:15:45 -05:00
var ifaceNumAsFloat64 = map [ string ] any {
2012-06-25 17:36:09 -04:00
"k1" : float64 ( 1 ) ,
"k2" : "s" ,
2021-12-01 12:15:45 -05:00
"k3" : [ ] any { float64 ( 1 ) , float64 ( 2.0 ) , float64 ( 3e-3 ) } ,
"k4" : map [ string ] any { "kk1" : "s" , "kk2" : float64 ( 2 ) } ,
2012-06-25 17:36:09 -04:00
}
2021-12-01 12:15:45 -05:00
var ifaceNumAsNumber = map [ string ] any {
2012-06-25 17:36:09 -04:00
"k1" : Number ( "1" ) ,
"k2" : "s" ,
2021-12-01 12:15:45 -05:00
"k3" : [ ] any { Number ( "1" ) , Number ( "2.0" ) , Number ( "3e-3" ) } ,
"k4" : map [ string ] any { "kk1" : "s" , "kk2" : Number ( "2" ) } ,
2012-06-25 17:36:09 -04:00
}
2010-09-28 14:40:23 -04:00
type tx struct {
x int
}
2016-04-13 16:51:25 -07:00
type u8 uint8
2010-11-08 15:33:00 -08:00
// A type that can unmarshal itself.
type unmarshaler struct {
T bool
}
2011-11-01 22:04:37 -04:00
func ( u * unmarshaler ) UnmarshalJSON ( b [ ] byte ) error {
2013-08-14 14:56:07 -04:00
* u = unmarshaler { true } // All we need to see that UnmarshalJSON is called.
2010-11-08 15:33:00 -08:00
return nil
}
2011-08-10 09:26:51 -04:00
type ustruct struct {
M unmarshaler
}
2013-08-14 14:56:07 -04:00
type unmarshalerText struct {
2016-03-08 12:41:35 -08:00
A , B string
2013-08-14 14:56:07 -04:00
}
// needed for re-marshaling tests
2016-03-08 12:41:35 -08:00
func ( u unmarshalerText ) MarshalText ( ) ( [ ] byte , error ) {
return [ ] byte ( u . A + ":" + u . B ) , nil
2013-08-14 14:56:07 -04:00
}
func ( u * unmarshalerText ) UnmarshalText ( b [ ] byte ) error {
2017-09-21 19:23:51 +02:00
pos := bytes . IndexByte ( b , ':' )
2016-03-08 12:41:35 -08:00
if pos == - 1 {
return errors . New ( "missing separator" )
}
u . A , u . B = string ( b [ : pos ] ) , string ( b [ pos + 1 : ] )
2013-08-14 14:56:07 -04:00
return nil
}
var _ encoding . TextUnmarshaler = ( * unmarshalerText ) ( nil )
type ustructText struct {
M unmarshalerText
}
2016-04-13 16:51:25 -07:00
// u8marshal is an integer type that can marshal/unmarshal itself.
type u8marshal uint8
func ( u8 u8marshal ) MarshalText ( ) ( [ ] byte , error ) {
return [ ] byte ( fmt . Sprintf ( "u%d" , u8 ) ) , nil
}
var errMissingU8Prefix = errors . New ( "missing 'u' prefix" )
func ( u8 * u8marshal ) UnmarshalText ( b [ ] byte ) error {
if ! bytes . HasPrefix ( b , [ ] byte { 'u' } ) {
return errMissingU8Prefix
}
n , err := strconv . Atoi ( string ( b [ 1 : ] ) )
if err != nil {
return err
}
* u8 = u8marshal ( n )
return nil
}
var _ encoding . TextUnmarshaler = ( * u8marshal ) ( nil )
2010-11-08 15:33:00 -08:00
var (
umtrue = unmarshaler { true }
2011-09-06 16:04:55 -07:00
umslice = [ ] unmarshaler { { true } }
2011-08-10 09:26:51 -04:00
umstruct = ustruct { unmarshaler { true } }
2013-08-14 14:56:07 -04:00
encoding/json: fix the broken "overwriting of data" tests
Because TestUnmarshal actually allocates a new value to decode into
using ptr's pointer type, any existing data is thrown away. This was
harmless in alomst all of the test cases, minus the "overwriting of
data" ones added in 2015 in CL 12209.
I spotted that nothing covered decoding a JSON array with few elements
into a slice which already had many elements. I initially assumed that
the code was buggy or that some code could be removed, when in fact
there simply wasn't any code covering the edge case.
Move those two tests to TestPrefilled, which already served a very
similar purpose. Remove the map case, as TestPrefilled already has
plenty of prefilled map cases. Moreover, we no longer reset an entire
map when decoding, as per the godoc:
To unmarshal a JSON object into a map, Unmarshal first
establishes a map to use. If the map is nil, Unmarshal allocates
a new map. Otherwise Unmarshal reuses the existing map, keeping
existing entries.
Finally, to ensure that ptr is used correctly in the future, make
TestUnmarshal error if it's anything other than a pointer to a zero
value. That is, the only correct use should be new(type). Don't rename
the ptr field, as that would be extremely noisy and cause unwanted merge
conflicts.
Change-Id: I41e3ecfeae42d877ac5443a6bd622ac3d6c8120c
Reviewed-on: https://go-review.googlesource.com/c/go/+/185738
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Emmanuel Odeke <emm.odeke@gmail.com>
2019-07-11 21:27:19 +09:00
umtrueXY = unmarshalerText { "x" , "y" }
umsliceXY = [ ] unmarshalerText { { "x" , "y" } }
umstructXY = ustructText { unmarshalerText { "x" , "y" } }
2016-03-08 12:41:35 -08:00
encoding/json: fix the broken "overwriting of data" tests
Because TestUnmarshal actually allocates a new value to decode into
using ptr's pointer type, any existing data is thrown away. This was
harmless in alomst all of the test cases, minus the "overwriting of
data" ones added in 2015 in CL 12209.
I spotted that nothing covered decoding a JSON array with few elements
into a slice which already had many elements. I initially assumed that
the code was buggy or that some code could be removed, when in fact
there simply wasn't any code covering the edge case.
Move those two tests to TestPrefilled, which already served a very
similar purpose. Remove the map case, as TestPrefilled already has
plenty of prefilled map cases. Moreover, we no longer reset an entire
map when decoding, as per the godoc:
To unmarshal a JSON object into a map, Unmarshal first
establishes a map to use. If the map is nil, Unmarshal allocates
a new map. Otherwise Unmarshal reuses the existing map, keeping
existing entries.
Finally, to ensure that ptr is used correctly in the future, make
TestUnmarshal error if it's anything other than a pointer to a zero
value. That is, the only correct use should be new(type). Don't rename
the ptr field, as that would be extremely noisy and cause unwanted merge
conflicts.
Change-Id: I41e3ecfeae42d877ac5443a6bd622ac3d6c8120c
Reviewed-on: https://go-review.googlesource.com/c/go/+/185738
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Emmanuel Odeke <emm.odeke@gmail.com>
2019-07-11 21:27:19 +09:00
ummapXY = map [ unmarshalerText ] bool { { "x" , "y" } : true }
2010-11-08 15:33:00 -08:00
)
2012-09-10 23:31:40 -04:00
// Test data structures for anonymous fields.
type Point struct {
Z int
}
type Top struct {
Level0 int
Embed0
* Embed0a
* Embed0b ` json:"e,omitempty" ` // treated as named
Embed0c ` json:"-" ` // ignored
Loop
Embed0p // has Point with X, Y, used
Embed0q // has Point with Z, used
2015-08-28 10:17:05 +02:00
embed // contains exported field
2012-09-10 23:31:40 -04:00
}
type Embed0 struct {
Level1a int // overridden by Embed0a's Level1a with json tag
Level1b int // used because Embed0a's Level1b is renamed
Level1c int // used because Embed0a's Level1c is ignored
Level1d int // annihilated by Embed0a's Level1d
Level1e int ` json:"x" ` // annihilated by Embed0a.Level1e
}
type Embed0a struct {
Level1a int ` json:"Level1a,omitempty" `
Level1b int ` json:"LEVEL1B,omitempty" `
Level1c int ` json:"-" `
Level1d int // annihilated by Embed0's Level1d
Level1f int ` json:"x" ` // annihilated by Embed0's Level1e
}
type Embed0b Embed0
type Embed0c Embed0
type Embed0p struct {
image . Point
}
type Embed0q struct {
Point
}
2015-08-28 10:17:05 +02:00
type embed struct {
Q int
}
2012-09-10 23:31:40 -04:00
type Loop struct {
Loop1 int ` json:",omitempty" `
Loop2 int ` json:",omitempty" `
* Loop
}
// From reflect test:
// The X in S6 and S7 annihilate, but they also block the X in S8.S9.
type S5 struct {
S6
S7
S8
}
type S6 struct {
X int
}
type S7 S6
type S8 struct {
S9
}
type S9 struct {
X int
Y int
}
// From reflect test:
// The X in S11.S6 and S12.S6 annihilate, but they also block the X in S13.S8.S9.
type S10 struct {
S11
S12
S13
}
type S11 struct {
S6
}
type S12 struct {
S6
}
type S13 struct {
S8
}
type Ambig struct {
// Given "hello", the first match should win.
First int ` json:"HELLO" `
Second int ` json:"Hello" `
}
2013-09-09 19:11:05 -04:00
type XYZ struct {
2021-12-01 12:15:45 -05:00
X any
Y any
Z any
2013-09-09 19:11:05 -04:00
}
2018-10-11 14:43:47 +01:00
type unexportedWithMethods struct { }
func ( unexportedWithMethods ) F ( ) { }
2016-05-23 12:21:57 -04:00
type byteWithMarshalJSON byte
func ( b byteWithMarshalJSON ) MarshalJSON ( ) ( [ ] byte , error ) {
return [ ] byte ( fmt . Sprintf ( ` "Z%.2x" ` , byte ( b ) ) ) , nil
}
func ( b * byteWithMarshalJSON ) UnmarshalJSON ( data [ ] byte ) error {
if len ( data ) != 5 || data [ 0 ] != '"' || data [ 1 ] != 'Z' || data [ 4 ] != '"' {
return fmt . Errorf ( "bad quoted string" )
}
i , err := strconv . ParseInt ( string ( data [ 2 : 4 ] ) , 16 , 8 )
if err != nil {
return fmt . Errorf ( "bad hex" )
}
* b = byteWithMarshalJSON ( i )
return nil
}
type byteWithPtrMarshalJSON byte
func ( b * byteWithPtrMarshalJSON ) MarshalJSON ( ) ( [ ] byte , error ) {
return byteWithMarshalJSON ( * b ) . MarshalJSON ( )
}
func ( b * byteWithPtrMarshalJSON ) UnmarshalJSON ( data [ ] byte ) error {
return ( * byteWithMarshalJSON ) ( b ) . UnmarshalJSON ( data )
}
type byteWithMarshalText byte
func ( b byteWithMarshalText ) MarshalText ( ) ( [ ] byte , error ) {
return [ ] byte ( fmt . Sprintf ( ` Z%.2x ` , byte ( b ) ) ) , nil
}
func ( b * byteWithMarshalText ) UnmarshalText ( data [ ] byte ) error {
if len ( data ) != 3 || data [ 0 ] != 'Z' {
return fmt . Errorf ( "bad quoted string" )
}
i , err := strconv . ParseInt ( string ( data [ 1 : 3 ] ) , 16 , 8 )
if err != nil {
return fmt . Errorf ( "bad hex" )
}
* b = byteWithMarshalText ( i )
return nil
}
type byteWithPtrMarshalText byte
func ( b * byteWithPtrMarshalText ) MarshalText ( ) ( [ ] byte , error ) {
return byteWithMarshalText ( * b ) . MarshalText ( )
}
func ( b * byteWithPtrMarshalText ) UnmarshalText ( data [ ] byte ) error {
return ( * byteWithMarshalText ) ( b ) . UnmarshalText ( data )
}
type intWithMarshalJSON int
func ( b intWithMarshalJSON ) MarshalJSON ( ) ( [ ] byte , error ) {
return [ ] byte ( fmt . Sprintf ( ` "Z%.2x" ` , int ( b ) ) ) , nil
}
func ( b * intWithMarshalJSON ) UnmarshalJSON ( data [ ] byte ) error {
if len ( data ) != 5 || data [ 0 ] != '"' || data [ 1 ] != 'Z' || data [ 4 ] != '"' {
return fmt . Errorf ( "bad quoted string" )
}
i , err := strconv . ParseInt ( string ( data [ 2 : 4 ] ) , 16 , 8 )
if err != nil {
return fmt . Errorf ( "bad hex" )
}
* b = intWithMarshalJSON ( i )
return nil
}
type intWithPtrMarshalJSON int
func ( b * intWithPtrMarshalJSON ) MarshalJSON ( ) ( [ ] byte , error ) {
return intWithMarshalJSON ( * b ) . MarshalJSON ( )
}
func ( b * intWithPtrMarshalJSON ) UnmarshalJSON ( data [ ] byte ) error {
return ( * intWithMarshalJSON ) ( b ) . UnmarshalJSON ( data )
}
type intWithMarshalText int
func ( b intWithMarshalText ) MarshalText ( ) ( [ ] byte , error ) {
return [ ] byte ( fmt . Sprintf ( ` Z%.2x ` , int ( b ) ) ) , nil
}
func ( b * intWithMarshalText ) UnmarshalText ( data [ ] byte ) error {
if len ( data ) != 3 || data [ 0 ] != 'Z' {
return fmt . Errorf ( "bad quoted string" )
}
i , err := strconv . ParseInt ( string ( data [ 1 : 3 ] ) , 16 , 8 )
if err != nil {
return fmt . Errorf ( "bad hex" )
}
* b = intWithMarshalText ( i )
return nil
}
type intWithPtrMarshalText int
func ( b * intWithPtrMarshalText ) MarshalText ( ) ( [ ] byte , error ) {
return intWithMarshalText ( * b ) . MarshalText ( )
}
func ( b * intWithPtrMarshalText ) UnmarshalText ( data [ ] byte ) error {
return ( * intWithMarshalText ) ( b ) . UnmarshalText ( data )
}
2018-08-28 15:56:10 +00:00
type mapStringToStringData struct {
Data map [ string ] string ` json:"data" `
}
2016-10-12 15:55:02 -04:00
type B struct {
B bool ` json:",string" `
}
2019-05-01 14:52:57 +02:00
type DoublePtr struct {
I * * int
J * * int
}
2023-08-24 12:49:10 -07:00
var unmarshalTests = [ ] struct {
CaseName
in string
ptr any // new(type)
out any
err error
useNumber bool
golden bool
disallowUnknownFields bool
} {
2010-04-21 16:40:53 -07:00
// basic types
2023-08-24 12:49:10 -07:00
{ CaseName : Name ( "" ) , in : ` true ` , ptr : new ( bool ) , out : true } ,
{ CaseName : Name ( "" ) , in : ` 1 ` , ptr : new ( int ) , out : 1 } ,
{ CaseName : Name ( "" ) , in : ` 1.2 ` , ptr : new ( float64 ) , out : 1.2 } ,
{ CaseName : Name ( "" ) , in : ` -5 ` , ptr : new ( int16 ) , out : int16 ( - 5 ) } ,
{ CaseName : Name ( "" ) , in : ` 2 ` , ptr : new ( Number ) , out : Number ( "2" ) , useNumber : true } ,
{ CaseName : Name ( "" ) , in : ` 2 ` , ptr : new ( Number ) , out : Number ( "2" ) } ,
{ CaseName : Name ( "" ) , in : ` 2 ` , ptr : new ( any ) , out : float64 ( 2.0 ) } ,
{ CaseName : Name ( "" ) , in : ` 2 ` , ptr : new ( any ) , out : Number ( "2" ) , useNumber : true } ,
{ CaseName : Name ( "" ) , in : ` "a\u1234" ` , ptr : new ( string ) , out : "a\u1234" } ,
{ CaseName : Name ( "" ) , in : ` "http:\/\/" ` , ptr : new ( string ) , out : "http://" } ,
{ CaseName : Name ( "" ) , in : ` "g-clef: \uD834\uDD1E" ` , ptr : new ( string ) , out : "g-clef: \U0001D11E" } ,
{ CaseName : Name ( "" ) , in : ` "invalid: \uD834x\uDD1E" ` , ptr : new ( string ) , out : "invalid: \uFFFDx\uFFFD" } ,
{ CaseName : Name ( "" ) , in : "null" , ptr : new ( any ) , out : nil } ,
{ CaseName : Name ( "" ) , in : ` { "X": [1,2,3], "Y": 4} ` , ptr : new ( T ) , out : T { Y : 4 } , err : & UnmarshalTypeError { "array" , reflect . TypeFor [ string ] ( ) , 7 , "T" , "X" } } ,
{ CaseName : Name ( "" ) , in : ` { "X": 23} ` , ptr : new ( T ) , out : T { } , err : & UnmarshalTypeError { "number" , reflect . TypeFor [ string ] ( ) , 8 , "T" , "X" } } ,
{ CaseName : Name ( "" ) , in : ` { "x": 1} ` , ptr : new ( tx ) , out : tx { } } ,
{ CaseName : Name ( "" ) , in : ` { "x": 1} ` , ptr : new ( tx ) , out : tx { } } ,
{ CaseName : Name ( "" ) , in : ` { "x": 1} ` , ptr : new ( tx ) , err : fmt . Errorf ( "json: unknown field \"x\"" ) , disallowUnknownFields : true } ,
{ CaseName : Name ( "" ) , in : ` { "S": 23} ` , ptr : new ( W ) , out : W { } , err : & UnmarshalTypeError { "number" , reflect . TypeFor [ SS ] ( ) , 0 , "W" , "S" } } ,
{ CaseName : Name ( "" ) , in : ` { "F1":1,"F2":2,"F3":3} ` , ptr : new ( V ) , out : V { F1 : float64 ( 1 ) , F2 : int32 ( 2 ) , F3 : Number ( "3" ) } } ,
{ CaseName : Name ( "" ) , in : ` { "F1":1,"F2":2,"F3":3} ` , ptr : new ( V ) , out : V { F1 : Number ( "1" ) , F2 : int32 ( 2 ) , F3 : Number ( "3" ) } , useNumber : true } ,
{ CaseName : Name ( "" ) , in : ` { "k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4": { "kk1":"s","kk2":2}} ` , ptr : new ( any ) , out : ifaceNumAsFloat64 } ,
{ CaseName : Name ( "" ) , in : ` { "k1":1,"k2":"s","k3":[1,2.0,3e-3],"k4": { "kk1":"s","kk2":2}} ` , ptr : new ( any ) , out : ifaceNumAsNumber , useNumber : true } ,
2009-11-30 13:55:09 -08:00
2013-01-10 17:58:45 -08:00
// raw values with whitespace
2023-08-24 12:49:10 -07:00
{ CaseName : Name ( "" ) , in : "\n true " , ptr : new ( bool ) , out : true } ,
{ CaseName : Name ( "" ) , in : "\t 1 " , ptr : new ( int ) , out : 1 } ,
{ CaseName : Name ( "" ) , in : "\r 1.2 " , ptr : new ( float64 ) , out : 1.2 } ,
{ CaseName : Name ( "" ) , in : "\t -5 \n" , ptr : new ( int16 ) , out : int16 ( - 5 ) } ,
{ CaseName : Name ( "" ) , in : "\t \"a\\u1234\" \n" , ptr : new ( string ) , out : "a\u1234" } ,
2013-01-10 17:58:45 -08:00
2011-09-15 08:09:43 +10:00
// Z has a "-" tag.
2023-08-24 12:49:10 -07:00
{ CaseName : Name ( "" ) , in : ` { "Y": 1, "Z": 2} ` , ptr : new ( T ) , out : T { Y : 1 } } ,
{ CaseName : Name ( "" ) , in : ` { "Y": 1, "Z": 2} ` , ptr : new ( T ) , err : fmt . Errorf ( "json: unknown field \"Z\"" ) , disallowUnknownFields : true } ,
2011-09-15 08:09:43 +10:00
2023-08-24 12:49:10 -07:00
{ CaseName : Name ( "" ) , in : ` { "alpha": "abc", "alphabet": "xyz"} ` , ptr : new ( U ) , out : U { Alphabet : "abc" } } ,
{ CaseName : Name ( "" ) , in : ` { "alpha": "abc", "alphabet": "xyz"} ` , ptr : new ( U ) , err : fmt . Errorf ( "json: unknown field \"alphabet\"" ) , disallowUnknownFields : true } ,
{ CaseName : Name ( "" ) , in : ` { "alpha": "abc"} ` , ptr : new ( U ) , out : U { Alphabet : "abc" } } ,
{ CaseName : Name ( "" ) , in : ` { "alphabet": "xyz"} ` , ptr : new ( U ) , out : U { } } ,
{ CaseName : Name ( "" ) , in : ` { "alphabet": "xyz"} ` , ptr : new ( U ) , err : fmt . Errorf ( "json: unknown field \"alphabet\"" ) , disallowUnknownFields : true } ,
2012-05-01 11:37:44 +10:00
2010-08-03 17:05:00 -07:00
// syntax errors
2023-08-24 12:49:10 -07:00
{ CaseName : Name ( "" ) , in : ` { "X": "foo", "Y"} ` , err : & SyntaxError { "invalid character '}' after object key" , 17 } } ,
{ CaseName : Name ( "" ) , in : ` [1, 2, 3+] ` , err : & SyntaxError { "invalid character '+' after array element" , 9 } } ,
{ CaseName : Name ( "" ) , in : ` { "X":12x} ` , err : & SyntaxError { "invalid character 'x' after object key:value pair" , 8 } , useNumber : true } ,
{ CaseName : Name ( "" ) , in : ` [2, 3 ` , err : & SyntaxError { msg : "unexpected end of JSON input" , Offset : 5 } } ,
{ CaseName : Name ( "" ) , in : ` { "F3": -} ` , ptr : new ( V ) , out : V { F3 : Number ( "-" ) } , err : & SyntaxError { msg : "invalid character '}' in numeric literal" , Offset : 9 } } ,
2011-12-19 15:32:06 -05:00
2013-01-10 17:58:45 -08:00
// raw value errors
2023-08-24 12:49:10 -07:00
{ CaseName : Name ( "" ) , in : "\x01 42" , err : & SyntaxError { "invalid character '\\x01' looking for beginning of value" , 1 } } ,
{ CaseName : Name ( "" ) , in : " 42 \x01" , err : & SyntaxError { "invalid character '\\x01' after top-level value" , 5 } } ,
{ CaseName : Name ( "" ) , in : "\x01 true" , err : & SyntaxError { "invalid character '\\x01' looking for beginning of value" , 1 } } ,
{ CaseName : Name ( "" ) , in : " false \x01" , err : & SyntaxError { "invalid character '\\x01' after top-level value" , 8 } } ,
{ CaseName : Name ( "" ) , in : "\x01 1.2" , err : & SyntaxError { "invalid character '\\x01' looking for beginning of value" , 1 } } ,
{ CaseName : Name ( "" ) , in : " 3.4 \x01" , err : & SyntaxError { "invalid character '\\x01' after top-level value" , 6 } } ,
{ CaseName : Name ( "" ) , in : "\x01 \"string\"" , err : & SyntaxError { "invalid character '\\x01' looking for beginning of value" , 1 } } ,
{ CaseName : Name ( "" ) , in : " \"string\" \x01" , err : & SyntaxError { "invalid character '\\x01' after top-level value" , 11 } } ,
2013-01-10 17:58:45 -08:00
2011-12-19 15:32:06 -05:00
// array tests
2023-08-24 12:49:10 -07:00
{ CaseName : Name ( "" ) , in : ` [1, 2, 3] ` , ptr : new ( [ 3 ] int ) , out : [ 3 ] int { 1 , 2 , 3 } } ,
{ CaseName : Name ( "" ) , in : ` [1, 2, 3] ` , ptr : new ( [ 1 ] int ) , out : [ 1 ] int { 1 } } ,
{ CaseName : Name ( "" ) , in : ` [1, 2, 3] ` , ptr : new ( [ 5 ] int ) , out : [ 5 ] int { 1 , 2 , 3 , 0 , 0 } } ,
{ CaseName : Name ( "" ) , in : ` [1, 2, 3] ` , ptr : new ( MustNotUnmarshalJSON ) , err : errors . New ( "MustNotUnmarshalJSON was used" ) } ,
2010-08-03 17:05:00 -07:00
2013-01-30 09:10:32 -08:00
// empty array to interface test
2023-08-24 12:49:10 -07:00
{ CaseName : Name ( "" ) , in : ` [] ` , ptr : new ( [ ] any ) , out : [ ] any { } } ,
{ CaseName : Name ( "" ) , in : ` null ` , ptr : new ( [ ] any ) , out : [ ] any ( nil ) } ,
{ CaseName : Name ( "" ) , in : ` { "T":[]} ` , ptr : new ( map [ string ] any ) , out : map [ string ] any { "T" : [ ] any { } } } ,
{ CaseName : Name ( "" ) , in : ` { "T":null} ` , ptr : new ( map [ string ] any ) , out : map [ string ] any { "T" : any ( nil ) } } ,
2013-01-30 09:10:32 -08:00
2010-04-21 16:40:53 -07:00
// composite tests
2023-08-24 12:49:10 -07:00
{ CaseName : Name ( "" ) , in : allValueIndent , ptr : new ( All ) , out : allValue } ,
{ CaseName : Name ( "" ) , in : allValueCompact , ptr : new ( All ) , out : allValue } ,
{ CaseName : Name ( "" ) , in : allValueIndent , ptr : new ( * All ) , out : & allValue } ,
{ CaseName : Name ( "" ) , in : allValueCompact , ptr : new ( * All ) , out : & allValue } ,
{ CaseName : Name ( "" ) , in : pallValueIndent , ptr : new ( All ) , out : pallValue } ,
{ CaseName : Name ( "" ) , in : pallValueCompact , ptr : new ( All ) , out : pallValue } ,
{ CaseName : Name ( "" ) , in : pallValueIndent , ptr : new ( * All ) , out : & pallValue } ,
{ CaseName : Name ( "" ) , in : pallValueCompact , ptr : new ( * All ) , out : & pallValue } ,
2010-11-08 15:33:00 -08:00
// unmarshal interface test
2023-08-24 12:49:10 -07:00
{ CaseName : Name ( "" ) , in : ` { "T":false} ` , ptr : new ( unmarshaler ) , out : umtrue } , // use "false" so test will fail if custom unmarshaler is not called
{ CaseName : Name ( "" ) , in : ` { "T":false} ` , ptr : new ( * unmarshaler ) , out : & umtrue } ,
{ CaseName : Name ( "" ) , in : ` [ { "T":false}] ` , ptr : new ( [ ] unmarshaler ) , out : umslice } ,
{ CaseName : Name ( "" ) , in : ` [ { "T":false}] ` , ptr : new ( * [ ] unmarshaler ) , out : & umslice } ,
{ CaseName : Name ( "" ) , in : ` { "M": { "T":"x:y"}} ` , ptr : new ( ustruct ) , out : umstruct } ,
2012-09-10 23:31:40 -04:00
2013-08-14 14:56:07 -04:00
// UnmarshalText interface test
2023-08-24 12:49:10 -07:00
{ CaseName : Name ( "" ) , in : ` "x:y" ` , ptr : new ( unmarshalerText ) , out : umtrueXY } ,
{ CaseName : Name ( "" ) , in : ` "x:y" ` , ptr : new ( * unmarshalerText ) , out : & umtrueXY } ,
{ CaseName : Name ( "" ) , in : ` ["x:y"] ` , ptr : new ( [ ] unmarshalerText ) , out : umsliceXY } ,
{ CaseName : Name ( "" ) , in : ` ["x:y"] ` , ptr : new ( * [ ] unmarshalerText ) , out : & umsliceXY } ,
{ CaseName : Name ( "" ) , in : ` { "M":"x:y"} ` , ptr : new ( ustructText ) , out : umstructXY } ,
2016-03-08 12:41:35 -08:00
2016-04-13 16:51:25 -07:00
// integer-keyed map test
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "-1":"a","0":"b","1":"c"} ` ,
ptr : new ( map [ int ] string ) ,
out : map [ int ] string { - 1 : "a" , 0 : "b" , 1 : "c" } ,
2016-04-13 16:51:25 -07:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "0":"a","10":"c","9":"b"} ` ,
ptr : new ( map [ u8 ] string ) ,
out : map [ u8 ] string { 0 : "a" , 9 : "b" , 10 : "c" } ,
2016-04-13 16:51:25 -07:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "-9223372036854775808":"min","9223372036854775807":"max"} ` ,
ptr : new ( map [ int64 ] string ) ,
out : map [ int64 ] string { math . MinInt64 : "min" , math . MaxInt64 : "max" } ,
2016-04-13 16:51:25 -07:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "18446744073709551615":"max"} ` ,
ptr : new ( map [ uint64 ] string ) ,
out : map [ uint64 ] string { math . MaxUint64 : "max" } ,
2016-04-13 16:51:25 -07:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "0":false,"10":true} ` ,
ptr : new ( map [ uintptr ] bool ) ,
out : map [ uintptr ] bool { 0 : false , 10 : true } ,
2016-04-13 16:51:25 -07:00
} ,
// Check that MarshalText and UnmarshalText take precedence
// over default integer handling in map keys.
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "u2":4} ` ,
ptr : new ( map [ u8marshal ] int ) ,
out : map [ u8marshal ] int { 2 : 4 } ,
2016-04-13 16:51:25 -07:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "2":4} ` ,
ptr : new ( map [ u8marshal ] int ) ,
err : errMissingU8Prefix ,
2016-04-13 16:51:25 -07:00
} ,
// integer-keyed map errors
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "abc":"abc"} ` ,
ptr : new ( map [ int ] string ) ,
err : & UnmarshalTypeError { Value : "number abc" , Type : reflect . TypeFor [ int ] ( ) , Offset : 2 } ,
2016-04-13 16:51:25 -07:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "256":"abc"} ` ,
ptr : new ( map [ uint8 ] string ) ,
err : & UnmarshalTypeError { Value : "number 256" , Type : reflect . TypeFor [ uint8 ] ( ) , Offset : 2 } ,
2016-04-13 16:51:25 -07:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "128":"abc"} ` ,
ptr : new ( map [ int8 ] string ) ,
err : & UnmarshalTypeError { Value : "number 128" , Type : reflect . TypeFor [ int8 ] ( ) , Offset : 2 } ,
2016-04-13 16:51:25 -07:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "-1":"abc"} ` ,
ptr : new ( map [ uint8 ] string ) ,
err : & UnmarshalTypeError { Value : "number -1" , Type : reflect . TypeFor [ uint8 ] ( ) , Offset : 2 } ,
2016-04-13 16:51:25 -07:00
} ,
encoding/json: fix "data changed underfoot?" panic
Given a program as follows:
data := []byte(`{"F": {
"a": 2,
"3": 4
}}`)
json.Unmarshal(data, &map[string]map[int]int{})
The JSON package should error, as "a" is not a valid integer. However,
we'd encounter a panic:
panic: JSON decoder out of sync - data changing underfoot?
The reason was that decodeState.object would return a nil error on
encountering the invalid map key string, while saving the key type error
for later. This broke if we were inside another object, as we would
abruptly end parsing the nested object, leaving the decoder in an
unexpected state.
To fix this, simply avoid storing the map element and continue decoding
the object, to leave the decoder state exactly as if we hadn't seen an
invalid key type.
This affected both signed and unsigned integer keys, so fix both and add
two test cases.
Updates #28189.
Change-Id: I8a6204cc3ff9fb04ed769df7a20a824c8b94faff
Reviewed-on: https://go-review.googlesource.com/c/142518
Reviewed-by: Ian Lance Taylor <iant@golang.org>
2018-10-16 12:04:55 +01:00
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "F": { "a":2,"3":4}} ` ,
ptr : new ( map [ string ] map [ int ] int ) ,
err : & UnmarshalTypeError { Value : "number a" , Type : reflect . TypeFor [ int ] ( ) , Offset : 7 } ,
encoding/json: fix "data changed underfoot?" panic
Given a program as follows:
data := []byte(`{"F": {
"a": 2,
"3": 4
}}`)
json.Unmarshal(data, &map[string]map[int]int{})
The JSON package should error, as "a" is not a valid integer. However,
we'd encounter a panic:
panic: JSON decoder out of sync - data changing underfoot?
The reason was that decodeState.object would return a nil error on
encountering the invalid map key string, while saving the key type error
for later. This broke if we were inside another object, as we would
abruptly end parsing the nested object, leaving the decoder in an
unexpected state.
To fix this, simply avoid storing the map element and continue decoding
the object, to leave the decoder state exactly as if we hadn't seen an
invalid key type.
This affected both signed and unsigned integer keys, so fix both and add
two test cases.
Updates #28189.
Change-Id: I8a6204cc3ff9fb04ed769df7a20a824c8b94faff
Reviewed-on: https://go-review.googlesource.com/c/142518
Reviewed-by: Ian Lance Taylor <iant@golang.org>
2018-10-16 12:04:55 +01:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "F": { "a":2,"3":4}} ` ,
ptr : new ( map [ string ] map [ uint ] int ) ,
err : & UnmarshalTypeError { Value : "number a" , Type : reflect . TypeFor [ uint ] ( ) , Offset : 7 } ,
encoding/json: fix "data changed underfoot?" panic
Given a program as follows:
data := []byte(`{"F": {
"a": 2,
"3": 4
}}`)
json.Unmarshal(data, &map[string]map[int]int{})
The JSON package should error, as "a" is not a valid integer. However,
we'd encounter a panic:
panic: JSON decoder out of sync - data changing underfoot?
The reason was that decodeState.object would return a nil error on
encountering the invalid map key string, while saving the key type error
for later. This broke if we were inside another object, as we would
abruptly end parsing the nested object, leaving the decoder in an
unexpected state.
To fix this, simply avoid storing the map element and continue decoding
the object, to leave the decoder state exactly as if we hadn't seen an
invalid key type.
This affected both signed and unsigned integer keys, so fix both and add
two test cases.
Updates #28189.
Change-Id: I8a6204cc3ff9fb04ed769df7a20a824c8b94faff
Reviewed-on: https://go-review.googlesource.com/c/142518
Reviewed-by: Ian Lance Taylor <iant@golang.org>
2018-10-16 12:04:55 +01:00
} ,
2016-04-13 16:51:25 -07:00
// Map keys can be encoding.TextUnmarshalers.
2023-08-24 12:49:10 -07:00
{ CaseName : Name ( "" ) , in : ` { "x:y":true} ` , ptr : new ( map [ unmarshalerText ] bool ) , out : ummapXY } ,
2016-03-08 12:41:35 -08:00
// If multiple values for the same key exists, only the most recent value is used.
2023-08-24 12:49:10 -07:00
{ CaseName : Name ( "" ) , in : ` { "x:y":false,"x:y":true} ` , ptr : new ( map [ unmarshalerText ] bool ) , out : ummapXY } ,
2015-07-14 21:32:47 -04:00
2012-09-10 23:31:40 -04:00
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
2012-09-10 23:31:40 -04:00
in : ` {
"Level0" : 1 ,
"Level1b" : 2 ,
"Level1c" : 3 ,
"x" : 4 ,
"Level1a" : 5 ,
"LEVEL1B" : 6 ,
"e" : {
"Level1a" : 8 ,
"Level1b" : 9 ,
"Level1c" : 10 ,
"Level1d" : 11 ,
"x" : 12
} ,
"Loop1" : 13 ,
"Loop2" : 14 ,
"X" : 15 ,
"Y" : 16 ,
2015-08-28 10:17:05 +02:00
"Z" : 17 ,
"Q" : 18
2012-09-10 23:31:40 -04:00
} ` ,
ptr : new ( Top ) ,
out : Top {
Level0 : 1 ,
Embed0 : Embed0 {
Level1b : 2 ,
Level1c : 3 ,
} ,
Embed0a : & Embed0a {
Level1a : 5 ,
Level1b : 6 ,
} ,
Embed0b : & Embed0b {
Level1a : 8 ,
Level1b : 9 ,
Level1c : 10 ,
Level1d : 11 ,
Level1e : 12 ,
} ,
Loop : Loop {
Loop1 : 13 ,
Loop2 : 14 ,
} ,
Embed0p : Embed0p {
Point : image . Point { X : 15 , Y : 16 } ,
} ,
Embed0q : Embed0q {
Point : Point { Z : 17 } ,
} ,
2015-08-28 10:17:05 +02:00
embed : embed {
Q : 18 ,
} ,
2012-09-10 23:31:40 -04:00
} ,
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "hello": 1} ` ,
ptr : new ( Ambig ) ,
out : Ambig { First : 1 } ,
2012-09-10 23:31:40 -04:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "X": 1,"Y":2} ` ,
ptr : new ( S5 ) ,
out : S5 { S8 : S8 { S9 : S9 { Y : 2 } } } ,
2012-09-10 23:31:40 -04:00
} ,
2017-10-31 13:16:38 -07:00
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
go/printer, gofmt: tuned table alignment for better results
The go/printer (and thus gofmt) uses a heuristic to determine
whether to break alignment between elements of an expression
list which is spread across multiple lines. The heuristic only
kicked in if the entry sizes (character length) was above a
certain threshold (20) and the ratio between the previous and
current entry size was above a certain value (4).
This heuristic worked reasonably most of the time, but also
led to unfortunate breaks in many cases where a single entry
was suddenly much smaller (or larger) then the previous one.
The behavior of gofmt was sufficiently mysterious in some of
these situations that many issues were filed against it.
The simplest solution to address this problem is to remove
the heuristic altogether and have a programmer introduce
empty lines to force different alignments if it improves
readability. The problem with that approach is that the
places where it really matters, very long tables with many
(hundreds, or more) entries, may be machine-generated and
not "post-processed" by a human (e.g., unicode/utf8/tables.go).
If a single one of those entries is overlong, the result
would be that the alignment would force all comments or
values in key:value pairs to be adjusted to that overlong
value, making the table hard to read (e.g., that entry may
not even be visible on screen and all other entries seem
spaced out too wide).
Instead, we opted for a slightly improved heuristic that
behaves much better for "normal", human-written code.
1) The threshold is increased from 20 to 40. This disables
the heuristic for many common cases yet even if the alignment
is not "ideal", 40 is not that many characters per line with
todays screens, making it very likely that the entire line
remains "visible" in an editor.
2) Changed the heuristic to not simply look at the size ratio
between current and previous line, but instead considering the
geometric mean of the sizes of the previous (aligned) lines.
This emphasizes the "overall picture" of the previous lines,
rather than a single one (which might be an outlier).
3) Changed the ratio from 4 to 2.5. Now that we ignore sizes
below 40, a ratio of 4 would mean that a new entry would have
to be 4 times bigger (160) or smaller (10) before alignment
would be broken. A ratio of 2.5 seems more sensible.
Applied updated gofmt to all of src and misc. Also tested
against several former issues that complained about this
and verified that the output for the given examples is
satisfactory (added respective test cases).
Some of the files changed because they were not gofmt-ed
in the first place.
For #644.
For #7335.
For #10392.
(and probably more related issues)
Fixes #22852.
Change-Id: I5e48b3d3b157a5cf2d649833b7297b33f43a6f6e
2018-04-03 17:05:47 -07:00
in : ` { "X": 1,"Y":2} ` ,
ptr : new ( S5 ) ,
err : fmt . Errorf ( "json: unknown field \"X\"" ) ,
2017-10-31 13:16:38 -07:00
disallowUnknownFields : true ,
} ,
2012-09-10 23:31:40 -04:00
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "X": 1,"Y":2} ` ,
ptr : new ( S10 ) ,
out : S10 { S13 : S13 { S8 : S8 { S9 : S9 { Y : 2 } } } } ,
2012-09-10 23:31:40 -04:00
} ,
2017-10-31 13:16:38 -07:00
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
go/printer, gofmt: tuned table alignment for better results
The go/printer (and thus gofmt) uses a heuristic to determine
whether to break alignment between elements of an expression
list which is spread across multiple lines. The heuristic only
kicked in if the entry sizes (character length) was above a
certain threshold (20) and the ratio between the previous and
current entry size was above a certain value (4).
This heuristic worked reasonably most of the time, but also
led to unfortunate breaks in many cases where a single entry
was suddenly much smaller (or larger) then the previous one.
The behavior of gofmt was sufficiently mysterious in some of
these situations that many issues were filed against it.
The simplest solution to address this problem is to remove
the heuristic altogether and have a programmer introduce
empty lines to force different alignments if it improves
readability. The problem with that approach is that the
places where it really matters, very long tables with many
(hundreds, or more) entries, may be machine-generated and
not "post-processed" by a human (e.g., unicode/utf8/tables.go).
If a single one of those entries is overlong, the result
would be that the alignment would force all comments or
values in key:value pairs to be adjusted to that overlong
value, making the table hard to read (e.g., that entry may
not even be visible on screen and all other entries seem
spaced out too wide).
Instead, we opted for a slightly improved heuristic that
behaves much better for "normal", human-written code.
1) The threshold is increased from 20 to 40. This disables
the heuristic for many common cases yet even if the alignment
is not "ideal", 40 is not that many characters per line with
todays screens, making it very likely that the entire line
remains "visible" in an editor.
2) Changed the heuristic to not simply look at the size ratio
between current and previous line, but instead considering the
geometric mean of the sizes of the previous (aligned) lines.
This emphasizes the "overall picture" of the previous lines,
rather than a single one (which might be an outlier).
3) Changed the ratio from 4 to 2.5. Now that we ignore sizes
below 40, a ratio of 4 would mean that a new entry would have
to be 4 times bigger (160) or smaller (10) before alignment
would be broken. A ratio of 2.5 seems more sensible.
Applied updated gofmt to all of src and misc. Also tested
against several former issues that complained about this
and verified that the output for the given examples is
satisfactory (added respective test cases).
Some of the files changed because they were not gofmt-ed
in the first place.
For #644.
For #7335.
For #10392.
(and probably more related issues)
Fixes #22852.
Change-Id: I5e48b3d3b157a5cf2d649833b7297b33f43a6f6e
2018-04-03 17:05:47 -07:00
in : ` { "X": 1,"Y":2} ` ,
ptr : new ( S10 ) ,
err : fmt . Errorf ( "json: unknown field \"X\"" ) ,
2017-10-31 13:16:38 -07:00
disallowUnknownFields : true ,
} ,
2019-05-01 14:52:57 +02:00
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "I": 0, "I": null, "J": null} ` ,
ptr : new ( DoublePtr ) ,
out : DoublePtr { I : nil , J : nil } ,
2019-05-01 14:52:57 +02:00
} ,
2013-02-14 14:56:01 -05:00
// invalid UTF-8 is coerced to valid UTF-8.
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : "\"hello\xffworld\"" ,
ptr : new ( string ) ,
out : "hello\ufffdworld" ,
2013-02-14 14:56:01 -05:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : "\"hello\xc2\xc2world\"" ,
ptr : new ( string ) ,
out : "hello\ufffd\ufffdworld" ,
2013-02-14 14:56:01 -05:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : "\"hello\xc2\xffworld\"" ,
ptr : new ( string ) ,
out : "hello\ufffd\ufffdworld" ,
2013-02-14 14:56:01 -05:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : "\"hello\\ud800world\"" ,
ptr : new ( string ) ,
out : "hello\ufffdworld" ,
2013-02-14 14:56:01 -05:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : "\"hello\\ud800\\ud800world\"" ,
ptr : new ( string ) ,
out : "hello\ufffd\ufffdworld" ,
2013-02-14 14:56:01 -05:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : "\"hello\\ud800\\ud800world\"" ,
ptr : new ( string ) ,
out : "hello\ufffd\ufffdworld" ,
2013-02-14 14:56:01 -05:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : "\"hello\xed\xa0\x80\xed\xb0\x80world\"" ,
ptr : new ( string ) ,
out : "hello\ufffd\ufffd\ufffd\ufffd\ufffd\ufffdworld" ,
2013-02-14 14:56:01 -05:00
} ,
2014-10-01 16:24:17 -07:00
2016-03-08 12:41:35 -08:00
// Used to be issue 8305, but time.Time implements encoding.TextUnmarshaler so this works now.
2014-10-01 16:24:17 -07:00
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "2009-11-10T23:00:00Z": "hello world"} ` ,
ptr : new ( map [ time . Time ] string ) ,
out : map [ time . Time ] string { time . Date ( 2009 , 11 , 10 , 23 , 0 , 0 , 0 , time . UTC ) : "hello world" } ,
2016-03-08 12:41:35 -08:00
} ,
// issue 8305
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "2009-11-10T23:00:00Z": "hello world"} ` ,
ptr : new ( map [ Point ] string ) ,
err : & UnmarshalTypeError { Value : "object" , Type : reflect . TypeFor [ map [ Point ] string ] ( ) , Offset : 1 } ,
2016-03-08 12:41:35 -08:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "asdf": "hello world"} ` ,
ptr : new ( map [ unmarshaler ] string ) ,
err : & UnmarshalTypeError { Value : "object" , Type : reflect . TypeFor [ map [ unmarshaler ] string ] ( ) , Offset : 1 } ,
2014-10-01 16:24:17 -07:00
} ,
2016-05-23 12:21:57 -04:00
// related to issue 13783.
// Go 1.7 changed marshaling a slice of typed byte to use the methods on the byte type,
// similar to marshaling a slice of typed int.
// These tests check that, assuming the byte type also has valid decoding methods,
// either the old base64 string encoding or the new per-element encoding can be
// successfully unmarshaled. The custom unmarshalers were accessible in earlier
// versions of Go, even though the custom marshaler was not.
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` "AQID" ` ,
ptr : new ( [ ] byteWithMarshalJSON ) ,
out : [ ] byteWithMarshalJSON { 1 , 2 , 3 } ,
2016-05-23 12:21:57 -04:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` ["Z01","Z02","Z03"] ` ,
ptr : new ( [ ] byteWithMarshalJSON ) ,
out : [ ] byteWithMarshalJSON { 1 , 2 , 3 } ,
golden : true ,
2016-05-23 12:21:57 -04:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` "AQID" ` ,
ptr : new ( [ ] byteWithMarshalText ) ,
out : [ ] byteWithMarshalText { 1 , 2 , 3 } ,
2016-05-23 12:21:57 -04:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` ["Z01","Z02","Z03"] ` ,
ptr : new ( [ ] byteWithMarshalText ) ,
out : [ ] byteWithMarshalText { 1 , 2 , 3 } ,
golden : true ,
2016-05-23 12:21:57 -04:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` "AQID" ` ,
ptr : new ( [ ] byteWithPtrMarshalJSON ) ,
out : [ ] byteWithPtrMarshalJSON { 1 , 2 , 3 } ,
2016-05-23 12:21:57 -04:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` ["Z01","Z02","Z03"] ` ,
ptr : new ( [ ] byteWithPtrMarshalJSON ) ,
out : [ ] byteWithPtrMarshalJSON { 1 , 2 , 3 } ,
golden : true ,
2016-05-23 12:21:57 -04:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` "AQID" ` ,
ptr : new ( [ ] byteWithPtrMarshalText ) ,
out : [ ] byteWithPtrMarshalText { 1 , 2 , 3 } ,
2016-05-23 12:21:57 -04:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` ["Z01","Z02","Z03"] ` ,
ptr : new ( [ ] byteWithPtrMarshalText ) ,
out : [ ] byteWithPtrMarshalText { 1 , 2 , 3 } ,
golden : true ,
2016-05-23 12:21:57 -04:00
} ,
// ints work with the marshaler but not the base64 []byte case
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` ["Z01","Z02","Z03"] ` ,
ptr : new ( [ ] intWithMarshalJSON ) ,
out : [ ] intWithMarshalJSON { 1 , 2 , 3 } ,
golden : true ,
2016-05-23 12:21:57 -04:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` ["Z01","Z02","Z03"] ` ,
ptr : new ( [ ] intWithMarshalText ) ,
out : [ ] intWithMarshalText { 1 , 2 , 3 } ,
golden : true ,
2016-05-23 12:21:57 -04:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` ["Z01","Z02","Z03"] ` ,
ptr : new ( [ ] intWithPtrMarshalJSON ) ,
out : [ ] intWithPtrMarshalJSON { 1 , 2 , 3 } ,
golden : true ,
2016-05-23 12:21:57 -04:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` ["Z01","Z02","Z03"] ` ,
ptr : new ( [ ] intWithPtrMarshalText ) ,
out : [ ] intWithPtrMarshalText { 1 , 2 , 3 } ,
golden : true ,
} ,
{ CaseName : Name ( "" ) , in : ` 0.000001 ` , ptr : new ( float64 ) , out : 0.000001 , golden : true } ,
{ CaseName : Name ( "" ) , in : ` 1e-7 ` , ptr : new ( float64 ) , out : 1e-7 , golden : true } ,
{ CaseName : Name ( "" ) , in : ` 100000000000000000000 ` , ptr : new ( float64 ) , out : 100000000000000000000.0 , golden : true } ,
{ CaseName : Name ( "" ) , in : ` 1e+21 ` , ptr : new ( float64 ) , out : 1e21 , golden : true } ,
{ CaseName : Name ( "" ) , in : ` -0.000001 ` , ptr : new ( float64 ) , out : - 0.000001 , golden : true } ,
{ CaseName : Name ( "" ) , in : ` -1e-7 ` , ptr : new ( float64 ) , out : - 1e-7 , golden : true } ,
{ CaseName : Name ( "" ) , in : ` -100000000000000000000 ` , ptr : new ( float64 ) , out : - 100000000000000000000.0 , golden : true } ,
{ CaseName : Name ( "" ) , in : ` -1e+21 ` , ptr : new ( float64 ) , out : - 1e21 , golden : true } ,
{ CaseName : Name ( "" ) , in : ` 999999999999999900000 ` , ptr : new ( float64 ) , out : 999999999999999900000.0 , golden : true } ,
{ CaseName : Name ( "" ) , in : ` 9007199254740992 ` , ptr : new ( float64 ) , out : 9007199254740992.0 , golden : true } ,
{ CaseName : Name ( "" ) , in : ` 9007199254740993 ` , ptr : new ( float64 ) , out : 9007199254740992.0 , golden : false } ,
2016-01-18 16:26:05 +01:00
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "V": { "F2": "hello"}} ` ,
ptr : new ( VOuter ) ,
2016-01-18 16:26:05 +01:00
err : & UnmarshalTypeError {
Value : "string" ,
Struct : "V" ,
2018-10-28 01:44:09 +07:00
Field : "V.F2" ,
2023-07-31 15:18:12 -07:00
Type : reflect . TypeFor [ int32 ] ( ) ,
2016-01-18 16:26:05 +01:00
Offset : 20 ,
} ,
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "V": { "F4": { }, "F2": "hello"}} ` ,
ptr : new ( VOuter ) ,
2016-01-18 16:26:05 +01:00
err : & UnmarshalTypeError {
Value : "string" ,
Struct : "V" ,
2018-10-28 01:44:09 +07:00
Field : "V.F2" ,
2023-07-31 15:18:12 -07:00
Type : reflect . TypeFor [ int32 ] ( ) ,
2016-01-18 16:26:05 +01:00
Offset : 30 ,
} ,
} ,
2016-10-12 15:55:02 -04:00
// issue 15146.
// invalid inputs in wrongStringTests below.
2023-08-24 12:49:10 -07:00
{ CaseName : Name ( "" ) , in : ` { "B":"true"} ` , ptr : new ( B ) , out : B { true } , golden : true } ,
{ CaseName : Name ( "" ) , in : ` { "B":"false"} ` , ptr : new ( B ) , out : B { false } , golden : true } ,
{ CaseName : Name ( "" ) , in : ` { "B": "maybe"} ` , ptr : new ( B ) , err : errors . New ( ` json: invalid use of ,string struct tag, trying to unmarshal "maybe" into bool ` ) } ,
{ CaseName : Name ( "" ) , in : ` { "B": "tru"} ` , ptr : new ( B ) , err : errors . New ( ` json: invalid use of ,string struct tag, trying to unmarshal "tru" into bool ` ) } ,
{ CaseName : Name ( "" ) , in : ` { "B": "False"} ` , ptr : new ( B ) , err : errors . New ( ` json: invalid use of ,string struct tag, trying to unmarshal "False" into bool ` ) } ,
{ CaseName : Name ( "" ) , in : ` { "B": "null"} ` , ptr : new ( B ) , out : B { false } } ,
{ CaseName : Name ( "" ) , in : ` { "B": "nul"} ` , ptr : new ( B ) , err : errors . New ( ` json: invalid use of ,string struct tag, trying to unmarshal "nul" into bool ` ) } ,
{ CaseName : Name ( "" ) , in : ` { "B": [2, 3]} ` , ptr : new ( B ) , err : errors . New ( ` json: invalid use of ,string struct tag, trying to unmarshal unquoted value into bool ` ) } ,
2017-10-31 13:16:38 -07:00
// additional tests for disallowUnknownFields
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
2017-10-31 13:16:38 -07:00
in : ` {
"Level0" : 1 ,
"Level1b" : 2 ,
"Level1c" : 3 ,
"x" : 4 ,
"Level1a" : 5 ,
"LEVEL1B" : 6 ,
"e" : {
"Level1a" : 8 ,
"Level1b" : 9 ,
"Level1c" : 10 ,
"Level1d" : 11 ,
"x" : 12
} ,
"Loop1" : 13 ,
"Loop2" : 14 ,
"X" : 15 ,
"Y" : 16 ,
"Z" : 17 ,
"Q" : 18 ,
"extra" : true
} ` ,
go/printer, gofmt: tuned table alignment for better results
The go/printer (and thus gofmt) uses a heuristic to determine
whether to break alignment between elements of an expression
list which is spread across multiple lines. The heuristic only
kicked in if the entry sizes (character length) was above a
certain threshold (20) and the ratio between the previous and
current entry size was above a certain value (4).
This heuristic worked reasonably most of the time, but also
led to unfortunate breaks in many cases where a single entry
was suddenly much smaller (or larger) then the previous one.
The behavior of gofmt was sufficiently mysterious in some of
these situations that many issues were filed against it.
The simplest solution to address this problem is to remove
the heuristic altogether and have a programmer introduce
empty lines to force different alignments if it improves
readability. The problem with that approach is that the
places where it really matters, very long tables with many
(hundreds, or more) entries, may be machine-generated and
not "post-processed" by a human (e.g., unicode/utf8/tables.go).
If a single one of those entries is overlong, the result
would be that the alignment would force all comments or
values in key:value pairs to be adjusted to that overlong
value, making the table hard to read (e.g., that entry may
not even be visible on screen and all other entries seem
spaced out too wide).
Instead, we opted for a slightly improved heuristic that
behaves much better for "normal", human-written code.
1) The threshold is increased from 20 to 40. This disables
the heuristic for many common cases yet even if the alignment
is not "ideal", 40 is not that many characters per line with
todays screens, making it very likely that the entire line
remains "visible" in an editor.
2) Changed the heuristic to not simply look at the size ratio
between current and previous line, but instead considering the
geometric mean of the sizes of the previous (aligned) lines.
This emphasizes the "overall picture" of the previous lines,
rather than a single one (which might be an outlier).
3) Changed the ratio from 4 to 2.5. Now that we ignore sizes
below 40, a ratio of 4 would mean that a new entry would have
to be 4 times bigger (160) or smaller (10) before alignment
would be broken. A ratio of 2.5 seems more sensible.
Applied updated gofmt to all of src and misc. Also tested
against several former issues that complained about this
and verified that the output for the given examples is
satisfactory (added respective test cases).
Some of the files changed because they were not gofmt-ed
in the first place.
For #644.
For #7335.
For #10392.
(and probably more related issues)
Fixes #22852.
Change-Id: I5e48b3d3b157a5cf2d649833b7297b33f43a6f6e
2018-04-03 17:05:47 -07:00
ptr : new ( Top ) ,
err : fmt . Errorf ( "json: unknown field \"extra\"" ) ,
2017-10-31 13:16:38 -07:00
disallowUnknownFields : true ,
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
2017-10-31 13:16:38 -07:00
in : ` {
"Level0" : 1 ,
"Level1b" : 2 ,
"Level1c" : 3 ,
"x" : 4 ,
"Level1a" : 5 ,
"LEVEL1B" : 6 ,
"e" : {
"Level1a" : 8 ,
"Level1b" : 9 ,
"Level1c" : 10 ,
"Level1d" : 11 ,
"x" : 12 ,
"extra" : null
} ,
"Loop1" : 13 ,
"Loop2" : 14 ,
"X" : 15 ,
"Y" : 16 ,
"Z" : 17 ,
"Q" : 18
} ` ,
go/printer, gofmt: tuned table alignment for better results
The go/printer (and thus gofmt) uses a heuristic to determine
whether to break alignment between elements of an expression
list which is spread across multiple lines. The heuristic only
kicked in if the entry sizes (character length) was above a
certain threshold (20) and the ratio between the previous and
current entry size was above a certain value (4).
This heuristic worked reasonably most of the time, but also
led to unfortunate breaks in many cases where a single entry
was suddenly much smaller (or larger) then the previous one.
The behavior of gofmt was sufficiently mysterious in some of
these situations that many issues were filed against it.
The simplest solution to address this problem is to remove
the heuristic altogether and have a programmer introduce
empty lines to force different alignments if it improves
readability. The problem with that approach is that the
places where it really matters, very long tables with many
(hundreds, or more) entries, may be machine-generated and
not "post-processed" by a human (e.g., unicode/utf8/tables.go).
If a single one of those entries is overlong, the result
would be that the alignment would force all comments or
values in key:value pairs to be adjusted to that overlong
value, making the table hard to read (e.g., that entry may
not even be visible on screen and all other entries seem
spaced out too wide).
Instead, we opted for a slightly improved heuristic that
behaves much better for "normal", human-written code.
1) The threshold is increased from 20 to 40. This disables
the heuristic for many common cases yet even if the alignment
is not "ideal", 40 is not that many characters per line with
todays screens, making it very likely that the entire line
remains "visible" in an editor.
2) Changed the heuristic to not simply look at the size ratio
between current and previous line, but instead considering the
geometric mean of the sizes of the previous (aligned) lines.
This emphasizes the "overall picture" of the previous lines,
rather than a single one (which might be an outlier).
3) Changed the ratio from 4 to 2.5. Now that we ignore sizes
below 40, a ratio of 4 would mean that a new entry would have
to be 4 times bigger (160) or smaller (10) before alignment
would be broken. A ratio of 2.5 seems more sensible.
Applied updated gofmt to all of src and misc. Also tested
against several former issues that complained about this
and verified that the output for the given examples is
satisfactory (added respective test cases).
Some of the files changed because they were not gofmt-ed
in the first place.
For #644.
For #7335.
For #10392.
(and probably more related issues)
Fixes #22852.
Change-Id: I5e48b3d3b157a5cf2d649833b7297b33f43a6f6e
2018-04-03 17:05:47 -07:00
ptr : new ( Top ) ,
err : fmt . Errorf ( "json: unknown field \"extra\"" ) ,
2017-10-31 13:16:38 -07:00
disallowUnknownFields : true ,
} ,
2018-08-28 15:56:10 +00:00
// issue 26444
// UnmarshalTypeError without field & struct values
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "data": { "test1": "bob", "test2": 123}} ` ,
ptr : new ( mapStringToStringData ) ,
err : & UnmarshalTypeError { Value : "number" , Type : reflect . TypeFor [ string ] ( ) , Offset : 37 , Struct : "mapStringToStringData" , Field : "data" } ,
2018-08-28 15:56:10 +00:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "data": { "test1": 123, "test2": "bob"}} ` ,
ptr : new ( mapStringToStringData ) ,
err : & UnmarshalTypeError { Value : "number" , Type : reflect . TypeFor [ string ] ( ) , Offset : 21 , Struct : "mapStringToStringData" , Field : "data" } ,
2018-08-28 15:56:10 +00:00
} ,
2018-09-11 22:09:00 +02:00
// trying to decode JSON arrays or objects via TextUnmarshaler
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` [1, 2, 3] ` ,
ptr : new ( MustNotUnmarshalText ) ,
err : & UnmarshalTypeError { Value : "array" , Type : reflect . TypeFor [ * MustNotUnmarshalText ] ( ) , Offset : 1 } ,
2018-09-11 22:09:00 +02:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "foo": "bar"} ` ,
ptr : new ( MustNotUnmarshalText ) ,
err : & UnmarshalTypeError { Value : "object" , Type : reflect . TypeFor [ * MustNotUnmarshalText ] ( ) , Offset : 1 } ,
2018-09-11 22:09:00 +02:00
} ,
2018-10-28 01:44:09 +07:00
// #22369
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "PP": { "T": { "Y": "bad-type"}}} ` ,
ptr : new ( P ) ,
2018-10-28 01:44:09 +07:00
err : & UnmarshalTypeError {
Value : "string" ,
Struct : "T" ,
Field : "PP.T.Y" ,
2023-07-31 15:18:12 -07:00
Type : reflect . TypeFor [ int ] ( ) ,
2018-10-28 01:44:09 +07:00
Offset : 29 ,
} ,
} ,
encoding/json: fix performance regression in the decoder
In golang.org/cl/145218, a feature was added where the JSON decoder
would keep track of the entire path to a field when reporting an
UnmarshalTypeError.
However, we all failed to check if this affected the benchmarks - myself
included, as a reviewer. Below are the numbers comparing the CL's parent
with itself, once it was merged:
name old time/op new time/op delta
CodeDecoder-8 12.9ms ± 1% 28.2ms ± 2% +119.33% (p=0.002 n=6+6)
name old speed new speed delta
CodeDecoder-8 151MB/s ± 1% 69MB/s ± 3% -54.40% (p=0.002 n=6+6)
name old alloc/op new alloc/op delta
CodeDecoder-8 2.74MB ± 0% 109.39MB ± 0% +3891.83% (p=0.002 n=6+6)
name old allocs/op new allocs/op delta
CodeDecoder-8 77.5k ± 0% 168.5k ± 0% +117.30% (p=0.004 n=6+5)
The reason why the decoder got twice as slow is because it now allocated
~40x as many objects, which puts a lot of pressure on the garbage
collector.
The reason is that the CL concatenated strings every time a nested field
was decoded. In other words, practically every field generated garbage
when decoded. This is hugely wasteful, especially considering that the
vast majority of JSON decoding inputs won't return UnmarshalTypeError.
Instead, use a stack of fields, and make sure to always use the same
backing array, to ensure we only need to grow the slice to the maximum
depth once.
The original CL also introduced a bug. The field stack string wasn't
reset to its original state when reaching "d.opcode == scanEndObject",
so the last field in a decoded struct could leak. For example, an added
test decodes a list of structs, and encoding/json before this CL would
fail:
got: cannot unmarshal string into Go struct field T.Ts.Y.Y.Y of type int
want: cannot unmarshal string into Go struct field T.Ts.Y of type int
To fix that, simply reset the stack after decoding every field, even if
it's the last.
Below is the original performance versus this CL. There's a tiny
performance hit, probably due to the append for every decoded field, but
at least we're back to the usual ~150MB/s.
name old time/op new time/op delta
CodeDecoder-8 12.9ms ± 1% 13.0ms ± 1% +1.25% (p=0.009 n=6+6)
name old speed new speed delta
CodeDecoder-8 151MB/s ± 1% 149MB/s ± 1% -1.24% (p=0.009 n=6+6)
name old alloc/op new alloc/op delta
CodeDecoder-8 2.74MB ± 0% 2.74MB ± 0% +0.00% (p=0.002 n=6+6)
name old allocs/op new allocs/op delta
CodeDecoder-8 77.5k ± 0% 77.5k ± 0% +0.00% (p=0.002 n=6+6)
Finally, make all of these benchmarks report allocs by default. The
decoder ones are pretty sensitive to generated garbage, so ReportAllocs
would have made the performance regression more obvious.
Change-Id: I67b50f86b2e72f55539429450c67bfb1a9464b67
Reviewed-on: https://go-review.googlesource.com/c/go/+/167978
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
2019-03-17 22:45:30 +00:00
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "Ts": [ { "Y": 1}, { "Y": 2}, { "Y": "bad-type"}]} ` ,
ptr : new ( PP ) ,
encoding/json: fix performance regression in the decoder
In golang.org/cl/145218, a feature was added where the JSON decoder
would keep track of the entire path to a field when reporting an
UnmarshalTypeError.
However, we all failed to check if this affected the benchmarks - myself
included, as a reviewer. Below are the numbers comparing the CL's parent
with itself, once it was merged:
name old time/op new time/op delta
CodeDecoder-8 12.9ms ± 1% 28.2ms ± 2% +119.33% (p=0.002 n=6+6)
name old speed new speed delta
CodeDecoder-8 151MB/s ± 1% 69MB/s ± 3% -54.40% (p=0.002 n=6+6)
name old alloc/op new alloc/op delta
CodeDecoder-8 2.74MB ± 0% 109.39MB ± 0% +3891.83% (p=0.002 n=6+6)
name old allocs/op new allocs/op delta
CodeDecoder-8 77.5k ± 0% 168.5k ± 0% +117.30% (p=0.004 n=6+5)
The reason why the decoder got twice as slow is because it now allocated
~40x as many objects, which puts a lot of pressure on the garbage
collector.
The reason is that the CL concatenated strings every time a nested field
was decoded. In other words, practically every field generated garbage
when decoded. This is hugely wasteful, especially considering that the
vast majority of JSON decoding inputs won't return UnmarshalTypeError.
Instead, use a stack of fields, and make sure to always use the same
backing array, to ensure we only need to grow the slice to the maximum
depth once.
The original CL also introduced a bug. The field stack string wasn't
reset to its original state when reaching "d.opcode == scanEndObject",
so the last field in a decoded struct could leak. For example, an added
test decodes a list of structs, and encoding/json before this CL would
fail:
got: cannot unmarshal string into Go struct field T.Ts.Y.Y.Y of type int
want: cannot unmarshal string into Go struct field T.Ts.Y of type int
To fix that, simply reset the stack after decoding every field, even if
it's the last.
Below is the original performance versus this CL. There's a tiny
performance hit, probably due to the append for every decoded field, but
at least we're back to the usual ~150MB/s.
name old time/op new time/op delta
CodeDecoder-8 12.9ms ± 1% 13.0ms ± 1% +1.25% (p=0.009 n=6+6)
name old speed new speed delta
CodeDecoder-8 151MB/s ± 1% 149MB/s ± 1% -1.24% (p=0.009 n=6+6)
name old alloc/op new alloc/op delta
CodeDecoder-8 2.74MB ± 0% 2.74MB ± 0% +0.00% (p=0.002 n=6+6)
name old allocs/op new allocs/op delta
CodeDecoder-8 77.5k ± 0% 77.5k ± 0% +0.00% (p=0.002 n=6+6)
Finally, make all of these benchmarks report allocs by default. The
decoder ones are pretty sensitive to generated garbage, so ReportAllocs
would have made the performance regression more obvious.
Change-Id: I67b50f86b2e72f55539429450c67bfb1a9464b67
Reviewed-on: https://go-review.googlesource.com/c/go/+/167978
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
2019-03-17 22:45:30 +00:00
err : & UnmarshalTypeError {
Value : "string" ,
Struct : "T" ,
Field : "Ts.Y" ,
2023-07-31 15:18:12 -07:00
Type : reflect . TypeFor [ int ] ( ) ,
encoding/json: fix performance regression in the decoder
In golang.org/cl/145218, a feature was added where the JSON decoder
would keep track of the entire path to a field when reporting an
UnmarshalTypeError.
However, we all failed to check if this affected the benchmarks - myself
included, as a reviewer. Below are the numbers comparing the CL's parent
with itself, once it was merged:
name old time/op new time/op delta
CodeDecoder-8 12.9ms ± 1% 28.2ms ± 2% +119.33% (p=0.002 n=6+6)
name old speed new speed delta
CodeDecoder-8 151MB/s ± 1% 69MB/s ± 3% -54.40% (p=0.002 n=6+6)
name old alloc/op new alloc/op delta
CodeDecoder-8 2.74MB ± 0% 109.39MB ± 0% +3891.83% (p=0.002 n=6+6)
name old allocs/op new allocs/op delta
CodeDecoder-8 77.5k ± 0% 168.5k ± 0% +117.30% (p=0.004 n=6+5)
The reason why the decoder got twice as slow is because it now allocated
~40x as many objects, which puts a lot of pressure on the garbage
collector.
The reason is that the CL concatenated strings every time a nested field
was decoded. In other words, practically every field generated garbage
when decoded. This is hugely wasteful, especially considering that the
vast majority of JSON decoding inputs won't return UnmarshalTypeError.
Instead, use a stack of fields, and make sure to always use the same
backing array, to ensure we only need to grow the slice to the maximum
depth once.
The original CL also introduced a bug. The field stack string wasn't
reset to its original state when reaching "d.opcode == scanEndObject",
so the last field in a decoded struct could leak. For example, an added
test decodes a list of structs, and encoding/json before this CL would
fail:
got: cannot unmarshal string into Go struct field T.Ts.Y.Y.Y of type int
want: cannot unmarshal string into Go struct field T.Ts.Y of type int
To fix that, simply reset the stack after decoding every field, even if
it's the last.
Below is the original performance versus this CL. There's a tiny
performance hit, probably due to the append for every decoded field, but
at least we're back to the usual ~150MB/s.
name old time/op new time/op delta
CodeDecoder-8 12.9ms ± 1% 13.0ms ± 1% +1.25% (p=0.009 n=6+6)
name old speed new speed delta
CodeDecoder-8 151MB/s ± 1% 149MB/s ± 1% -1.24% (p=0.009 n=6+6)
name old alloc/op new alloc/op delta
CodeDecoder-8 2.74MB ± 0% 2.74MB ± 0% +0.00% (p=0.002 n=6+6)
name old allocs/op new allocs/op delta
CodeDecoder-8 77.5k ± 0% 77.5k ± 0% +0.00% (p=0.002 n=6+6)
Finally, make all of these benchmarks report allocs by default. The
decoder ones are pretty sensitive to generated garbage, so ReportAllocs
would have made the performance regression more obvious.
Change-Id: I67b50f86b2e72f55539429450c67bfb1a9464b67
Reviewed-on: https://go-review.googlesource.com/c/go/+/167978
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
2019-03-17 22:45:30 +00:00
Offset : 29 ,
} ,
} ,
2019-09-16 19:46:12 +00:00
// #14702
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` invalid ` ,
ptr : new ( Number ) ,
2019-09-16 19:46:12 +00:00
err : & SyntaxError {
msg : "invalid character 'i' looking for beginning of value" ,
Offset : 1 ,
} ,
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` "invalid" ` ,
ptr : new ( Number ) ,
err : fmt . Errorf ( "json: invalid number literal, trying to unmarshal %q into Number" , ` "invalid" ` ) ,
2019-09-16 19:46:12 +00:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "A":"invalid"} ` ,
ptr : new ( struct { A Number } ) ,
err : fmt . Errorf ( "json: invalid number literal, trying to unmarshal %q into Number" , ` "invalid" ` ) ,
2019-09-16 19:46:12 +00:00
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "A":"invalid"} ` ,
2019-09-16 19:46:12 +00:00
ptr : new ( struct {
A Number ` json:",string" `
} ) ,
err : fmt . Errorf ( "json: invalid use of ,string struct tag, trying to unmarshal %q into json.Number" , ` invalid ` ) ,
} ,
{
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "A":"invalid"} ` ,
ptr : new ( map [ string ] Number ) ,
err : fmt . Errorf ( "json: invalid number literal, trying to unmarshal %q into Number" , ` "invalid" ` ) ,
2019-09-16 19:46:12 +00:00
} ,
2009-11-30 13:55:09 -08:00
}
2010-04-21 16:40:53 -07:00
func TestMarshal ( t * testing . T ) {
b , err := Marshal ( allValue )
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Marshal error: %v" , err )
2010-04-21 16:40:53 -07:00
}
if string ( b ) != allValueCompact {
2023-08-24 12:49:10 -07:00
t . Errorf ( "Marshal:" )
2010-04-21 16:40:53 -07:00
diff ( t , b , [ ] byte ( allValueCompact ) )
return
}
2009-11-30 13:55:09 -08:00
2010-04-21 16:40:53 -07:00
b , err = Marshal ( pallValue )
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Marshal error: %v" , err )
2010-04-21 16:40:53 -07:00
}
if string ( b ) != pallValueCompact {
2023-08-24 12:49:10 -07:00
t . Errorf ( "Marshal:" )
2010-04-21 16:40:53 -07:00
diff ( t , b , [ ] byte ( pallValueCompact ) )
return
}
2009-11-30 13:55:09 -08:00
}
2023-08-24 12:49:10 -07:00
func TestMarshalInvalidUTF8 ( t * testing . T ) {
tests := [ ] struct {
CaseName
in string
want string
} {
{ Name ( "" ) , "hello\xffworld" , ` "hello\ufffdworld" ` } ,
{ Name ( "" ) , "" , ` "" ` } ,
{ Name ( "" ) , "\xff" , ` "\ufffd" ` } ,
{ Name ( "" ) , "\xff\xff" , ` "\ufffd\ufffd" ` } ,
{ Name ( "" ) , "a\xffb" , ` "a\ufffdb" ` } ,
{ Name ( "" ) , "\xe6\x97\xa5\xe6\x9c\xac\xff\xaa\x9e" , ` "日本\ufffd\ufffd\ufffd" ` } ,
}
for _ , tt := range tests {
t . Run ( tt . Name , func ( t * testing . T ) {
got , err := Marshal ( tt . in )
if string ( got ) != tt . want || err != nil {
t . Errorf ( "%s: Marshal(%q):\n\tgot: (%q, %v)\n\twant: (%q, nil)" , tt . Where , tt . in , got , err , tt . want )
}
} )
2010-12-13 15:51:11 -05:00
}
}
2012-06-25 17:36:09 -04:00
func TestMarshalNumberZeroVal ( t * testing . T ) {
var n Number
out , err := Marshal ( n )
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Marshal error: %v" , err )
2012-06-25 17:36:09 -04:00
}
2023-08-24 12:49:10 -07:00
got := string ( out )
if got != "0" {
t . Fatalf ( "Marshal: got %s, want 0" , got )
2012-06-25 17:36:09 -04:00
}
}
encoding/json: faster encoding
The old code was caching per-type struct field info. Instead,
cache type-specific encoding funcs, tailored for that
particular type to avoid unnecessary reflection at runtime.
Once the machine is built once, future encodings of that type
just run the func.
benchmark old ns/op new ns/op delta
BenchmarkCodeEncoder 48424939 36975320 -23.64%
benchmark old MB/s new MB/s speedup
BenchmarkCodeEncoder 40.07 52.48 1.31x
Additionally, the numbers seem stable now at ~52 MB/s, whereas
the numbers for the old code were all over the place: 11 MB/s,
40 MB/s, 13 MB/s, 39 MB/s, etc. In the benchmark above I compared
against the best I saw the old code do.
R=rsc, adg
CC=gobot, golang-dev, r
https://golang.org/cl/9129044
2013-08-09 09:46:47 -07:00
func TestMarshalEmbeds ( t * testing . T ) {
top := & Top {
Level0 : 1 ,
Embed0 : Embed0 {
Level1b : 2 ,
Level1c : 3 ,
} ,
Embed0a : & Embed0a {
Level1a : 5 ,
Level1b : 6 ,
} ,
Embed0b : & Embed0b {
Level1a : 8 ,
Level1b : 9 ,
Level1c : 10 ,
Level1d : 11 ,
Level1e : 12 ,
} ,
Loop : Loop {
Loop1 : 13 ,
Loop2 : 14 ,
} ,
Embed0p : Embed0p {
Point : image . Point { X : 15 , Y : 16 } ,
} ,
Embed0q : Embed0q {
Point : Point { Z : 17 } ,
} ,
2015-08-28 10:17:05 +02:00
embed : embed {
Q : 18 ,
} ,
encoding/json: faster encoding
The old code was caching per-type struct field info. Instead,
cache type-specific encoding funcs, tailored for that
particular type to avoid unnecessary reflection at runtime.
Once the machine is built once, future encodings of that type
just run the func.
benchmark old ns/op new ns/op delta
BenchmarkCodeEncoder 48424939 36975320 -23.64%
benchmark old MB/s new MB/s speedup
BenchmarkCodeEncoder 40.07 52.48 1.31x
Additionally, the numbers seem stable now at ~52 MB/s, whereas
the numbers for the old code were all over the place: 11 MB/s,
40 MB/s, 13 MB/s, 39 MB/s, etc. In the benchmark above I compared
against the best I saw the old code do.
R=rsc, adg
CC=gobot, golang-dev, r
https://golang.org/cl/9129044
2013-08-09 09:46:47 -07:00
}
2023-08-24 12:49:10 -07:00
got , err := Marshal ( top )
encoding/json: faster encoding
The old code was caching per-type struct field info. Instead,
cache type-specific encoding funcs, tailored for that
particular type to avoid unnecessary reflection at runtime.
Once the machine is built once, future encodings of that type
just run the func.
benchmark old ns/op new ns/op delta
BenchmarkCodeEncoder 48424939 36975320 -23.64%
benchmark old MB/s new MB/s speedup
BenchmarkCodeEncoder 40.07 52.48 1.31x
Additionally, the numbers seem stable now at ~52 MB/s, whereas
the numbers for the old code were all over the place: 11 MB/s,
40 MB/s, 13 MB/s, 39 MB/s, etc. In the benchmark above I compared
against the best I saw the old code do.
R=rsc, adg
CC=gobot, golang-dev, r
https://golang.org/cl/9129044
2013-08-09 09:46:47 -07:00
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Marshal error: %v" , err )
encoding/json: faster encoding
The old code was caching per-type struct field info. Instead,
cache type-specific encoding funcs, tailored for that
particular type to avoid unnecessary reflection at runtime.
Once the machine is built once, future encodings of that type
just run the func.
benchmark old ns/op new ns/op delta
BenchmarkCodeEncoder 48424939 36975320 -23.64%
benchmark old MB/s new MB/s speedup
BenchmarkCodeEncoder 40.07 52.48 1.31x
Additionally, the numbers seem stable now at ~52 MB/s, whereas
the numbers for the old code were all over the place: 11 MB/s,
40 MB/s, 13 MB/s, 39 MB/s, etc. In the benchmark above I compared
against the best I saw the old code do.
R=rsc, adg
CC=gobot, golang-dev, r
https://golang.org/cl/9129044
2013-08-09 09:46:47 -07:00
}
2015-08-28 10:17:05 +02:00
want := "{\"Level0\":1,\"Level1b\":2,\"Level1c\":3,\"Level1a\":5,\"LEVEL1B\":6,\"e\":{\"Level1a\":8,\"Level1b\":9,\"Level1c\":10,\"Level1d\":11,\"x\":12},\"Loop1\":13,\"Loop2\":14,\"X\":15,\"Y\":16,\"Z\":17,\"Q\":18}"
2023-08-24 12:49:10 -07:00
if string ( got ) != want {
t . Errorf ( "Marshal:\n\tgot: %s\n\twant: %s" , got , want )
encoding/json: faster encoding
The old code was caching per-type struct field info. Instead,
cache type-specific encoding funcs, tailored for that
particular type to avoid unnecessary reflection at runtime.
Once the machine is built once, future encodings of that type
just run the func.
benchmark old ns/op new ns/op delta
BenchmarkCodeEncoder 48424939 36975320 -23.64%
benchmark old MB/s new MB/s speedup
BenchmarkCodeEncoder 40.07 52.48 1.31x
Additionally, the numbers seem stable now at ~52 MB/s, whereas
the numbers for the old code were all over the place: 11 MB/s,
40 MB/s, 13 MB/s, 39 MB/s, etc. In the benchmark above I compared
against the best I saw the old code do.
R=rsc, adg
CC=gobot, golang-dev, r
https://golang.org/cl/9129044
2013-08-09 09:46:47 -07:00
}
}
2019-02-08 17:52:08 +01:00
func equalError ( a , b error ) bool {
2023-08-24 12:49:10 -07:00
if a == nil || b == nil {
return a == nil && b == nil
2019-02-08 17:52:08 +01:00
}
return a . Error ( ) == b . Error ( )
}
2010-04-21 16:40:53 -07:00
func TestUnmarshal ( t * testing . T ) {
2023-08-24 12:49:10 -07:00
for _ , tt := range unmarshalTests {
t . Run ( tt . Name , func ( t * testing . T ) {
in := [ ] byte ( tt . in )
var scan scanner
if err := checkValid ( in , & scan ) ; err != nil {
if ! equalError ( err , tt . err ) {
t . Fatalf ( "%s: checkValid error: %#v" , tt . Where , err )
}
}
if tt . ptr == nil {
return
2010-08-03 17:05:00 -07:00
}
2012-05-29 18:02:40 -04:00
2023-08-24 12:49:10 -07:00
typ := reflect . TypeOf ( tt . ptr )
if typ . Kind ( ) != reflect . Pointer {
t . Fatalf ( "%s: unmarshalTest.ptr %T is not a pointer type" , tt . Where , tt . ptr )
2012-05-29 18:02:40 -04:00
}
2023-08-24 12:49:10 -07:00
typ = typ . Elem ( )
// v = new(right-type)
v := reflect . New ( typ )
if ! reflect . DeepEqual ( tt . ptr , v . Interface ( ) ) {
// There's no reason for ptr to point to non-zero data,
// as we decode into new(right-type), so the data is
// discarded.
// This can easily mean tests that silently don't test
// what they should. To test decoding into existing
// data, see TestPrefilled.
t . Fatalf ( "%s: unmarshalTest.ptr %#v is not a pointer to a zero value" , tt . Where , tt . ptr )
2016-05-23 12:21:57 -04:00
}
2023-08-24 12:49:10 -07:00
dec := NewDecoder ( bytes . NewReader ( in ) )
2012-06-25 17:36:09 -04:00
if tt . useNumber {
dec . UseNumber ( )
}
2023-08-24 12:49:10 -07:00
if tt . disallowUnknownFields {
dec . DisallowUnknownFields ( )
2012-05-29 18:02:40 -04:00
}
2023-08-24 12:49:10 -07:00
if err := dec . Decode ( v . Interface ( ) ) ; ! equalError ( err , tt . err ) {
t . Fatalf ( "%s: Decode error:\n\tgot: %v\n\twant: %v" , tt . Where , err , tt . err )
} else if err != nil {
return
2012-05-29 18:02:40 -04:00
}
2023-08-24 12:49:10 -07:00
if got := v . Elem ( ) . Interface ( ) ; ! reflect . DeepEqual ( got , tt . out ) {
gotJSON , _ := Marshal ( got )
wantJSON , _ := Marshal ( tt . out )
t . Fatalf ( "%s: Decode:\n\tgot: %#+v\n\twant: %#+v\n\n\tgotJSON: %s\n\twantJSON: %s" , tt . Where , got , tt . out , gotJSON , wantJSON )
}
// Check round trip also decodes correctly.
if tt . err == nil {
enc , err := Marshal ( v . Interface ( ) )
if err != nil {
t . Fatalf ( "%s: Marshal error after roundtrip: %v" , tt . Where , err )
}
if tt . golden && ! bytes . Equal ( enc , in ) {
t . Errorf ( "%s: Marshal:\n\tgot: %s\n\twant: %s" , tt . Where , enc , in )
}
vv := reflect . New ( reflect . TypeOf ( tt . ptr ) . Elem ( ) )
dec = NewDecoder ( bytes . NewReader ( enc ) )
if tt . useNumber {
dec . UseNumber ( )
}
if err := dec . Decode ( vv . Interface ( ) ) ; err != nil {
t . Fatalf ( "%s: Decode(%#q) error after roundtrip: %v" , tt . Where , enc , err )
}
if ! reflect . DeepEqual ( v . Elem ( ) . Interface ( ) , vv . Elem ( ) . Interface ( ) ) {
t . Fatalf ( "%s: Decode:\n\tgot: %#+v\n\twant: %#+v\n\n\tgotJSON: %s\n\twantJSON: %s" ,
tt . Where , v . Elem ( ) . Interface ( ) , vv . Elem ( ) . Interface ( ) ,
stripWhitespace ( string ( enc ) ) , stripWhitespace ( string ( in ) ) )
}
}
} )
2010-04-21 16:40:53 -07:00
}
2009-11-30 13:55:09 -08:00
}
2010-04-21 16:40:53 -07:00
func TestUnmarshalMarshal ( t * testing . T ) {
2011-03-26 11:25:22 -07:00
initBig ( )
2021-12-01 12:15:45 -05:00
var v any
2010-04-21 16:40:53 -07:00
if err := Unmarshal ( jsonBig , & v ) ; err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Unmarshal error: %v" , err )
2010-04-21 16:40:53 -07:00
}
b , err := Marshal ( v )
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Marshal error: %v" , err )
2010-04-21 16:40:53 -07:00
}
2013-01-07 10:03:49 +11:00
if ! bytes . Equal ( jsonBig , b ) {
2023-08-24 12:49:10 -07:00
t . Errorf ( "Marshal:" )
2010-04-21 16:40:53 -07:00
diff ( t , b , jsonBig )
return
}
2009-11-30 13:55:09 -08:00
}
2012-06-25 17:36:09 -04:00
// Independent of Decode, basic coverage of the accessors in Number
func TestNumberAccessors ( t * testing . T ) {
2023-08-24 12:49:10 -07:00
tests := [ ] struct {
CaseName
in string
i int64
intErr string
f float64
floatErr string
} {
{ CaseName : Name ( "" ) , in : "-1.23e1" , intErr : "strconv.ParseInt: parsing \"-1.23e1\": invalid syntax" , f : - 1.23e1 } ,
{ CaseName : Name ( "" ) , in : "-12" , i : - 12 , f : - 12.0 } ,
{ CaseName : Name ( "" ) , in : "1e1000" , intErr : "strconv.ParseInt: parsing \"1e1000\": invalid syntax" , floatErr : "strconv.ParseFloat: parsing \"1e1000\": value out of range" } ,
}
for _ , tt := range tests {
t . Run ( tt . Name , func ( t * testing . T ) {
n := Number ( tt . in )
if got := n . String ( ) ; got != tt . in {
t . Errorf ( "%s: Number(%q).String() = %s, want %s" , tt . Where , tt . in , got , tt . in )
}
if i , err := n . Int64 ( ) ; err == nil && tt . intErr == "" && i != tt . i {
t . Errorf ( "%s: Number(%q).Int64() = %d, want %d" , tt . Where , tt . in , i , tt . i )
} else if ( err == nil && tt . intErr != "" ) || ( err != nil && err . Error ( ) != tt . intErr ) {
t . Errorf ( "%s: Number(%q).Int64() error:\n\tgot: %v\n\twant: %v" , tt . Where , tt . in , err , tt . intErr )
}
if f , err := n . Float64 ( ) ; err == nil && tt . floatErr == "" && f != tt . f {
t . Errorf ( "%s: Number(%q).Float64() = %g, want %g" , tt . Where , tt . in , f , tt . f )
} else if ( err == nil && tt . floatErr != "" ) || ( err != nil && err . Error ( ) != tt . floatErr ) {
t . Errorf ( "%s: Number(%q).Float64() error:\n\tgot %v\n\twant: %v" , tt . Where , tt . in , err , tt . floatErr )
}
} )
2012-06-25 17:36:09 -04:00
}
}
2011-02-23 11:32:29 -05:00
func TestLargeByteSlice ( t * testing . T ) {
s0 := make ( [ ] byte , 2000 )
for i := range s0 {
s0 [ i ] = byte ( i )
}
b , err := Marshal ( s0 )
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Marshal error: %v" , err )
2011-02-23 11:32:29 -05:00
}
var s1 [ ] byte
if err := Unmarshal ( b , & s1 ) ; err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Unmarshal error: %v" , err )
2011-02-23 11:32:29 -05:00
}
2013-01-07 10:03:49 +11:00
if ! bytes . Equal ( s0 , s1 ) {
2023-08-24 12:49:10 -07:00
t . Errorf ( "Marshal:" )
2011-02-23 11:32:29 -05:00
diff ( t , s0 , s1 )
}
}
2010-04-27 10:24:00 -07:00
type Xint struct {
X int
}
func TestUnmarshalInterface ( t * testing . T ) {
var xint Xint
2021-12-01 12:15:45 -05:00
var i any = & xint
2010-04-27 10:24:00 -07:00
if err := Unmarshal ( [ ] byte ( ` { "X":1} ` ) , & i ) ; err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Unmarshal error: %v" , err )
2010-04-27 10:24:00 -07:00
}
if xint . X != 1 {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "xint.X = %d, want 1" , xint . X )
2010-04-27 10:24:00 -07:00
}
}
func TestUnmarshalPtrPtr ( t * testing . T ) {
var xint Xint
pxint := & xint
if err := Unmarshal ( [ ] byte ( ` { "X":1} ` ) , & pxint ) ; err != nil {
t . Fatalf ( "Unmarshal: %v" , err )
}
if xint . X != 1 {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "xint.X = %d, want 1" , xint . X )
2010-04-27 10:24:00 -07:00
}
}
2011-07-14 13:30:08 +10:00
func TestEscape ( t * testing . T ) {
2013-07-12 14:35:55 +10:00
const input = ` "foobar"<html> ` + " [\u2028 \u2029]"
2023-08-24 12:49:10 -07:00
const want = ` "\"foobar\"\u003chtml\u003e [\u2028 \u2029]" `
got , err := Marshal ( input )
2011-07-14 13:30:08 +10:00
if err != nil {
t . Fatalf ( "Marshal error: %v" , err )
}
2023-08-24 12:49:10 -07:00
if string ( got ) != want {
t . Errorf ( "Marshal(%#q):\n\tgot: %s\n\twant: %s" , input , got , want )
2011-07-14 13:30:08 +10:00
}
}
2011-12-15 10:02:47 -08:00
// If people misuse the ,string modifier, the error message should be
// helpful, telling the user that they're doing it wrong.
func TestErrorMessageFromMisusedString ( t * testing . T ) {
2023-08-24 12:49:10 -07:00
// WrongString is a struct that's misusing the ,string modifier.
type WrongString struct {
Message string ` json:"result,string" `
2011-12-15 10:02:47 -08:00
}
2023-08-24 12:49:10 -07:00
tests := [ ] struct {
CaseName
in , err string
} {
{ Name ( "" ) , ` { "result":"x"} ` , ` json: invalid use of ,string struct tag, trying to unmarshal "x" into string ` } ,
{ Name ( "" ) , ` { "result":"foo"} ` , ` json: invalid use of ,string struct tag, trying to unmarshal "foo" into string ` } ,
{ Name ( "" ) , ` { "result":"123"} ` , ` json: invalid use of ,string struct tag, trying to unmarshal "123" into string ` } ,
{ Name ( "" ) , ` { "result":123} ` , ` json: invalid use of ,string struct tag, trying to unmarshal unquoted value into string ` } ,
{ Name ( "" ) , ` { "result":"\""} ` , ` json: invalid use of ,string struct tag, trying to unmarshal "\"" into string ` } ,
{ Name ( "" ) , ` { "result":"\"foo"} ` , ` json: invalid use of ,string struct tag, trying to unmarshal "\"foo" into string ` } ,
}
for _ , tt := range tests {
t . Run ( tt . Name , func ( t * testing . T ) {
r := strings . NewReader ( tt . in )
var s WrongString
err := NewDecoder ( r ) . Decode ( & s )
got := fmt . Sprintf ( "%v" , err )
if got != tt . err {
t . Errorf ( "%s: Decode error:\n\tgot: %s\n\twant: %s" , tt . Where , got , tt . err )
}
} )
2010-04-21 16:40:53 -07:00
}
2009-11-30 13:55:09 -08:00
}
2010-04-21 16:40:53 -07:00
type All struct {
Bool bool
Int int
Int8 int8
Int16 int16
Int32 int32
Int64 int64
Uint uint
Uint8 uint8
Uint16 uint16
Uint32 uint32
Uint64 uint64
Uintptr uintptr
Float32 float32
Float64 float64
2009-11-30 13:55:09 -08:00
2011-08-26 12:27:33 +04:00
Foo string ` json:"bar" `
Foo2 string ` json:"bar2,dummyopt" `
2009-11-30 13:55:09 -08:00
2017-11-08 07:16:07 -08:00
IntStr int64 ` json:",string" `
UintptrStr uintptr ` json:",string" `
2011-08-29 12:46:32 -07:00
2010-04-21 16:40:53 -07:00
PBool * bool
PInt * int
PInt8 * int8
PInt16 * int16
PInt32 * int32
PInt64 * int64
PUint * uint
PUint8 * uint8
PUint16 * uint16
PUint32 * uint32
PUint64 * uint64
PUintptr * uintptr
PFloat32 * float32
PFloat64 * float64
String string
PString * string
2009-11-30 13:55:09 -08:00
2010-04-21 16:40:53 -07:00
Map map [ string ] Small
MapP map [ string ] * Small
PMap * map [ string ] Small
PMapP * map [ string ] * Small
EmptyMap map [ string ] Small
NilMap map [ string ] Small
Slice [ ] Small
SliceP [ ] * Small
PSlice * [ ] Small
PSliceP * [ ] * Small
EmptySlice [ ] Small
NilSlice [ ] Small
StringSlice [ ] string
ByteSlice [ ] byte
Small Small
PSmall * Small
PPSmall * * Small
2021-12-01 12:15:45 -05:00
Interface any
PInterface * any
2011-01-12 11:59:33 +11:00
unexported int
2009-11-30 13:55:09 -08:00
}
2010-04-21 16:40:53 -07:00
type Small struct {
Tag string
2009-11-30 13:55:09 -08:00
}
2010-04-21 16:40:53 -07:00
var allValue = All {
2017-11-08 07:16:07 -08:00
Bool : true ,
Int : 2 ,
Int8 : 3 ,
Int16 : 4 ,
Int32 : 5 ,
Int64 : 6 ,
Uint : 7 ,
Uint8 : 8 ,
Uint16 : 9 ,
Uint32 : 10 ,
Uint64 : 11 ,
Uintptr : 12 ,
Float32 : 14.1 ,
Float64 : 15.1 ,
Foo : "foo" ,
Foo2 : "foo2" ,
IntStr : 42 ,
UintptrStr : 44 ,
String : "16" ,
2010-04-21 16:40:53 -07:00
Map : map [ string ] Small {
2010-10-22 10:06:33 -07:00
"17" : { Tag : "tag17" } ,
"18" : { Tag : "tag18" } ,
2010-04-21 16:40:53 -07:00
} ,
MapP : map [ string ] * Small {
2011-12-02 14:14:25 -05:00
"19" : { Tag : "tag19" } ,
2010-04-21 16:40:53 -07:00
"20" : nil ,
} ,
EmptyMap : map [ string ] Small { } ,
2010-10-22 10:06:33 -07:00
Slice : [ ] Small { { Tag : "tag20" } , { Tag : "tag21" } } ,
2011-12-02 14:14:25 -05:00
SliceP : [ ] * Small { { Tag : "tag22" } , nil , { Tag : "tag23" } } ,
2010-04-21 16:40:53 -07:00
EmptySlice : [ ] Small { } ,
StringSlice : [ ] string { "str24" , "str25" , "str26" } ,
ByteSlice : [ ] byte { 27 , 28 , 29 } ,
Small : Small { Tag : "tag30" } ,
PSmall : & Small { Tag : "tag31" } ,
2011-01-19 23:09:00 -05:00
Interface : 5.2 ,
2009-11-30 13:55:09 -08:00
}
2010-04-21 16:40:53 -07:00
var pallValue = All {
PBool : & allValue . Bool ,
PInt : & allValue . Int ,
PInt8 : & allValue . Int8 ,
PInt16 : & allValue . Int16 ,
PInt32 : & allValue . Int32 ,
PInt64 : & allValue . Int64 ,
PUint : & allValue . Uint ,
PUint8 : & allValue . Uint8 ,
PUint16 : & allValue . Uint16 ,
PUint32 : & allValue . Uint32 ,
PUint64 : & allValue . Uint64 ,
PUintptr : & allValue . Uintptr ,
PFloat32 : & allValue . Float32 ,
PFloat64 : & allValue . Float64 ,
PString : & allValue . String ,
PMap : & allValue . Map ,
PMapP : & allValue . MapP ,
PSlice : & allValue . Slice ,
PSliceP : & allValue . SliceP ,
PPSmall : & allValue . PSmall ,
PInterface : & allValue . Interface ,
2009-11-30 13:55:09 -08:00
}
2010-04-21 16:40:53 -07:00
var allValueIndent = ` {
2010-04-27 10:24:00 -07:00
"Bool" : true ,
"Int" : 2 ,
"Int8" : 3 ,
"Int16" : 4 ,
"Int32" : 5 ,
"Int64" : 6 ,
"Uint" : 7 ,
"Uint8" : 8 ,
"Uint16" : 9 ,
"Uint32" : 10 ,
"Uint64" : 11 ,
"Uintptr" : 12 ,
"Float32" : 14.1 ,
"Float64" : 15.1 ,
2010-04-21 16:40:53 -07:00
"bar" : "foo" ,
2011-08-26 12:27:33 +04:00
"bar2" : "foo2" ,
2011-08-29 12:46:32 -07:00
"IntStr" : "42" ,
2017-11-08 07:16:07 -08:00
"UintptrStr" : "44" ,
2010-04-27 10:24:00 -07:00
"PBool" : null ,
"PInt" : null ,
"PInt8" : null ,
"PInt16" : null ,
"PInt32" : null ,
"PInt64" : null ,
"PUint" : null ,
"PUint8" : null ,
"PUint16" : null ,
"PUint32" : null ,
"PUint64" : null ,
"PUintptr" : null ,
"PFloat32" : null ,
"PFloat64" : null ,
"String" : "16" ,
"PString" : null ,
"Map" : {
2010-04-21 16:40:53 -07:00
"17" : {
2010-04-27 10:24:00 -07:00
"Tag" : "tag17"
2010-04-21 16:40:53 -07:00
} ,
"18" : {
2010-04-27 10:24:00 -07:00
"Tag" : "tag18"
2009-11-30 13:55:09 -08:00
}
2010-04-21 16:40:53 -07:00
} ,
2010-04-27 10:24:00 -07:00
"MapP" : {
2010-04-21 16:40:53 -07:00
"19" : {
2010-04-27 10:24:00 -07:00
"Tag" : "tag19"
2010-04-21 16:40:53 -07:00
} ,
"20" : null
} ,
2010-04-27 10:24:00 -07:00
"PMap" : null ,
"PMapP" : null ,
"EmptyMap" : { } ,
"NilMap" : null ,
"Slice" : [
2010-04-21 16:40:53 -07:00
{
2010-04-27 10:24:00 -07:00
"Tag" : "tag20"
2010-04-21 16:40:53 -07:00
} ,
{
2010-04-27 10:24:00 -07:00
"Tag" : "tag21"
2010-04-21 16:40:53 -07:00
}
] ,
2010-04-27 10:24:00 -07:00
"SliceP" : [
2010-04-21 16:40:53 -07:00
{
2010-04-27 10:24:00 -07:00
"Tag" : "tag22"
2010-04-21 16:40:53 -07:00
} ,
null ,
{
2010-04-27 10:24:00 -07:00
"Tag" : "tag23"
2010-04-21 16:40:53 -07:00
}
] ,
2010-04-27 10:24:00 -07:00
"PSlice" : null ,
"PSliceP" : null ,
"EmptySlice" : [ ] ,
2011-10-31 13:59:23 -04:00
"NilSlice" : null ,
2010-04-27 10:24:00 -07:00
"StringSlice" : [
2010-04-21 16:40:53 -07:00
"str24" ,
"str25" ,
"str26"
] ,
2011-02-23 11:32:29 -05:00
"ByteSlice" : "Gxwd" ,
2010-04-27 10:24:00 -07:00
"Small" : {
"Tag" : "tag30"
2010-04-21 16:40:53 -07:00
} ,
2010-04-27 10:24:00 -07:00
"PSmall" : {
"Tag" : "tag31"
2010-04-21 16:40:53 -07:00
} ,
2010-04-27 10:24:00 -07:00
"PPSmall" : null ,
"Interface" : 5.2 ,
"PInterface" : null
2010-04-21 16:40:53 -07:00
} `
2023-08-24 12:49:10 -07:00
var allValueCompact = stripWhitespace ( allValueIndent )
2010-04-21 16:40:53 -07:00
var pallValueIndent = ` {
2010-04-27 10:24:00 -07:00
"Bool" : false ,
"Int" : 0 ,
"Int8" : 0 ,
"Int16" : 0 ,
"Int32" : 0 ,
"Int64" : 0 ,
"Uint" : 0 ,
"Uint8" : 0 ,
"Uint16" : 0 ,
"Uint32" : 0 ,
"Uint64" : 0 ,
"Uintptr" : 0 ,
"Float32" : 0 ,
"Float64" : 0 ,
2010-04-21 16:40:53 -07:00
"bar" : "" ,
2011-08-26 12:27:33 +04:00
"bar2" : "" ,
2011-08-29 12:46:32 -07:00
"IntStr" : "0" ,
2017-11-08 07:16:07 -08:00
"UintptrStr" : "0" ,
2010-04-27 10:24:00 -07:00
"PBool" : true ,
"PInt" : 2 ,
"PInt8" : 3 ,
"PInt16" : 4 ,
"PInt32" : 5 ,
"PInt64" : 6 ,
"PUint" : 7 ,
"PUint8" : 8 ,
"PUint16" : 9 ,
"PUint32" : 10 ,
"PUint64" : 11 ,
"PUintptr" : 12 ,
"PFloat32" : 14.1 ,
"PFloat64" : 15.1 ,
"String" : "" ,
"PString" : "16" ,
"Map" : null ,
"MapP" : null ,
"PMap" : {
2010-04-21 16:40:53 -07:00
"17" : {
2010-04-27 10:24:00 -07:00
"Tag" : "tag17"
2010-04-21 16:40:53 -07:00
} ,
"18" : {
2010-04-27 10:24:00 -07:00
"Tag" : "tag18"
2010-04-21 16:40:53 -07:00
}
} ,
2010-04-27 10:24:00 -07:00
"PMapP" : {
2010-04-21 16:40:53 -07:00
"19" : {
2010-04-27 10:24:00 -07:00
"Tag" : "tag19"
2010-04-21 16:40:53 -07:00
} ,
"20" : null
} ,
2010-04-27 10:24:00 -07:00
"EmptyMap" : null ,
"NilMap" : null ,
2011-10-31 13:59:23 -04:00
"Slice" : null ,
"SliceP" : null ,
2010-04-27 10:24:00 -07:00
"PSlice" : [
2010-04-21 16:40:53 -07:00
{
2010-04-27 10:24:00 -07:00
"Tag" : "tag20"
2010-04-21 16:40:53 -07:00
} ,
{
2010-04-27 10:24:00 -07:00
"Tag" : "tag21"
2010-04-21 16:40:53 -07:00
}
] ,
2010-04-27 10:24:00 -07:00
"PSliceP" : [
2010-04-21 16:40:53 -07:00
{
2010-04-27 10:24:00 -07:00
"Tag" : "tag22"
2010-04-21 16:40:53 -07:00
} ,
null ,
{
2010-04-27 10:24:00 -07:00
"Tag" : "tag23"
2010-04-21 16:40:53 -07:00
}
] ,
2011-10-31 13:59:23 -04:00
"EmptySlice" : null ,
"NilSlice" : null ,
"StringSlice" : null ,
"ByteSlice" : null ,
2010-04-27 10:24:00 -07:00
"Small" : {
"Tag" : ""
2010-04-21 16:40:53 -07:00
} ,
2010-04-27 10:24:00 -07:00
"PSmall" : null ,
"PPSmall" : {
"Tag" : "tag31"
2010-04-21 16:40:53 -07:00
} ,
2010-04-27 10:24:00 -07:00
"Interface" : null ,
"PInterface" : 5.2
2010-04-21 16:40:53 -07:00
} `
2023-08-24 12:49:10 -07:00
var pallValueCompact = stripWhitespace ( pallValueIndent )
2012-02-03 11:15:06 +11:00
func TestRefUnmarshal ( t * testing . T ) {
type S struct {
// Ref is defined in encode_test.go.
R0 Ref
R1 * Ref
2013-08-14 14:56:07 -04:00
R2 RefText
R3 * RefText
2012-02-03 11:15:06 +11:00
}
want := S {
R0 : 12 ,
R1 : new ( Ref ) ,
2013-08-14 14:56:07 -04:00
R2 : 13 ,
R3 : new ( RefText ) ,
2012-02-03 11:15:06 +11:00
}
* want . R1 = 12
2013-08-14 14:56:07 -04:00
* want . R3 = 13
2012-02-03 11:15:06 +11:00
var got S
2013-08-14 14:56:07 -04:00
if err := Unmarshal ( [ ] byte ( ` { "R0":"ref","R1":"ref","R2":"ref","R3":"ref"} ` ) , & got ) ; err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Unmarshal error: %v" , err )
2012-02-03 11:15:06 +11:00
}
if ! reflect . DeepEqual ( got , want ) {
2023-08-24 12:49:10 -07:00
t . Errorf ( "Unmarsha:\n\tgot: %+v\n\twant: %+v" , got , want )
2012-02-03 11:15:06 +11:00
}
}
2012-02-19 00:27:05 -05:00
2012-05-03 17:35:44 -04:00
// Test that the empty string doesn't panic decoding when ,string is specified
// Issue 3450
func TestEmptyString ( t * testing . T ) {
type T2 struct {
Number1 int ` json:",string" `
Number2 int ` json:",string" `
}
data := ` { "Number1":"1", "Number2":""} `
dec := NewDecoder ( strings . NewReader ( data ) )
2023-08-24 12:49:10 -07:00
var got T2
switch err := dec . Decode ( & got ) ; {
case err == nil :
t . Fatalf ( "Decode error: got nil, want non-nil" )
case got . Number1 != 1 :
t . Fatalf ( "Decode: got.Number1 = %d, want 1" , got . Number1 )
2012-05-03 17:35:44 -04:00
}
}
2012-06-07 01:48:55 -04:00
2014-10-07 11:07:04 -04:00
// Test that a null for ,string is not replaced with the previous quoted string (issue 7046).
// It should also not be an error (issue 2540, issue 8587).
2014-01-03 10:13:28 -08:00
func TestNullString ( t * testing . T ) {
type T struct {
2014-10-07 11:07:04 -04:00
A int ` json:",string" `
B int ` json:",string" `
C * int ` json:",string" `
2014-01-03 10:13:28 -08:00
}
2014-10-07 11:07:04 -04:00
data := [ ] byte ( ` { "A": "1", "B": null, "C": null} ` )
2014-01-03 10:13:28 -08:00
var s T
2014-10-07 11:07:04 -04:00
s . B = 1
s . C = new ( int )
* s . C = 2
2023-08-24 12:49:10 -07:00
switch err := Unmarshal ( data , & s ) ; {
case err != nil :
t . Fatalf ( "Unmarshal error: %v" , err )
case s . B != 1 :
t . Fatalf ( "Unmarshal: s.B = %d, want 1" , s . B )
case s . C != nil :
t . Fatalf ( "Unmarshal: s.C = %d, want non-nil" , s . C )
2014-01-03 10:13:28 -08:00
}
}
2012-06-07 01:48:55 -04:00
func intp ( x int ) * int {
p := new ( int )
* p = x
return p
}
func intpp ( x * int ) * * int {
pp := new ( * int )
* pp = x
return pp
}
func TestInterfaceSet ( t * testing . T ) {
2023-08-24 12:49:10 -07:00
tests := [ ] struct {
CaseName
pre any
json string
post any
} {
{ Name ( "" ) , "foo" , ` "bar" ` , "bar" } ,
{ Name ( "" ) , "foo" , ` 2 ` , 2.0 } ,
{ Name ( "" ) , "foo" , ` true ` , true } ,
{ Name ( "" ) , "foo" , ` null ` , nil } ,
{ Name ( "" ) , nil , ` null ` , nil } ,
{ Name ( "" ) , new ( int ) , ` null ` , nil } ,
{ Name ( "" ) , ( * int ) ( nil ) , ` null ` , nil } ,
{ Name ( "" ) , new ( * int ) , ` null ` , new ( * int ) } ,
{ Name ( "" ) , ( * * int ) ( nil ) , ` null ` , nil } ,
{ Name ( "" ) , intp ( 1 ) , ` null ` , nil } ,
{ Name ( "" ) , intpp ( nil ) , ` null ` , intpp ( nil ) } ,
{ Name ( "" ) , intpp ( intp ( 1 ) ) , ` null ` , intpp ( nil ) } ,
}
for _ , tt := range tests {
t . Run ( tt . Name , func ( t * testing . T ) {
b := struct { X any } { tt . pre }
blob := ` { "X": ` + tt . json + ` } `
if err := Unmarshal ( [ ] byte ( blob ) , & b ) ; err != nil {
t . Fatalf ( "%s: Unmarshal(%#q) error: %v" , tt . Where , blob , err )
}
if ! reflect . DeepEqual ( b . X , tt . post ) {
t . Errorf ( "%s: Unmarshal(%#q):\n\tpre.X: %#v\n\tgot.X: %#v\n\twant.X: %#v" , tt . Where , blob , tt . pre , b . X , tt . post )
}
} )
2012-06-07 01:48:55 -04:00
}
}
2012-11-12 15:35:11 -05:00
2016-10-12 16:54:02 -04:00
type NullTest struct {
Bool bool
Int int
Int8 int8
Int16 int16
Int32 int32
Int64 int64
Uint uint
Uint8 uint8
Uint16 uint16
Uint32 uint32
Uint64 uint64
Float32 float32
Float64 float64
String string
PBool * bool
Map map [ string ] string
Slice [ ] string
2021-12-01 12:15:45 -05:00
Interface any
2016-10-12 16:54:02 -04:00
PRaw * RawMessage
PTime * time . Time
PBigInt * big . Int
PText * MustNotUnmarshalText
PBuffer * bytes . Buffer // has methods, just not relevant ones
PStruct * struct { }
Raw RawMessage
Time time . Time
BigInt big . Int
Text MustNotUnmarshalText
Buffer bytes . Buffer
Struct struct { }
}
2012-11-12 15:35:11 -05:00
// JSON null values should be ignored for primitives and string values instead of resulting in an error.
// Issue 2540
func TestUnmarshalNulls ( t * testing . T ) {
2016-10-12 16:54:02 -04:00
// Unmarshal docs:
// The JSON null value unmarshals into an interface, map, pointer, or slice
// by setting that Go value to nil. Because null is often used in JSON to mean
// ``not present,'' unmarshaling a JSON null into any other Go type has no effect
// on the value and produces no error.
2012-11-12 15:35:11 -05:00
jsonData := [ ] byte ( ` {
2016-10-12 16:54:02 -04:00
"Bool" : null ,
"Int" : null ,
"Int8" : null ,
"Int16" : null ,
"Int32" : null ,
"Int64" : null ,
"Uint" : null ,
"Uint8" : null ,
"Uint16" : null ,
"Uint32" : null ,
"Uint64" : null ,
"Float32" : null ,
"Float64" : null ,
"String" : null ,
"PBool" : null ,
"Map" : null ,
"Slice" : null ,
"Interface" : null ,
"PRaw" : null ,
"PTime" : null ,
"PBigInt" : null ,
"PText" : null ,
"PBuffer" : null ,
"PStruct" : null ,
"Raw" : null ,
"Time" : null ,
"BigInt" : null ,
"Text" : null ,
"Buffer" : null ,
"Struct" : null
} ` )
nulls := NullTest {
Bool : true ,
Int : 2 ,
Int8 : 3 ,
Int16 : 4 ,
Int32 : 5 ,
Int64 : 6 ,
Uint : 7 ,
Uint8 : 8 ,
Uint16 : 9 ,
Uint32 : 10 ,
Uint64 : 11 ,
Float32 : 12.1 ,
Float64 : 13.1 ,
String : "14" ,
PBool : new ( bool ) ,
Map : map [ string ] string { } ,
Slice : [ ] string { } ,
Interface : new ( MustNotUnmarshalJSON ) ,
PRaw : new ( RawMessage ) ,
PTime : new ( time . Time ) ,
PBigInt : new ( big . Int ) ,
PText : new ( MustNotUnmarshalText ) ,
PStruct : new ( struct { } ) ,
PBuffer : new ( bytes . Buffer ) ,
Raw : RawMessage ( "123" ) ,
Time : time . Unix ( 123456789 , 0 ) ,
BigInt : * big . NewInt ( 123 ) ,
}
before := nulls . Time . String ( )
2012-11-12 15:35:11 -05:00
err := Unmarshal ( jsonData , & nulls )
if err != nil {
t . Errorf ( "Unmarshal of null values failed: %v" , err )
}
if ! nulls . Bool || nulls . Int != 2 || nulls . Int8 != 3 || nulls . Int16 != 4 || nulls . Int32 != 5 || nulls . Int64 != 6 ||
nulls . Uint != 7 || nulls . Uint8 != 8 || nulls . Uint16 != 9 || nulls . Uint32 != 10 || nulls . Uint64 != 11 ||
nulls . Float32 != 12.1 || nulls . Float64 != 13.1 || nulls . String != "14" {
t . Errorf ( "Unmarshal of null values affected primitives" )
}
2016-10-12 16:54:02 -04:00
if nulls . PBool != nil {
t . Errorf ( "Unmarshal of null did not clear nulls.PBool" )
}
if nulls . Map != nil {
t . Errorf ( "Unmarshal of null did not clear nulls.Map" )
}
if nulls . Slice != nil {
t . Errorf ( "Unmarshal of null did not clear nulls.Slice" )
}
if nulls . Interface != nil {
t . Errorf ( "Unmarshal of null did not clear nulls.Interface" )
}
if nulls . PRaw != nil {
t . Errorf ( "Unmarshal of null did not clear nulls.PRaw" )
}
if nulls . PTime != nil {
t . Errorf ( "Unmarshal of null did not clear nulls.PTime" )
}
if nulls . PBigInt != nil {
t . Errorf ( "Unmarshal of null did not clear nulls.PBigInt" )
}
if nulls . PText != nil {
t . Errorf ( "Unmarshal of null did not clear nulls.PText" )
}
if nulls . PBuffer != nil {
t . Errorf ( "Unmarshal of null did not clear nulls.PBuffer" )
}
if nulls . PStruct != nil {
t . Errorf ( "Unmarshal of null did not clear nulls.PStruct" )
}
if string ( nulls . Raw ) != "null" {
t . Errorf ( "Unmarshal of RawMessage null did not record null: %v" , string ( nulls . Raw ) )
}
if nulls . Time . String ( ) != before {
t . Errorf ( "Unmarshal of time.Time null set time to %v" , nulls . Time . String ( ) )
}
if nulls . BigInt . String ( ) != "123" {
t . Errorf ( "Unmarshal of big.Int null set int to %v" , nulls . BigInt . String ( ) )
}
}
type MustNotUnmarshalJSON struct { }
func ( x MustNotUnmarshalJSON ) UnmarshalJSON ( data [ ] byte ) error {
return errors . New ( "MustNotUnmarshalJSON was used" )
}
type MustNotUnmarshalText struct { }
func ( x MustNotUnmarshalText ) UnmarshalText ( text [ ] byte ) error {
return errors . New ( "MustNotUnmarshalText was used" )
2012-11-12 15:35:11 -05:00
}
2012-12-30 15:40:42 +11:00
func TestStringKind ( t * testing . T ) {
type stringKind string
2023-08-24 12:49:10 -07:00
want := map [ stringKind ] int { "foo" : 42 }
data , err := Marshal ( want )
2012-12-30 15:40:42 +11:00
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Marshal error: %v" , err )
2012-12-30 15:40:42 +11:00
}
2023-08-24 12:49:10 -07:00
var got map [ stringKind ] int
err = Unmarshal ( data , & got )
2012-12-30 15:40:42 +11:00
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Unmarshal error: %v" , err )
2012-12-30 15:40:42 +11:00
}
2023-08-24 12:49:10 -07:00
if ! reflect . DeepEqual ( got , want ) {
t . Fatalf ( "Marshal/Unmarshal mismatch:\n\tgot: %v\n\twant: %v" , got , want )
2012-12-30 15:40:42 +11:00
}
2015-04-26 23:52:42 +02:00
}
2016-11-09 14:49:12 -08:00
// Custom types with []byte as underlying type could not be marshaled
// and then unmarshaled.
2015-04-26 23:52:42 +02:00
// Issue 8962.
func TestByteKind ( t * testing . T ) {
type byteKind [ ] byte
2023-08-24 12:49:10 -07:00
want := byteKind ( "hello" )
data , err := Marshal ( want )
2015-04-26 23:52:42 +02:00
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Marshal error: %v" , err )
2015-04-26 23:52:42 +02:00
}
2023-08-24 12:49:10 -07:00
var got byteKind
err = Unmarshal ( data , & got )
2015-04-26 23:52:42 +02:00
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Unmarshal error: %v" , err )
2015-04-26 23:52:42 +02:00
}
2023-08-24 12:49:10 -07:00
if ! reflect . DeepEqual ( got , want ) {
t . Fatalf ( "Marshal/Unmarshal mismatch:\n\tgot: %v\n\twant: %v" , got , want )
2015-04-26 23:52:42 +02:00
}
2012-12-30 15:40:42 +11:00
}
2013-01-14 08:44:16 +01:00
2015-10-25 22:42:41 +01:00
// The fix for issue 8962 introduced a regression.
// Issue 12921.
func TestSliceOfCustomByte ( t * testing . T ) {
type Uint8 uint8
2023-08-24 12:49:10 -07:00
want := [ ] Uint8 ( "hello" )
data , err := Marshal ( want )
2015-10-25 22:42:41 +01:00
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Marshal error: %v" , err )
2015-10-25 22:42:41 +01:00
}
2023-08-24 12:49:10 -07:00
var got [ ] Uint8
err = Unmarshal ( data , & got )
2015-10-25 22:42:41 +01:00
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Unmarshal error: %v" , err )
2015-10-25 22:42:41 +01:00
}
2023-08-24 12:49:10 -07:00
if ! reflect . DeepEqual ( got , want ) {
t . Fatalf ( "Marshal/Unmarshal mismatch:\n\tgot: %v\n\twant: %v" , got , want )
2015-10-25 22:42:41 +01:00
}
}
2013-01-14 08:44:16 +01:00
func TestUnmarshalTypeError ( t * testing . T ) {
2023-08-24 12:49:10 -07:00
tests := [ ] struct {
CaseName
dest any
in string
} {
{ Name ( "" ) , new ( string ) , ` { "user": "name"} ` } , // issue 4628.
{ Name ( "" ) , new ( error ) , ` { } ` } , // issue 4222
{ Name ( "" ) , new ( error ) , ` [] ` } ,
{ Name ( "" ) , new ( error ) , ` "" ` } ,
{ Name ( "" ) , new ( error ) , ` 123 ` } ,
{ Name ( "" ) , new ( error ) , ` true ` } ,
}
for _ , tt := range tests {
t . Run ( tt . Name , func ( t * testing . T ) {
err := Unmarshal ( [ ] byte ( tt . in ) , tt . dest )
if _ , ok := err . ( * UnmarshalTypeError ) ; ! ok {
t . Errorf ( "%s: Unmarshal(%#q, %T):\n\tgot: %T\n\twant: %T" ,
tt . Where , tt . in , tt . dest , err , new ( UnmarshalTypeError ) )
}
} )
2013-01-14 08:44:16 +01:00
}
}
2013-01-22 17:49:07 -05:00
2013-01-29 13:34:18 -08:00
func TestUnmarshalSyntax ( t * testing . T ) {
2021-12-01 12:15:45 -05:00
var x any
2023-08-24 12:49:10 -07:00
tests := [ ] struct {
CaseName
in string
} {
{ Name ( "" ) , "tru" } ,
{ Name ( "" ) , "fals" } ,
{ Name ( "" ) , "nul" } ,
{ Name ( "" ) , "123e" } ,
{ Name ( "" ) , ` "hello ` } ,
{ Name ( "" ) , ` [1,2,3 ` } ,
{ Name ( "" ) , ` { "key":1 ` } ,
{ Name ( "" ) , ` { "key":1, ` } ,
}
for _ , tt := range tests {
t . Run ( tt . Name , func ( t * testing . T ) {
err := Unmarshal ( [ ] byte ( tt . in ) , & x )
if _ , ok := err . ( * SyntaxError ) ; ! ok {
t . Errorf ( "%s: Unmarshal(%#q, any):\n\tgot: %T\n\twant: %T" ,
tt . Where , tt . in , err , new ( SyntaxError ) )
}
} )
2013-01-29 13:34:18 -08:00
}
}
2013-01-22 17:49:07 -05:00
// Test handling of unexported fields that should be ignored.
// Issue 4660
type unexportedFields struct {
Name string
2021-12-01 12:15:45 -05:00
m map [ string ] any ` json:"-" `
m2 map [ string ] any ` json:"abcd" `
2018-09-11 22:09:00 +02:00
s [ ] int ` json:"-" `
2013-01-22 17:49:07 -05:00
}
func TestUnmarshalUnexported ( t * testing . T ) {
2018-09-11 22:09:00 +02:00
input := ` { "Name": "Bob", "m": { "x": 123}, "m2": { "y": 456}, "abcd": { "z": 789}, "s": [2, 3]} `
2013-01-22 17:49:07 -05:00
want := & unexportedFields { Name : "Bob" }
out := & unexportedFields { }
err := Unmarshal ( [ ] byte ( input ) , out )
if err != nil {
2023-08-24 12:49:10 -07:00
t . Errorf ( "Unmarshal error: %v" , err )
2013-01-22 17:49:07 -05:00
}
if ! reflect . DeepEqual ( out , want ) {
2023-08-24 12:49:10 -07:00
t . Errorf ( "Unmarshal:\n\tgot: %+v\n\twant: %+v" , out , want )
2013-01-22 17:49:07 -05:00
}
}
2013-02-14 14:46:15 -05:00
// Time3339 is a time.Time which encodes to and from JSON
// as an RFC 3339 time in UTC.
type Time3339 time . Time
func ( t * Time3339 ) UnmarshalJSON ( b [ ] byte ) error {
if len ( b ) < 2 || b [ 0 ] != '"' || b [ len ( b ) - 1 ] != '"' {
2013-02-25 12:43:03 -08:00
return fmt . Errorf ( "types: failed to unmarshal non-string value %q as an RFC 3339 time" , b )
2013-02-14 14:46:15 -05:00
}
tm , err := time . Parse ( time . RFC3339 , string ( b [ 1 : len ( b ) - 1 ] ) )
if err != nil {
return err
}
* t = Time3339 ( tm )
return nil
}
2015-07-30 14:05:04 +00:00
func TestUnmarshalJSONLiteralError ( t * testing . T ) {
2013-02-14 14:46:15 -05:00
var t3 Time3339
2023-08-24 12:49:10 -07:00
switch err := Unmarshal ( [ ] byte ( ` "0000-00-00T00:00:00Z" ` ) , & t3 ) ; {
case err == nil :
t . Fatalf ( "Unmarshal error: got nil, want non-nil" )
case ! strings . Contains ( err . Error ( ) , "range" ) :
t . Errorf ( "Unmarshal error:\n\tgot: %v\n\twant: out of range" , err )
2013-02-14 14:46:15 -05:00
}
}
2013-03-13 14:53:03 -04:00
// Test that extra object elements in an array do not result in a
// "data changing underfoot" error.
// Issue 3717
func TestSkipArrayObjects ( t * testing . T ) {
json := ` [ { }] `
2021-12-01 12:15:45 -05:00
var dest [ 0 ] any
2013-03-13 14:53:03 -04:00
err := Unmarshal ( [ ] byte ( json ) , & dest )
if err != nil {
2023-08-24 12:49:10 -07:00
t . Errorf ( "Unmarshal error: %v" , err )
2013-03-13 14:53:03 -04:00
}
}
2013-09-09 19:11:05 -04:00
encoding/json: fix the broken "overwriting of data" tests
Because TestUnmarshal actually allocates a new value to decode into
using ptr's pointer type, any existing data is thrown away. This was
harmless in alomst all of the test cases, minus the "overwriting of
data" ones added in 2015 in CL 12209.
I spotted that nothing covered decoding a JSON array with few elements
into a slice which already had many elements. I initially assumed that
the code was buggy or that some code could be removed, when in fact
there simply wasn't any code covering the edge case.
Move those two tests to TestPrefilled, which already served a very
similar purpose. Remove the map case, as TestPrefilled already has
plenty of prefilled map cases. Moreover, we no longer reset an entire
map when decoding, as per the godoc:
To unmarshal a JSON object into a map, Unmarshal first
establishes a map to use. If the map is nil, Unmarshal allocates
a new map. Otherwise Unmarshal reuses the existing map, keeping
existing entries.
Finally, to ensure that ptr is used correctly in the future, make
TestUnmarshal error if it's anything other than a pointer to a zero
value. That is, the only correct use should be new(type). Don't rename
the ptr field, as that would be extremely noisy and cause unwanted merge
conflicts.
Change-Id: I41e3ecfeae42d877ac5443a6bd622ac3d6c8120c
Reviewed-on: https://go-review.googlesource.com/c/go/+/185738
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Emmanuel Odeke <emm.odeke@gmail.com>
2019-07-11 21:27:19 +09:00
// Test semantics of pre-filled data, such as struct fields, map elements,
// slices, and arrays.
// Issues 4900 and 8837, among others.
2013-09-09 19:11:05 -04:00
func TestPrefilled ( t * testing . T ) {
2020-07-01 11:31:15 +00:00
// Values here change, cannot reuse table across runs.
2023-08-24 12:49:10 -07:00
tests := [ ] struct {
CaseName
2013-09-09 19:11:05 -04:00
in string
2021-12-01 12:15:45 -05:00
ptr any
out any
2023-08-24 12:49:10 -07:00
} { {
CaseName : Name ( "" ) ,
in : ` { "X": 1, "Y": 2} ` ,
ptr : & XYZ { X : float32 ( 3 ) , Y : int16 ( 4 ) , Z : 1.5 } ,
out : & XYZ { X : float64 ( 1 ) , Y : float64 ( 2 ) , Z : 1.5 } ,
} , {
CaseName : Name ( "" ) ,
in : ` { "X": 1, "Y": 2} ` ,
ptr : & map [ string ] any { "X" : float32 ( 3 ) , "Y" : int16 ( 4 ) , "Z" : 1.5 } ,
out : & map [ string ] any { "X" : float64 ( 1 ) , "Y" : float64 ( 2 ) , "Z" : 1.5 } ,
} , {
CaseName : Name ( "" ) ,
in : ` [2] ` ,
ptr : & [ ] int { 1 } ,
out : & [ ] int { 2 } ,
} , {
CaseName : Name ( "" ) ,
in : ` [2, 3] ` ,
ptr : & [ ] int { 1 } ,
out : & [ ] int { 2 , 3 } ,
} , {
CaseName : Name ( "" ) ,
in : ` [2, 3] ` ,
ptr : & [ ... ] int { 1 } ,
out : & [ ... ] int { 2 } ,
} , {
CaseName : Name ( "" ) ,
in : ` [3] ` ,
ptr : & [ ... ] int { 1 , 2 } ,
out : & [ ... ] int { 3 , 0 } ,
} }
for _ , tt := range tests {
t . Run ( tt . Name , func ( t * testing . T ) {
ptrstr := fmt . Sprintf ( "%v" , tt . ptr )
err := Unmarshal ( [ ] byte ( tt . in ) , tt . ptr ) // tt.ptr edited here
if err != nil {
t . Errorf ( "%s: Unmarshal error: %v" , tt . Where , err )
}
if ! reflect . DeepEqual ( tt . ptr , tt . out ) {
t . Errorf ( "%s: Unmarshal(%#q, %T):\n\tgot: %v\n\twant: %v" , tt . Where , tt . in , ptrstr , tt . ptr , tt . out )
}
} )
2013-09-09 19:11:05 -04:00
}
}
2014-01-02 09:49:55 +11:00
func TestInvalidUnmarshal ( t * testing . T ) {
buf := [ ] byte ( ` { "a":"1"} ` )
2023-08-24 12:49:10 -07:00
tests := [ ] struct {
CaseName
v any
want string
} {
{ Name ( "" ) , nil , "json: Unmarshal(nil)" } ,
{ Name ( "" ) , struct { } { } , "json: Unmarshal(non-pointer struct {})" } ,
{ Name ( "" ) , ( * int ) ( nil ) , "json: Unmarshal(nil *int)" } ,
}
for _ , tt := range tests {
t . Run ( tt . Name , func ( t * testing . T ) {
err := Unmarshal ( buf , tt . v )
if err == nil {
t . Fatalf ( "%s: Unmarshal error: got nil, want non-nil" , tt . Where )
}
if got := err . Error ( ) ; got != tt . want {
t . Errorf ( "%s: Unmarshal error:\n\tgot: %s\n\twant: %s" , tt . Where , got , tt . want )
}
} )
2014-01-02 09:49:55 +11:00
}
}
2015-05-16 23:01:39 -04:00
2015-07-14 19:31:44 -04:00
func TestInvalidUnmarshalText ( t * testing . T ) {
buf := [ ] byte ( ` 123 ` )
2023-08-24 12:49:10 -07:00
tests := [ ] struct {
CaseName
v any
want string
} {
{ Name ( "" ) , nil , "json: Unmarshal(nil)" } ,
{ Name ( "" ) , struct { } { } , "json: Unmarshal(non-pointer struct {})" } ,
{ Name ( "" ) , ( * int ) ( nil ) , "json: Unmarshal(nil *int)" } ,
{ Name ( "" ) , new ( net . IP ) , "json: cannot unmarshal number into Go value of type *net.IP" } ,
}
for _ , tt := range tests {
t . Run ( tt . Name , func ( t * testing . T ) {
err := Unmarshal ( buf , tt . v )
if err == nil {
t . Fatalf ( "%s: Unmarshal error: got nil, want non-nil" , tt . Where )
}
if got := err . Error ( ) ; got != tt . want {
t . Errorf ( "%s: Unmarshal error:\n\tgot: %s\n\twant: %s" , tt . Where , got , tt . want )
}
} )
2015-07-14 19:31:44 -04:00
}
}
2015-05-16 23:01:39 -04:00
// Test that string option is ignored for invalid types.
// Issue 9812.
func TestInvalidStringOption ( t * testing . T ) {
num := 0
item := struct {
T time . Time ` json:",string" `
M map [ string ] string ` json:",string" `
S [ ] string ` json:",string" `
A [ 1 ] string ` json:",string" `
2021-12-01 12:15:45 -05:00
I any ` json:",string" `
2015-05-16 23:01:39 -04:00
P * int ` json:",string" `
} { M : make ( map [ string ] string ) , S : make ( [ ] string , 0 ) , I : num , P : & num }
data , err := Marshal ( item )
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Marshal error: %v" , err )
2015-05-16 23:01:39 -04:00
}
err = Unmarshal ( data , & item )
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Unmarshal error: %v" , err )
2015-05-16 23:01:39 -04:00
}
}
2017-12-05 22:38:36 -08:00
2018-02-28 13:45:06 -08:00
// Test unmarshal behavior with regards to embedded unexported structs.
//
// (Issue 21357) If the embedded struct is a pointer and is unallocated,
// this returns an error because unmarshal cannot set the field.
//
// (Issue 24152) If the embedded struct is given an explicit name,
// ensure that the normal unmarshal logic does not panic in reflect.
2018-10-11 14:43:47 +01:00
//
// (Issue 28145) If the embedded struct is given an explicit name and has
// exported methods, don't cause a panic trying to get its value.
2018-02-28 13:45:06 -08:00
func TestUnmarshalEmbeddedUnexported ( t * testing . T ) {
2017-12-05 22:38:36 -08:00
type (
embed1 struct { Q int }
embed2 struct { Q int }
embed3 struct {
Q int64 ` json:",string" `
}
S1 struct {
* embed1
R int
}
S2 struct {
* embed1
Q int
}
S3 struct {
embed1
R int
}
S4 struct {
* embed1
embed2
}
S5 struct {
* embed3
R int
}
2018-02-28 13:45:06 -08:00
S6 struct {
embed1 ` json:"embed1" `
}
S7 struct {
embed1 ` json:"embed1" `
embed2
}
S8 struct {
embed1 ` json:"embed1" `
embed2 ` json:"embed2" `
Q int
}
2018-10-11 14:43:47 +01:00
S9 struct {
unexportedWithMethods ` json:"embed" `
}
2017-12-05 22:38:36 -08:00
)
tests := [ ] struct {
2023-08-24 12:49:10 -07:00
CaseName
2017-12-05 22:38:36 -08:00
in string
2021-12-01 12:15:45 -05:00
ptr any
out any
2017-12-05 22:38:36 -08:00
err error
} { {
// Error since we cannot set S1.embed1, but still able to set S1.R.
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "R":2,"Q":1} ` ,
ptr : new ( S1 ) ,
out : & S1 { R : 2 } ,
err : fmt . Errorf ( "json: cannot set embedded pointer to unexported struct: json.embed1" ) ,
2017-12-05 22:38:36 -08:00
} , {
// The top level Q field takes precedence.
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "Q":1} ` ,
ptr : new ( S2 ) ,
out : & S2 { Q : 1 } ,
2017-12-05 22:38:36 -08:00
} , {
// No issue with non-pointer variant.
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "R":2,"Q":1} ` ,
ptr : new ( S3 ) ,
out : & S3 { embed1 : embed1 { Q : 1 } , R : 2 } ,
2017-12-05 22:38:36 -08:00
} , {
// No error since both embedded structs have field R, which annihilate each other.
// Thus, no attempt is made at setting S4.embed1.
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "R":2} ` ,
ptr : new ( S4 ) ,
out : new ( S4 ) ,
2017-12-05 22:38:36 -08:00
} , {
// Error since we cannot set S5.embed1, but still able to set S5.R.
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "R":2,"Q":1} ` ,
ptr : new ( S5 ) ,
out : & S5 { R : 2 } ,
err : fmt . Errorf ( "json: cannot set embedded pointer to unexported struct: json.embed3" ) ,
2018-02-28 13:45:06 -08:00
} , {
// Issue 24152, ensure decodeState.indirect does not panic.
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "embed1": { "Q": 1}} ` ,
ptr : new ( S6 ) ,
out : & S6 { embed1 { 1 } } ,
2018-02-28 13:45:06 -08:00
} , {
// Issue 24153, check that we can still set forwarded fields even in
// the presence of a name conflict.
//
// This relies on obscure behavior of reflect where it is possible
// to set a forwarded exported field on an unexported embedded struct
// even though there is a name conflict, even when it would have been
// impossible to do so according to Go visibility rules.
// Go forbids this because it is ambiguous whether S7.Q refers to
// S7.embed1.Q or S7.embed2.Q. Since embed1 and embed2 are unexported,
// it should be impossible for an external package to set either Q.
//
// It is probably okay for a future reflect change to break this.
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "embed1": { "Q": 1}, "Q": 2} ` ,
ptr : new ( S7 ) ,
out : & S7 { embed1 { 1 } , embed2 { 2 } } ,
2018-02-28 13:45:06 -08:00
} , {
// Issue 24153, similar to the S7 case.
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "embed1": { "Q": 1}, "embed2": { "Q": 2}, "Q": 3} ` ,
ptr : new ( S8 ) ,
out : & S8 { embed1 { 1 } , embed2 { 2 } , 3 } ,
2018-10-11 14:43:47 +01:00
} , {
// Issue 228145, similar to the cases above.
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` { "embed": { }} ` ,
ptr : new ( S9 ) ,
out : & S9 { } ,
2017-12-05 22:38:36 -08:00
} }
2023-08-24 12:49:10 -07:00
for _ , tt := range tests {
t . Run ( tt . Name , func ( t * testing . T ) {
err := Unmarshal ( [ ] byte ( tt . in ) , tt . ptr )
if ! equalError ( err , tt . err ) {
t . Errorf ( "%s: Unmarshal error:\n\tgot: %v\n\twant: %v" , tt . Where , err , tt . err )
}
if ! reflect . DeepEqual ( tt . ptr , tt . out ) {
t . Errorf ( "%s: Unmarshal:\n\tgot: %#+v\n\twant: %#+v" , tt . Where , tt . ptr , tt . out )
}
} )
2017-12-05 22:38:36 -08:00
}
}
2017-12-05 22:53:48 -08:00
2019-09-02 00:07:40 +00:00
func TestUnmarshalErrorAfterMultipleJSON ( t * testing . T ) {
tests := [ ] struct {
2023-08-24 12:49:10 -07:00
CaseName
2019-09-02 00:07:40 +00:00
in string
err error
} { {
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` 1 false null : ` ,
err : & SyntaxError { "invalid character ':' looking for beginning of value" , 14 } ,
2019-09-02 00:07:40 +00:00
} , {
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` 1 [] [,] ` ,
err : & SyntaxError { "invalid character ',' looking for beginning of value" , 7 } ,
2019-09-02 00:07:40 +00:00
} , {
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` 1 [] [true:] ` ,
err : & SyntaxError { "invalid character ':' after array element" , 11 } ,
2019-09-02 00:07:40 +00:00
} , {
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` 1 { } { "x"=} ` ,
err : & SyntaxError { "invalid character '=' after object key" , 14 } ,
2019-09-02 00:07:40 +00:00
} , {
2023-08-24 12:49:10 -07:00
CaseName : Name ( "" ) ,
in : ` falsetruenul# ` ,
err : & SyntaxError { "invalid character '#' in literal null (expecting 'l')" , 13 } ,
2019-09-02 00:07:40 +00:00
} }
2023-08-24 12:49:10 -07:00
for _ , tt := range tests {
t . Run ( tt . Name , func ( t * testing . T ) {
dec := NewDecoder ( strings . NewReader ( tt . in ) )
var err error
for err == nil {
var v any
err = dec . Decode ( & v )
2019-09-02 00:07:40 +00:00
}
2023-08-24 12:49:10 -07:00
if ! reflect . DeepEqual ( err , tt . err ) {
t . Errorf ( "%s: Decode error:\n\tgot: %v\n\twant: %v" , tt . Where , err , tt . err )
}
} )
2019-09-02 00:07:40 +00:00
}
}
2017-12-05 22:53:48 -08:00
type unmarshalPanic struct { }
func ( unmarshalPanic ) UnmarshalJSON ( [ ] byte ) error { panic ( 0xdead ) }
func TestUnmarshalPanic ( t * testing . T ) {
defer func ( ) {
if got := recover ( ) ; ! reflect . DeepEqual ( got , 0xdead ) {
t . Errorf ( "panic() = (%T)(%v), want 0xdead" , got , got )
}
} ( )
Unmarshal ( [ ] byte ( "{}" ) , & unmarshalPanic { } )
t . Fatalf ( "Unmarshal should have panicked" )
}
2019-04-29 22:57:25 +07:00
// The decoder used to hang if decoding into an interface pointing to its own address.
// See golang.org/issues/31740.
func TestUnmarshalRecursivePointer ( t * testing . T ) {
2021-12-01 12:15:45 -05:00
var v any
2019-04-29 22:57:25 +07:00
v = & v
data := [ ] byte ( ` { "a": "b"} ` )
if err := Unmarshal ( data , v ) ; err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Unmarshal error: %v" , err )
2019-04-29 22:57:25 +07:00
}
}
2019-10-10 10:19:12 +07:00
type textUnmarshalerString string
func ( m * textUnmarshalerString ) UnmarshalText ( text [ ] byte ) error {
* m = textUnmarshalerString ( strings . ToLower ( string ( text ) ) )
return nil
}
encoding/json: don't mangle strings in an edge case when decoding
The added comment contains some context. The original optimization
assumed that each call to unquoteBytes (or unquote) followed its
corresponding call to rescanLiteral. Otherwise, unquoting a literal
might use d.safeUnquote from another re-scanned literal.
Unfortunately, this assumption is wrong. When decoding {"foo": "bar"}
into a map[T]string where T implements TextUnmarshaler, the sequence of
calls would be as follows:
1) rescanLiteral "foo"
2) unquoteBytes "foo"
3) rescanLiteral "bar"
4) unquoteBytes "foo" (for UnmarshalText)
5) unquoteBytes "bar"
Note that the call to UnmarshalText happens in literalStore, which
repeats the work to unquote the input string literal. But, since that
happens after we've re-scanned "bar", we're using the wrong safeUnquote
field value.
In the added test case, the second string had a non-zero number of safe
bytes, and the first string had none since it was all non-ASCII. Thus,
"safely" unquoting a number of the first string's bytes could cut a rune
in half, and thus mangle the runes.
A rather simple fix, without a full revert, is to only allow one use of
safeUnquote per call to unquoteBytes. Each call to rescanLiteral when
we have a string is soon followed by a call to unquoteBytes, so it's no
longer possible for us to use the wrong index.
Also add a test case from #38126, which is the same underlying bug, but
affecting the ",string" option.
Before the fix, the test would fail, just like in the original two issues:
--- FAIL: TestUnmarshalRescanLiteralMangledUnquote (0.00s)
decode_test.go:2443: Key "开源" does not exist in map: map[开���:12345开源]
decode_test.go:2458: Unmarshal unexpected error: json: invalid use of ,string struct tag, trying to unmarshal "\"aaa\tbbb\"" into string
Fixes #38105.
For #38126.
Change-Id: I761e54924e9a971a4f9eaa70bbf72014bb1476e6
Reviewed-on: https://go-review.googlesource.com/c/go/+/226218
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
2020-03-27 23:56:09 +00:00
// Test unmarshal to a map, where the map key is a user defined type.
2019-10-10 10:19:12 +07:00
// See golang.org/issues/34437.
func TestUnmarshalMapWithTextUnmarshalerStringKey ( t * testing . T ) {
var p map [ textUnmarshalerString ] string
if err := Unmarshal ( [ ] byte ( ` { "FOO": "1"} ` ) , & p ) ; err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Unmarshal error: %v" , err )
2019-10-10 10:19:12 +07:00
}
if _ , ok := p [ "foo" ] ; ! ok {
2023-08-24 12:49:10 -07:00
t . Errorf ( ` key "foo" missing in map: %v ` , p )
encoding/json: don't mangle strings in an edge case when decoding
The added comment contains some context. The original optimization
assumed that each call to unquoteBytes (or unquote) followed its
corresponding call to rescanLiteral. Otherwise, unquoting a literal
might use d.safeUnquote from another re-scanned literal.
Unfortunately, this assumption is wrong. When decoding {"foo": "bar"}
into a map[T]string where T implements TextUnmarshaler, the sequence of
calls would be as follows:
1) rescanLiteral "foo"
2) unquoteBytes "foo"
3) rescanLiteral "bar"
4) unquoteBytes "foo" (for UnmarshalText)
5) unquoteBytes "bar"
Note that the call to UnmarshalText happens in literalStore, which
repeats the work to unquote the input string literal. But, since that
happens after we've re-scanned "bar", we're using the wrong safeUnquote
field value.
In the added test case, the second string had a non-zero number of safe
bytes, and the first string had none since it was all non-ASCII. Thus,
"safely" unquoting a number of the first string's bytes could cut a rune
in half, and thus mangle the runes.
A rather simple fix, without a full revert, is to only allow one use of
safeUnquote per call to unquoteBytes. Each call to rescanLiteral when
we have a string is soon followed by a call to unquoteBytes, so it's no
longer possible for us to use the wrong index.
Also add a test case from #38126, which is the same underlying bug, but
affecting the ",string" option.
Before the fix, the test would fail, just like in the original two issues:
--- FAIL: TestUnmarshalRescanLiteralMangledUnquote (0.00s)
decode_test.go:2443: Key "开源" does not exist in map: map[开���:12345开源]
decode_test.go:2458: Unmarshal unexpected error: json: invalid use of ,string struct tag, trying to unmarshal "\"aaa\tbbb\"" into string
Fixes #38105.
For #38126.
Change-Id: I761e54924e9a971a4f9eaa70bbf72014bb1476e6
Reviewed-on: https://go-review.googlesource.com/c/go/+/226218
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
2020-03-27 23:56:09 +00:00
}
}
func TestUnmarshalRescanLiteralMangledUnquote ( t * testing . T ) {
// See golang.org/issues/38105.
var p map [ textUnmarshalerString ] string
if err := Unmarshal ( [ ] byte ( ` { "开源":"12345开源"} ` ) , & p ) ; err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Unmarshal error: %v" , err )
encoding/json: don't mangle strings in an edge case when decoding
The added comment contains some context. The original optimization
assumed that each call to unquoteBytes (or unquote) followed its
corresponding call to rescanLiteral. Otherwise, unquoting a literal
might use d.safeUnquote from another re-scanned literal.
Unfortunately, this assumption is wrong. When decoding {"foo": "bar"}
into a map[T]string where T implements TextUnmarshaler, the sequence of
calls would be as follows:
1) rescanLiteral "foo"
2) unquoteBytes "foo"
3) rescanLiteral "bar"
4) unquoteBytes "foo" (for UnmarshalText)
5) unquoteBytes "bar"
Note that the call to UnmarshalText happens in literalStore, which
repeats the work to unquote the input string literal. But, since that
happens after we've re-scanned "bar", we're using the wrong safeUnquote
field value.
In the added test case, the second string had a non-zero number of safe
bytes, and the first string had none since it was all non-ASCII. Thus,
"safely" unquoting a number of the first string's bytes could cut a rune
in half, and thus mangle the runes.
A rather simple fix, without a full revert, is to only allow one use of
safeUnquote per call to unquoteBytes. Each call to rescanLiteral when
we have a string is soon followed by a call to unquoteBytes, so it's no
longer possible for us to use the wrong index.
Also add a test case from #38126, which is the same underlying bug, but
affecting the ",string" option.
Before the fix, the test would fail, just like in the original two issues:
--- FAIL: TestUnmarshalRescanLiteralMangledUnquote (0.00s)
decode_test.go:2443: Key "开源" does not exist in map: map[开���:12345开源]
decode_test.go:2458: Unmarshal unexpected error: json: invalid use of ,string struct tag, trying to unmarshal "\"aaa\tbbb\"" into string
Fixes #38105.
For #38126.
Change-Id: I761e54924e9a971a4f9eaa70bbf72014bb1476e6
Reviewed-on: https://go-review.googlesource.com/c/go/+/226218
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
2020-03-27 23:56:09 +00:00
}
if _ , ok := p [ "开源" ] ; ! ok {
2023-08-24 12:49:10 -07:00
t . Errorf ( ` key "开源" missing in map: %v ` , p )
encoding/json: don't mangle strings in an edge case when decoding
The added comment contains some context. The original optimization
assumed that each call to unquoteBytes (or unquote) followed its
corresponding call to rescanLiteral. Otherwise, unquoting a literal
might use d.safeUnquote from another re-scanned literal.
Unfortunately, this assumption is wrong. When decoding {"foo": "bar"}
into a map[T]string where T implements TextUnmarshaler, the sequence of
calls would be as follows:
1) rescanLiteral "foo"
2) unquoteBytes "foo"
3) rescanLiteral "bar"
4) unquoteBytes "foo" (for UnmarshalText)
5) unquoteBytes "bar"
Note that the call to UnmarshalText happens in literalStore, which
repeats the work to unquote the input string literal. But, since that
happens after we've re-scanned "bar", we're using the wrong safeUnquote
field value.
In the added test case, the second string had a non-zero number of safe
bytes, and the first string had none since it was all non-ASCII. Thus,
"safely" unquoting a number of the first string's bytes could cut a rune
in half, and thus mangle the runes.
A rather simple fix, without a full revert, is to only allow one use of
safeUnquote per call to unquoteBytes. Each call to rescanLiteral when
we have a string is soon followed by a call to unquoteBytes, so it's no
longer possible for us to use the wrong index.
Also add a test case from #38126, which is the same underlying bug, but
affecting the ",string" option.
Before the fix, the test would fail, just like in the original two issues:
--- FAIL: TestUnmarshalRescanLiteralMangledUnquote (0.00s)
decode_test.go:2443: Key "开源" does not exist in map: map[开���:12345开源]
decode_test.go:2458: Unmarshal unexpected error: json: invalid use of ,string struct tag, trying to unmarshal "\"aaa\tbbb\"" into string
Fixes #38105.
For #38126.
Change-Id: I761e54924e9a971a4f9eaa70bbf72014bb1476e6
Reviewed-on: https://go-review.googlesource.com/c/go/+/226218
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
2020-03-27 23:56:09 +00:00
}
// See golang.org/issues/38126.
type T struct {
F1 string ` json:"F1,string" `
}
2023-08-24 12:49:10 -07:00
wantT := T { "aaa\tbbb" }
encoding/json: don't mangle strings in an edge case when decoding
The added comment contains some context. The original optimization
assumed that each call to unquoteBytes (or unquote) followed its
corresponding call to rescanLiteral. Otherwise, unquoting a literal
might use d.safeUnquote from another re-scanned literal.
Unfortunately, this assumption is wrong. When decoding {"foo": "bar"}
into a map[T]string where T implements TextUnmarshaler, the sequence of
calls would be as follows:
1) rescanLiteral "foo"
2) unquoteBytes "foo"
3) rescanLiteral "bar"
4) unquoteBytes "foo" (for UnmarshalText)
5) unquoteBytes "bar"
Note that the call to UnmarshalText happens in literalStore, which
repeats the work to unquote the input string literal. But, since that
happens after we've re-scanned "bar", we're using the wrong safeUnquote
field value.
In the added test case, the second string had a non-zero number of safe
bytes, and the first string had none since it was all non-ASCII. Thus,
"safely" unquoting a number of the first string's bytes could cut a rune
in half, and thus mangle the runes.
A rather simple fix, without a full revert, is to only allow one use of
safeUnquote per call to unquoteBytes. Each call to rescanLiteral when
we have a string is soon followed by a call to unquoteBytes, so it's no
longer possible for us to use the wrong index.
Also add a test case from #38126, which is the same underlying bug, but
affecting the ",string" option.
Before the fix, the test would fail, just like in the original two issues:
--- FAIL: TestUnmarshalRescanLiteralMangledUnquote (0.00s)
decode_test.go:2443: Key "开源" does not exist in map: map[开���:12345开源]
decode_test.go:2458: Unmarshal unexpected error: json: invalid use of ,string struct tag, trying to unmarshal "\"aaa\tbbb\"" into string
Fixes #38105.
For #38126.
Change-Id: I761e54924e9a971a4f9eaa70bbf72014bb1476e6
Reviewed-on: https://go-review.googlesource.com/c/go/+/226218
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
2020-03-27 23:56:09 +00:00
2023-08-24 12:49:10 -07:00
b , err := Marshal ( wantT )
encoding/json: don't mangle strings in an edge case when decoding
The added comment contains some context. The original optimization
assumed that each call to unquoteBytes (or unquote) followed its
corresponding call to rescanLiteral. Otherwise, unquoting a literal
might use d.safeUnquote from another re-scanned literal.
Unfortunately, this assumption is wrong. When decoding {"foo": "bar"}
into a map[T]string where T implements TextUnmarshaler, the sequence of
calls would be as follows:
1) rescanLiteral "foo"
2) unquoteBytes "foo"
3) rescanLiteral "bar"
4) unquoteBytes "foo" (for UnmarshalText)
5) unquoteBytes "bar"
Note that the call to UnmarshalText happens in literalStore, which
repeats the work to unquote the input string literal. But, since that
happens after we've re-scanned "bar", we're using the wrong safeUnquote
field value.
In the added test case, the second string had a non-zero number of safe
bytes, and the first string had none since it was all non-ASCII. Thus,
"safely" unquoting a number of the first string's bytes could cut a rune
in half, and thus mangle the runes.
A rather simple fix, without a full revert, is to only allow one use of
safeUnquote per call to unquoteBytes. Each call to rescanLiteral when
we have a string is soon followed by a call to unquoteBytes, so it's no
longer possible for us to use the wrong index.
Also add a test case from #38126, which is the same underlying bug, but
affecting the ",string" option.
Before the fix, the test would fail, just like in the original two issues:
--- FAIL: TestUnmarshalRescanLiteralMangledUnquote (0.00s)
decode_test.go:2443: Key "开源" does not exist in map: map[开���:12345开源]
decode_test.go:2458: Unmarshal unexpected error: json: invalid use of ,string struct tag, trying to unmarshal "\"aaa\tbbb\"" into string
Fixes #38105.
For #38126.
Change-Id: I761e54924e9a971a4f9eaa70bbf72014bb1476e6
Reviewed-on: https://go-review.googlesource.com/c/go/+/226218
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
2020-03-27 23:56:09 +00:00
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Marshal error: %v" , err )
encoding/json: don't mangle strings in an edge case when decoding
The added comment contains some context. The original optimization
assumed that each call to unquoteBytes (or unquote) followed its
corresponding call to rescanLiteral. Otherwise, unquoting a literal
might use d.safeUnquote from another re-scanned literal.
Unfortunately, this assumption is wrong. When decoding {"foo": "bar"}
into a map[T]string where T implements TextUnmarshaler, the sequence of
calls would be as follows:
1) rescanLiteral "foo"
2) unquoteBytes "foo"
3) rescanLiteral "bar"
4) unquoteBytes "foo" (for UnmarshalText)
5) unquoteBytes "bar"
Note that the call to UnmarshalText happens in literalStore, which
repeats the work to unquote the input string literal. But, since that
happens after we've re-scanned "bar", we're using the wrong safeUnquote
field value.
In the added test case, the second string had a non-zero number of safe
bytes, and the first string had none since it was all non-ASCII. Thus,
"safely" unquoting a number of the first string's bytes could cut a rune
in half, and thus mangle the runes.
A rather simple fix, without a full revert, is to only allow one use of
safeUnquote per call to unquoteBytes. Each call to rescanLiteral when
we have a string is soon followed by a call to unquoteBytes, so it's no
longer possible for us to use the wrong index.
Also add a test case from #38126, which is the same underlying bug, but
affecting the ",string" option.
Before the fix, the test would fail, just like in the original two issues:
--- FAIL: TestUnmarshalRescanLiteralMangledUnquote (0.00s)
decode_test.go:2443: Key "开源" does not exist in map: map[开���:12345开源]
decode_test.go:2458: Unmarshal unexpected error: json: invalid use of ,string struct tag, trying to unmarshal "\"aaa\tbbb\"" into string
Fixes #38105.
For #38126.
Change-Id: I761e54924e9a971a4f9eaa70bbf72014bb1476e6
Reviewed-on: https://go-review.googlesource.com/c/go/+/226218
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
2020-03-27 23:56:09 +00:00
}
2023-08-24 12:49:10 -07:00
var gotT T
if err := Unmarshal ( b , & gotT ) ; err != nil {
t . Fatalf ( "Unmarshal error: %v" , err )
encoding/json: don't mangle strings in an edge case when decoding
The added comment contains some context. The original optimization
assumed that each call to unquoteBytes (or unquote) followed its
corresponding call to rescanLiteral. Otherwise, unquoting a literal
might use d.safeUnquote from another re-scanned literal.
Unfortunately, this assumption is wrong. When decoding {"foo": "bar"}
into a map[T]string where T implements TextUnmarshaler, the sequence of
calls would be as follows:
1) rescanLiteral "foo"
2) unquoteBytes "foo"
3) rescanLiteral "bar"
4) unquoteBytes "foo" (for UnmarshalText)
5) unquoteBytes "bar"
Note that the call to UnmarshalText happens in literalStore, which
repeats the work to unquote the input string literal. But, since that
happens after we've re-scanned "bar", we're using the wrong safeUnquote
field value.
In the added test case, the second string had a non-zero number of safe
bytes, and the first string had none since it was all non-ASCII. Thus,
"safely" unquoting a number of the first string's bytes could cut a rune
in half, and thus mangle the runes.
A rather simple fix, without a full revert, is to only allow one use of
safeUnquote per call to unquoteBytes. Each call to rescanLiteral when
we have a string is soon followed by a call to unquoteBytes, so it's no
longer possible for us to use the wrong index.
Also add a test case from #38126, which is the same underlying bug, but
affecting the ",string" option.
Before the fix, the test would fail, just like in the original two issues:
--- FAIL: TestUnmarshalRescanLiteralMangledUnquote (0.00s)
decode_test.go:2443: Key "开源" does not exist in map: map[开���:12345开源]
decode_test.go:2458: Unmarshal unexpected error: json: invalid use of ,string struct tag, trying to unmarshal "\"aaa\tbbb\"" into string
Fixes #38105.
For #38126.
Change-Id: I761e54924e9a971a4f9eaa70bbf72014bb1476e6
Reviewed-on: https://go-review.googlesource.com/c/go/+/226218
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
2020-03-27 23:56:09 +00:00
}
2023-08-24 12:49:10 -07:00
if gotT != wantT {
t . Errorf ( "Marshal/Unmarshal roundtrip:\n\tgot: %q\n\twant: %q" , gotT , wantT )
2019-10-10 10:19:12 +07:00
}
encoding/json: revert "avoid work when unquoting strings, take 2"
This reverts golang.org/cl/190659 and golang.org/cl/226218, minus the
regression tests in the latter.
The original work happened in golang.org/cl/151157, which was reverted
in golang.org/cl/190909 due to a crash found by fuzzing.
We tried a second time in golang.org/cl/190659, which shipped with Go
1.14. A bug was found, where strings would be mangled in certain edge
cases. The fix for that was golang.org/cl/226218, which was backported
into Go 1.14.4.
Unfortunately, a second regression was just reported in #39555, which is
a similar case of strings getting mangled when decoding under certain
conditions. It would be possible to come up with another small patch to
fix that edge case, but instead, let's just revert the entire
optimization, as it has proved to do more harm than good. Moreover, it's
hard to argue or prove that there will be no more such regressions.
However, all the work wasn't for nothing. First, we learned that the way
the decoder unquotes tokenized strings isn't simple; initially, we had
wrongly assumed that each string was unquoted exactly once and in order.
Second, we have gained a number of regression tests which will be useful
to prevent the same mistakes in the future, including the test cases we
add in this CL.
Fixes #39555.
Change-Id: I66a6919c2dd6d9789232482ba6cf3814eaa70f61
Reviewed-on: https://go-review.googlesource.com/c/go/+/237838
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Andrew Bonventre <andybons@golang.org>
2020-06-14 22:09:18 +01:00
// See golang.org/issues/39555.
input := map [ textUnmarshalerString ] string { "FOO" : "" , ` " ` : "" }
encoded , err := Marshal ( input )
if err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Marshal error: %v" , err )
encoding/json: revert "avoid work when unquoting strings, take 2"
This reverts golang.org/cl/190659 and golang.org/cl/226218, minus the
regression tests in the latter.
The original work happened in golang.org/cl/151157, which was reverted
in golang.org/cl/190909 due to a crash found by fuzzing.
We tried a second time in golang.org/cl/190659, which shipped with Go
1.14. A bug was found, where strings would be mangled in certain edge
cases. The fix for that was golang.org/cl/226218, which was backported
into Go 1.14.4.
Unfortunately, a second regression was just reported in #39555, which is
a similar case of strings getting mangled when decoding under certain
conditions. It would be possible to come up with another small patch to
fix that edge case, but instead, let's just revert the entire
optimization, as it has proved to do more harm than good. Moreover, it's
hard to argue or prove that there will be no more such regressions.
However, all the work wasn't for nothing. First, we learned that the way
the decoder unquotes tokenized strings isn't simple; initially, we had
wrongly assumed that each string was unquoted exactly once and in order.
Second, we have gained a number of regression tests which will be useful
to prevent the same mistakes in the future, including the test cases we
add in this CL.
Fixes #39555.
Change-Id: I66a6919c2dd6d9789232482ba6cf3814eaa70f61
Reviewed-on: https://go-review.googlesource.com/c/go/+/237838
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Andrew Bonventre <andybons@golang.org>
2020-06-14 22:09:18 +01:00
}
var got map [ textUnmarshalerString ] string
if err := Unmarshal ( encoded , & got ) ; err != nil {
2023-08-24 12:49:10 -07:00
t . Fatalf ( "Unmarshal error: %v" , err )
encoding/json: revert "avoid work when unquoting strings, take 2"
This reverts golang.org/cl/190659 and golang.org/cl/226218, minus the
regression tests in the latter.
The original work happened in golang.org/cl/151157, which was reverted
in golang.org/cl/190909 due to a crash found by fuzzing.
We tried a second time in golang.org/cl/190659, which shipped with Go
1.14. A bug was found, where strings would be mangled in certain edge
cases. The fix for that was golang.org/cl/226218, which was backported
into Go 1.14.4.
Unfortunately, a second regression was just reported in #39555, which is
a similar case of strings getting mangled when decoding under certain
conditions. It would be possible to come up with another small patch to
fix that edge case, but instead, let's just revert the entire
optimization, as it has proved to do more harm than good. Moreover, it's
hard to argue or prove that there will be no more such regressions.
However, all the work wasn't for nothing. First, we learned that the way
the decoder unquotes tokenized strings isn't simple; initially, we had
wrongly assumed that each string was unquoted exactly once and in order.
Second, we have gained a number of regression tests which will be useful
to prevent the same mistakes in the future, including the test cases we
add in this CL.
Fixes #39555.
Change-Id: I66a6919c2dd6d9789232482ba6cf3814eaa70f61
Reviewed-on: https://go-review.googlesource.com/c/go/+/237838
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Andrew Bonventre <andybons@golang.org>
2020-06-14 22:09:18 +01:00
}
want := map [ textUnmarshalerString ] string { "foo" : "" , ` " ` : "" }
2023-08-24 12:49:10 -07:00
if ! reflect . DeepEqual ( got , want ) {
t . Errorf ( "Marshal/Unmarshal roundtrip:\n\tgot: %q\n\twant: %q" , gotT , wantT )
encoding/json: revert "avoid work when unquoting strings, take 2"
This reverts golang.org/cl/190659 and golang.org/cl/226218, minus the
regression tests in the latter.
The original work happened in golang.org/cl/151157, which was reverted
in golang.org/cl/190909 due to a crash found by fuzzing.
We tried a second time in golang.org/cl/190659, which shipped with Go
1.14. A bug was found, where strings would be mangled in certain edge
cases. The fix for that was golang.org/cl/226218, which was backported
into Go 1.14.4.
Unfortunately, a second regression was just reported in #39555, which is
a similar case of strings getting mangled when decoding under certain
conditions. It would be possible to come up with another small patch to
fix that edge case, but instead, let's just revert the entire
optimization, as it has proved to do more harm than good. Moreover, it's
hard to argue or prove that there will be no more such regressions.
However, all the work wasn't for nothing. First, we learned that the way
the decoder unquotes tokenized strings isn't simple; initially, we had
wrongly assumed that each string was unquoted exactly once and in order.
Second, we have gained a number of regression tests which will be useful
to prevent the same mistakes in the future, including the test cases we
add in this CL.
Fixes #39555.
Change-Id: I66a6919c2dd6d9789232482ba6cf3814eaa70f61
Reviewed-on: https://go-review.googlesource.com/c/go/+/237838
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Andrew Bonventre <andybons@golang.org>
2020-06-14 22:09:18 +01:00
}
2019-10-10 10:19:12 +07:00
}
encoding/json: limit max nesting depth
Limit the maximum nesting depth when parsing to protect against stack
overflow, permitted by https://tools.ietf.org/html/rfc7159#section-9
A nesting depth limit of 10,000 was chosen to be a conservative
balance between avoiding stack overflow and avoiding impacting
legitimate JSON documents.
10,000 is less than 1% of the experimental stack depth limit
with the default stack size:
* On 64-bit systems, the default stack limit is 1GB,
which allows ~2,800,000 frames of recursive parsing
* On 32-bit systems, the default stack limit is 250MB,
which allows ~1,100,000 frames of recursive parsing
Fixes #31789
Change-Id: I4f5a90e89dcb4ab1a957ad9d02e1fa0efafaccf6
Reviewed-on: https://go-review.googlesource.com/c/go/+/199837
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
2019-10-08 13:19:48 -04:00
func TestUnmarshalMaxDepth ( t * testing . T ) {
2023-08-24 12:49:10 -07:00
tests := [ ] struct {
CaseName
encoding/json: limit max nesting depth
Limit the maximum nesting depth when parsing to protect against stack
overflow, permitted by https://tools.ietf.org/html/rfc7159#section-9
A nesting depth limit of 10,000 was chosen to be a conservative
balance between avoiding stack overflow and avoiding impacting
legitimate JSON documents.
10,000 is less than 1% of the experimental stack depth limit
with the default stack size:
* On 64-bit systems, the default stack limit is 1GB,
which allows ~2,800,000 frames of recursive parsing
* On 32-bit systems, the default stack limit is 250MB,
which allows ~1,100,000 frames of recursive parsing
Fixes #31789
Change-Id: I4f5a90e89dcb4ab1a957ad9d02e1fa0efafaccf6
Reviewed-on: https://go-review.googlesource.com/c/go/+/199837
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
2019-10-08 13:19:48 -04:00
data string
errMaxDepth bool
2023-08-24 12:49:10 -07:00
} { {
CaseName : Name ( "ArrayUnderMaxNestingDepth" ) ,
data : ` { "a": ` + strings . Repeat ( ` [ ` , 10000 - 1 ) + strings . Repeat ( ` ] ` , 10000 - 1 ) + ` } ` ,
errMaxDepth : false ,
} , {
CaseName : Name ( "ArrayOverMaxNestingDepth" ) ,
data : ` { "a": ` + strings . Repeat ( ` [ ` , 10000 ) + strings . Repeat ( ` ] ` , 10000 ) + ` } ` ,
errMaxDepth : true ,
} , {
CaseName : Name ( "ArrayOverStackDepth" ) ,
data : ` { "a": ` + strings . Repeat ( ` [ ` , 3000000 ) + strings . Repeat ( ` ] ` , 3000000 ) + ` } ` ,
errMaxDepth : true ,
} , {
CaseName : Name ( "ObjectUnderMaxNestingDepth" ) ,
data : ` { "a": ` + strings . Repeat ( ` { "a": ` , 10000 - 1 ) + ` 0 ` + strings . Repeat ( ` } ` , 10000 - 1 ) + ` } ` ,
errMaxDepth : false ,
} , {
CaseName : Name ( "ObjectOverMaxNestingDepth" ) ,
data : ` { "a": ` + strings . Repeat ( ` { "a": ` , 10000 ) + ` 0 ` + strings . Repeat ( ` } ` , 10000 ) + ` } ` ,
errMaxDepth : true ,
} , {
CaseName : Name ( "ObjectOverStackDepth" ) ,
data : ` { "a": ` + strings . Repeat ( ` { "a": ` , 3000000 ) + ` 0 ` + strings . Repeat ( ` } ` , 3000000 ) + ` } ` ,
errMaxDepth : true ,
} }
encoding/json: limit max nesting depth
Limit the maximum nesting depth when parsing to protect against stack
overflow, permitted by https://tools.ietf.org/html/rfc7159#section-9
A nesting depth limit of 10,000 was chosen to be a conservative
balance between avoiding stack overflow and avoiding impacting
legitimate JSON documents.
10,000 is less than 1% of the experimental stack depth limit
with the default stack size:
* On 64-bit systems, the default stack limit is 1GB,
which allows ~2,800,000 frames of recursive parsing
* On 32-bit systems, the default stack limit is 250MB,
which allows ~1,100,000 frames of recursive parsing
Fixes #31789
Change-Id: I4f5a90e89dcb4ab1a957ad9d02e1fa0efafaccf6
Reviewed-on: https://go-review.googlesource.com/c/go/+/199837
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
2019-10-08 13:19:48 -04:00
targets := [ ] struct {
2023-08-24 12:49:10 -07:00
CaseName
2021-12-01 12:15:45 -05:00
newValue func ( ) any
2023-08-24 12:49:10 -07:00
} { {
CaseName : Name ( "unstructured" ) ,
newValue : func ( ) any {
var v any
return & v
encoding/json: limit max nesting depth
Limit the maximum nesting depth when parsing to protect against stack
overflow, permitted by https://tools.ietf.org/html/rfc7159#section-9
A nesting depth limit of 10,000 was chosen to be a conservative
balance between avoiding stack overflow and avoiding impacting
legitimate JSON documents.
10,000 is less than 1% of the experimental stack depth limit
with the default stack size:
* On 64-bit systems, the default stack limit is 1GB,
which allows ~2,800,000 frames of recursive parsing
* On 32-bit systems, the default stack limit is 250MB,
which allows ~1,100,000 frames of recursive parsing
Fixes #31789
Change-Id: I4f5a90e89dcb4ab1a957ad9d02e1fa0efafaccf6
Reviewed-on: https://go-review.googlesource.com/c/go/+/199837
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
2019-10-08 13:19:48 -04:00
} ,
2023-08-24 12:49:10 -07:00
} , {
CaseName : Name ( "typed named field" ) ,
newValue : func ( ) any {
v := struct {
A any ` json:"a" `
} { }
return & v
encoding/json: limit max nesting depth
Limit the maximum nesting depth when parsing to protect against stack
overflow, permitted by https://tools.ietf.org/html/rfc7159#section-9
A nesting depth limit of 10,000 was chosen to be a conservative
balance between avoiding stack overflow and avoiding impacting
legitimate JSON documents.
10,000 is less than 1% of the experimental stack depth limit
with the default stack size:
* On 64-bit systems, the default stack limit is 1GB,
which allows ~2,800,000 frames of recursive parsing
* On 32-bit systems, the default stack limit is 250MB,
which allows ~1,100,000 frames of recursive parsing
Fixes #31789
Change-Id: I4f5a90e89dcb4ab1a957ad9d02e1fa0efafaccf6
Reviewed-on: https://go-review.googlesource.com/c/go/+/199837
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
2019-10-08 13:19:48 -04:00
} ,
2023-08-24 12:49:10 -07:00
} , {
CaseName : Name ( "typed missing field" ) ,
newValue : func ( ) any {
v := struct {
B any ` json:"b" `
} { }
return & v
encoding/json: limit max nesting depth
Limit the maximum nesting depth when parsing to protect against stack
overflow, permitted by https://tools.ietf.org/html/rfc7159#section-9
A nesting depth limit of 10,000 was chosen to be a conservative
balance between avoiding stack overflow and avoiding impacting
legitimate JSON documents.
10,000 is less than 1% of the experimental stack depth limit
with the default stack size:
* On 64-bit systems, the default stack limit is 1GB,
which allows ~2,800,000 frames of recursive parsing
* On 32-bit systems, the default stack limit is 250MB,
which allows ~1,100,000 frames of recursive parsing
Fixes #31789
Change-Id: I4f5a90e89dcb4ab1a957ad9d02e1fa0efafaccf6
Reviewed-on: https://go-review.googlesource.com/c/go/+/199837
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
2019-10-08 13:19:48 -04:00
} ,
2023-08-24 12:49:10 -07:00
} , {
CaseName : Name ( "custom unmarshaler" ) ,
newValue : func ( ) any {
v := unmarshaler { }
return & v
encoding/json: limit max nesting depth
Limit the maximum nesting depth when parsing to protect against stack
overflow, permitted by https://tools.ietf.org/html/rfc7159#section-9
A nesting depth limit of 10,000 was chosen to be a conservative
balance between avoiding stack overflow and avoiding impacting
legitimate JSON documents.
10,000 is less than 1% of the experimental stack depth limit
with the default stack size:
* On 64-bit systems, the default stack limit is 1GB,
which allows ~2,800,000 frames of recursive parsing
* On 32-bit systems, the default stack limit is 250MB,
which allows ~1,100,000 frames of recursive parsing
Fixes #31789
Change-Id: I4f5a90e89dcb4ab1a957ad9d02e1fa0efafaccf6
Reviewed-on: https://go-review.googlesource.com/c/go/+/199837
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
2019-10-08 13:19:48 -04:00
} ,
2023-08-24 12:49:10 -07:00
} }
encoding/json: limit max nesting depth
Limit the maximum nesting depth when parsing to protect against stack
overflow, permitted by https://tools.ietf.org/html/rfc7159#section-9
A nesting depth limit of 10,000 was chosen to be a conservative
balance between avoiding stack overflow and avoiding impacting
legitimate JSON documents.
10,000 is less than 1% of the experimental stack depth limit
with the default stack size:
* On 64-bit systems, the default stack limit is 1GB,
which allows ~2,800,000 frames of recursive parsing
* On 32-bit systems, the default stack limit is 250MB,
which allows ~1,100,000 frames of recursive parsing
Fixes #31789
Change-Id: I4f5a90e89dcb4ab1a957ad9d02e1fa0efafaccf6
Reviewed-on: https://go-review.googlesource.com/c/go/+/199837
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
2019-10-08 13:19:48 -04:00
2023-08-24 12:49:10 -07:00
for _ , tt := range tests {
encoding/json: limit max nesting depth
Limit the maximum nesting depth when parsing to protect against stack
overflow, permitted by https://tools.ietf.org/html/rfc7159#section-9
A nesting depth limit of 10,000 was chosen to be a conservative
balance between avoiding stack overflow and avoiding impacting
legitimate JSON documents.
10,000 is less than 1% of the experimental stack depth limit
with the default stack size:
* On 64-bit systems, the default stack limit is 1GB,
which allows ~2,800,000 frames of recursive parsing
* On 32-bit systems, the default stack limit is 250MB,
which allows ~1,100,000 frames of recursive parsing
Fixes #31789
Change-Id: I4f5a90e89dcb4ab1a957ad9d02e1fa0efafaccf6
Reviewed-on: https://go-review.googlesource.com/c/go/+/199837
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
2019-10-08 13:19:48 -04:00
for _ , target := range targets {
2023-08-24 12:49:10 -07:00
t . Run ( target . Name + "-" + tt . Name , func ( t * testing . T ) {
err := Unmarshal ( [ ] byte ( tt . data ) , target . newValue ( ) )
if ! tt . errMaxDepth {
encoding/json: limit max nesting depth
Limit the maximum nesting depth when parsing to protect against stack
overflow, permitted by https://tools.ietf.org/html/rfc7159#section-9
A nesting depth limit of 10,000 was chosen to be a conservative
balance between avoiding stack overflow and avoiding impacting
legitimate JSON documents.
10,000 is less than 1% of the experimental stack depth limit
with the default stack size:
* On 64-bit systems, the default stack limit is 1GB,
which allows ~2,800,000 frames of recursive parsing
* On 32-bit systems, the default stack limit is 250MB,
which allows ~1,100,000 frames of recursive parsing
Fixes #31789
Change-Id: I4f5a90e89dcb4ab1a957ad9d02e1fa0efafaccf6
Reviewed-on: https://go-review.googlesource.com/c/go/+/199837
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
2019-10-08 13:19:48 -04:00
if err != nil {
2023-08-24 12:49:10 -07:00
t . Errorf ( "%s: %s: Unmarshal error: %v" , tt . Where , target . Where , err )
encoding/json: limit max nesting depth
Limit the maximum nesting depth when parsing to protect against stack
overflow, permitted by https://tools.ietf.org/html/rfc7159#section-9
A nesting depth limit of 10,000 was chosen to be a conservative
balance between avoiding stack overflow and avoiding impacting
legitimate JSON documents.
10,000 is less than 1% of the experimental stack depth limit
with the default stack size:
* On 64-bit systems, the default stack limit is 1GB,
which allows ~2,800,000 frames of recursive parsing
* On 32-bit systems, the default stack limit is 250MB,
which allows ~1,100,000 frames of recursive parsing
Fixes #31789
Change-Id: I4f5a90e89dcb4ab1a957ad9d02e1fa0efafaccf6
Reviewed-on: https://go-review.googlesource.com/c/go/+/199837
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
2019-10-08 13:19:48 -04:00
}
} else {
2023-08-24 12:49:10 -07:00
if err == nil || ! strings . Contains ( err . Error ( ) , "exceeded max depth" ) {
t . Errorf ( "%s: %s: Unmarshal error:\n\tgot: %v\n\twant: exceeded max depth" , tt . Where , target . Where , err )
encoding/json: limit max nesting depth
Limit the maximum nesting depth when parsing to protect against stack
overflow, permitted by https://tools.ietf.org/html/rfc7159#section-9
A nesting depth limit of 10,000 was chosen to be a conservative
balance between avoiding stack overflow and avoiding impacting
legitimate JSON documents.
10,000 is less than 1% of the experimental stack depth limit
with the default stack size:
* On 64-bit systems, the default stack limit is 1GB,
which allows ~2,800,000 frames of recursive parsing
* On 32-bit systems, the default stack limit is 250MB,
which allows ~1,100,000 frames of recursive parsing
Fixes #31789
Change-Id: I4f5a90e89dcb4ab1a957ad9d02e1fa0efafaccf6
Reviewed-on: https://go-review.googlesource.com/c/go/+/199837
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Daniel Martí <mvdan@mvdan.cc>
2019-10-08 13:19:48 -04:00
}
}
} )
}
}
}