2016-03-01 22:57:46 +00:00
|
|
|
|
// Copyright 2010 The Go Authors. All rights reserved.
|
2010-04-27 10:46:37 -07:00
|
|
|
|
// Use of this source code is governed by a BSD-style
|
|
|
|
|
|
// license that can be found in the LICENSE file.
|
|
|
|
|
|
|
|
|
|
|
|
package json
|
|
|
|
|
|
|
|
|
|
|
|
import (
|
|
|
|
|
|
"bytes"
|
2015-04-18 03:23:32 -04:00
|
|
|
|
"io"
|
2013-01-28 16:31:46 -08:00
|
|
|
|
"io/ioutil"
|
2015-07-27 21:33:53 -04:00
|
|
|
|
"log"
|
2012-09-18 14:22:55 -04:00
|
|
|
|
"net"
|
2015-07-27 21:33:53 -04:00
|
|
|
|
"net/http"
|
|
|
|
|
|
"net/http/httptest"
|
2010-04-27 10:46:37 -07:00
|
|
|
|
"reflect"
|
2013-01-28 16:31:46 -08:00
|
|
|
|
"strings"
|
2010-04-27 10:46:37 -07:00
|
|
|
|
"testing"
|
|
|
|
|
|
)
|
|
|
|
|
|
|
|
|
|
|
|
// Test values for the stream test.
|
|
|
|
|
|
// One of each JSON kind.
|
|
|
|
|
|
var streamTest = []interface{}{
|
2011-01-19 23:09:00 -05:00
|
|
|
|
0.1,
|
2010-04-27 10:46:37 -07:00
|
|
|
|
"hello",
|
|
|
|
|
|
nil,
|
|
|
|
|
|
true,
|
|
|
|
|
|
false,
|
|
|
|
|
|
[]interface{}{"a", "b", "c"},
|
|
|
|
|
|
map[string]interface{}{"K": "Kelvin", "ß": "long s"},
|
2011-01-19 23:09:00 -05:00
|
|
|
|
3.14, // another value to make sure something can follow map
|
2010-04-27 10:46:37 -07:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
var streamEncoded = `0.1
|
|
|
|
|
|
"hello"
|
|
|
|
|
|
null
|
|
|
|
|
|
true
|
|
|
|
|
|
false
|
|
|
|
|
|
["a","b","c"]
|
|
|
|
|
|
{"ß":"long s","K":"Kelvin"}
|
|
|
|
|
|
3.14
|
|
|
|
|
|
`
|
|
|
|
|
|
|
|
|
|
|
|
func TestEncoder(t *testing.T) {
|
|
|
|
|
|
for i := 0; i <= len(streamTest); i++ {
|
|
|
|
|
|
var buf bytes.Buffer
|
|
|
|
|
|
enc := NewEncoder(&buf)
|
2016-05-23 12:28:56 -04:00
|
|
|
|
// Check that enc.SetIndent("", "") turns off indentation.
|
|
|
|
|
|
enc.SetIndent(">", ".")
|
|
|
|
|
|
enc.SetIndent("", "")
|
2010-04-27 10:46:37 -07:00
|
|
|
|
for j, v := range streamTest[0:i] {
|
|
|
|
|
|
if err := enc.Encode(v); err != nil {
|
|
|
|
|
|
t.Fatalf("encode #%d: %v", j, err)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
if have, want := buf.String(), nlines(streamEncoded, i); have != want {
|
|
|
|
|
|
t.Errorf("encoding %d items: mismatch", i)
|
|
|
|
|
|
diff(t, []byte(have), []byte(want))
|
|
|
|
|
|
break
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2016-03-23 23:14:35 -07:00
|
|
|
|
var streamEncodedIndent = `0.1
|
|
|
|
|
|
"hello"
|
|
|
|
|
|
null
|
|
|
|
|
|
true
|
|
|
|
|
|
false
|
|
|
|
|
|
[
|
|
|
|
|
|
>."a",
|
|
|
|
|
|
>."b",
|
|
|
|
|
|
>."c"
|
|
|
|
|
|
>]
|
|
|
|
|
|
{
|
|
|
|
|
|
>."ß": "long s",
|
|
|
|
|
|
>."K": "Kelvin"
|
|
|
|
|
|
>}
|
|
|
|
|
|
3.14
|
|
|
|
|
|
`
|
|
|
|
|
|
|
|
|
|
|
|
func TestEncoderIndent(t *testing.T) {
|
|
|
|
|
|
var buf bytes.Buffer
|
|
|
|
|
|
enc := NewEncoder(&buf)
|
2016-05-23 12:28:56 -04:00
|
|
|
|
enc.SetIndent(">", ".")
|
2016-03-23 23:14:35 -07:00
|
|
|
|
for _, v := range streamTest {
|
|
|
|
|
|
enc.Encode(v)
|
|
|
|
|
|
}
|
|
|
|
|
|
if have, want := buf.String(), streamEncodedIndent; have != want {
|
|
|
|
|
|
t.Error("indented encoding mismatch")
|
|
|
|
|
|
diff(t, []byte(have), []byte(want))
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2019-07-02 23:56:41 +02:00
|
|
|
|
type strMarshaler string
|
|
|
|
|
|
|
|
|
|
|
|
func (s strMarshaler) MarshalJSON() ([]byte, error) {
|
|
|
|
|
|
return []byte(s), nil
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
type strPtrMarshaler string
|
|
|
|
|
|
|
|
|
|
|
|
func (s *strPtrMarshaler) MarshalJSON() ([]byte, error) {
|
|
|
|
|
|
return []byte(*s), nil
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2016-05-23 11:41:00 -04:00
|
|
|
|
func TestEncoderSetEscapeHTML(t *testing.T) {
|
2016-04-09 21:18:22 -07:00
|
|
|
|
var c C
|
|
|
|
|
|
var ct CText
|
encoding/json: encode struct field names ahead of time
Struct field names are static, so we can run HTMLEscape on them when
building each struct type encoder. Then, when running the struct
encoder, we can select either the original or the escaped field name to
write directly.
When the encoder is not escaping HTML, using the original string works
because neither Go struct field names nor JSON tags allow any characters
that would need to be escaped, like '"', '\\', or '\n'.
When the encoder is escaping HTML, the only difference is that '<', '>',
and '&' are allowed via JSON struct field tags, hence why we use
HTMLEscape to properly escape them.
All of the above lets us encode field names with a simple if/else and
WriteString calls, which are considerably simpler and faster than
encoding an arbitrary string.
While at it, also include the quotes and colon in these strings, to
avoid three WriteByte calls in the loop hot path.
Also added a few tests, to ensure that the behavior in these edge cases
is not broken. The output of the tests is the same if this optimization
is reverted.
name old time/op new time/op delta
CodeEncoder-4 7.12ms ± 0% 6.14ms ± 0% -13.85% (p=0.004 n=6+5)
name old speed new speed delta
CodeEncoder-4 272MB/s ± 0% 316MB/s ± 0% +16.08% (p=0.004 n=6+5)
name old alloc/op new alloc/op delta
CodeEncoder-4 91.9kB ± 0% 93.2kB ± 0% +1.43% (p=0.002 n=6+6)
name old allocs/op new allocs/op delta
CodeEncoder-4 0.00 0.00 ~ (all equal)
Updates #5683.
Change-Id: I6f6a340d0de4670799ce38cf95b2092822d2e3ef
Reviewed-on: https://go-review.googlesource.com/122460
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
2018-07-07 15:59:20 +01:00
|
|
|
|
var tagStruct struct {
|
|
|
|
|
|
Valid int `json:"<>&#! "`
|
|
|
|
|
|
Invalid int `json:"\\"`
|
|
|
|
|
|
}
|
2019-07-02 23:56:41 +02:00
|
|
|
|
|
|
|
|
|
|
// This case is particularly interesting, as we force the encoder to
|
|
|
|
|
|
// take the address of the Ptr field to use its MarshalJSON method. This
|
|
|
|
|
|
// is why the '&' is important.
|
|
|
|
|
|
marshalerStruct := &struct {
|
|
|
|
|
|
NonPtr strMarshaler
|
|
|
|
|
|
Ptr strPtrMarshaler
|
|
|
|
|
|
}{`"<str>"`, `"<str>"`}
|
|
|
|
|
|
|
2019-09-10 18:52:02 +00:00
|
|
|
|
// https://golang.org/issue/34154
|
|
|
|
|
|
stringOption := struct {
|
|
|
|
|
|
Bar string `json:"bar,string"`
|
|
|
|
|
|
}{`<html>foobar</html>`}
|
|
|
|
|
|
|
2016-04-09 21:18:22 -07:00
|
|
|
|
for _, tt := range []struct {
|
|
|
|
|
|
name string
|
|
|
|
|
|
v interface{}
|
|
|
|
|
|
wantEscape string
|
|
|
|
|
|
want string
|
|
|
|
|
|
}{
|
|
|
|
|
|
{"c", c, `"\u003c\u0026\u003e"`, `"<&>"`},
|
|
|
|
|
|
{"ct", ct, `"\"\u003c\u0026\u003e\""`, `"\"<&>\""`},
|
|
|
|
|
|
{`"<&>"`, "<&>", `"\u003c\u0026\u003e"`, `"<&>"`},
|
encoding/json: encode struct field names ahead of time
Struct field names are static, so we can run HTMLEscape on them when
building each struct type encoder. Then, when running the struct
encoder, we can select either the original or the escaped field name to
write directly.
When the encoder is not escaping HTML, using the original string works
because neither Go struct field names nor JSON tags allow any characters
that would need to be escaped, like '"', '\\', or '\n'.
When the encoder is escaping HTML, the only difference is that '<', '>',
and '&' are allowed via JSON struct field tags, hence why we use
HTMLEscape to properly escape them.
All of the above lets us encode field names with a simple if/else and
WriteString calls, which are considerably simpler and faster than
encoding an arbitrary string.
While at it, also include the quotes and colon in these strings, to
avoid three WriteByte calls in the loop hot path.
Also added a few tests, to ensure that the behavior in these edge cases
is not broken. The output of the tests is the same if this optimization
is reverted.
name old time/op new time/op delta
CodeEncoder-4 7.12ms ± 0% 6.14ms ± 0% -13.85% (p=0.004 n=6+5)
name old speed new speed delta
CodeEncoder-4 272MB/s ± 0% 316MB/s ± 0% +16.08% (p=0.004 n=6+5)
name old alloc/op new alloc/op delta
CodeEncoder-4 91.9kB ± 0% 93.2kB ± 0% +1.43% (p=0.002 n=6+6)
name old allocs/op new allocs/op delta
CodeEncoder-4 0.00 0.00 ~ (all equal)
Updates #5683.
Change-Id: I6f6a340d0de4670799ce38cf95b2092822d2e3ef
Reviewed-on: https://go-review.googlesource.com/122460
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Brad Fitzpatrick <bradfitz@golang.org>
2018-07-07 15:59:20 +01:00
|
|
|
|
{
|
|
|
|
|
|
"tagStruct", tagStruct,
|
|
|
|
|
|
`{"\u003c\u003e\u0026#! ":0,"Invalid":0}`,
|
|
|
|
|
|
`{"<>&#! ":0,"Invalid":0}`,
|
|
|
|
|
|
},
|
2019-07-02 23:56:41 +02:00
|
|
|
|
{
|
|
|
|
|
|
`"<str>"`, marshalerStruct,
|
|
|
|
|
|
`{"NonPtr":"\u003cstr\u003e","Ptr":"\u003cstr\u003e"}`,
|
|
|
|
|
|
`{"NonPtr":"<str>","Ptr":"<str>"}`,
|
|
|
|
|
|
},
|
2019-09-10 18:52:02 +00:00
|
|
|
|
{
|
|
|
|
|
|
"stringOption", stringOption,
|
encoding/json: properly encode strings with ",string" again
golang.org/cl/193604 fixed one bug when one encodes a string with the
",string" option: if SetEscapeHTML(false) is used, we should not be
using HTML escaping for the inner string encoding. The CL correctly
fixed that.
The CL also tried to speed up this edge case. By avoiding an entire new
call to Marshal, the new Issue34127 benchmark reduced its time/op by
45%, and lowered the allocs/op from 3 to 2.
However, that last optimization wasn't correct:
Since Go 1.2 every string can be marshaled to JSON without error
even if it contains invalid UTF-8 byte sequences. Therefore
there is no need to use Marshal again for the only reason of
enclosing the string in double quotes.
JSON string encoding isn't just about adding quotes and taking care of
invalid UTF-8. We also need to escape some characters, like tabs and
newlines.
The new code failed to do that. The bug resulted in the added test case
failing to roundtrip properly; before our fix here, we'd see an error:
invalid use of ,string struct tag, trying to unmarshal "\"\b\f\n\r\t\"\\\"" into string
If you pay close attention, you'll notice that the special characters
like tab and newline are only encoded once, not twice. When decoding
with the ",string" option, the outer string decode works, but the inner
string decode fails, as we are now decoding a JSON string with unescaped
special characters.
The fix we apply here isn't to go back to Marshal, as that would
re-introduce the bug with SetEscapeHTML(false). Instead, we can use a
new encode state from the pool - it results in minimal performance
impact, and even reduces allocs/op further. The performance impact seems
fair, given that we need to check the entire string for characters that
need to be escaped.
name old time/op new time/op delta
Issue34127-8 89.7ns ± 2% 100.8ns ± 1% +12.27% (p=0.000 n=8+8)
name old alloc/op new alloc/op delta
Issue34127-8 40.0B ± 0% 32.0B ± 0% -20.00% (p=0.000 n=8+8)
name old allocs/op new allocs/op delta
Issue34127-8 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=8+8)
Instead of adding another standalone test, we convert an existing
"string tag" test to be table-based, and add another test case there.
One test case from the original CL also had to be amended, due to the
same problem - when escaping '<' due to SetEscapeHTML(true), we need to
end up with double escaping, since we're using ",string".
Fixes #38173.
Change-Id: I2b0df9e4f1d3452fff74fe910e189c930dde4b5b
Reviewed-on: https://go-review.googlesource.com/c/go/+/226498
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
2020-03-31 12:20:15 +01:00
|
|
|
|
`{"bar":"\"\\u003chtml\\u003efoobar\\u003c/html\\u003e\""}`,
|
2019-09-10 18:52:02 +00:00
|
|
|
|
`{"bar":"\"<html>foobar</html>\""}`,
|
|
|
|
|
|
},
|
2016-04-09 21:18:22 -07:00
|
|
|
|
} {
|
|
|
|
|
|
var buf bytes.Buffer
|
|
|
|
|
|
enc := NewEncoder(&buf)
|
|
|
|
|
|
if err := enc.Encode(tt.v); err != nil {
|
encoding/json: properly encode strings with ",string" again
golang.org/cl/193604 fixed one bug when one encodes a string with the
",string" option: if SetEscapeHTML(false) is used, we should not be
using HTML escaping for the inner string encoding. The CL correctly
fixed that.
The CL also tried to speed up this edge case. By avoiding an entire new
call to Marshal, the new Issue34127 benchmark reduced its time/op by
45%, and lowered the allocs/op from 3 to 2.
However, that last optimization wasn't correct:
Since Go 1.2 every string can be marshaled to JSON without error
even if it contains invalid UTF-8 byte sequences. Therefore
there is no need to use Marshal again for the only reason of
enclosing the string in double quotes.
JSON string encoding isn't just about adding quotes and taking care of
invalid UTF-8. We also need to escape some characters, like tabs and
newlines.
The new code failed to do that. The bug resulted in the added test case
failing to roundtrip properly; before our fix here, we'd see an error:
invalid use of ,string struct tag, trying to unmarshal "\"\b\f\n\r\t\"\\\"" into string
If you pay close attention, you'll notice that the special characters
like tab and newline are only encoded once, not twice. When decoding
with the ",string" option, the outer string decode works, but the inner
string decode fails, as we are now decoding a JSON string with unescaped
special characters.
The fix we apply here isn't to go back to Marshal, as that would
re-introduce the bug with SetEscapeHTML(false). Instead, we can use a
new encode state from the pool - it results in minimal performance
impact, and even reduces allocs/op further. The performance impact seems
fair, given that we need to check the entire string for characters that
need to be escaped.
name old time/op new time/op delta
Issue34127-8 89.7ns ± 2% 100.8ns ± 1% +12.27% (p=0.000 n=8+8)
name old alloc/op new alloc/op delta
Issue34127-8 40.0B ± 0% 32.0B ± 0% -20.00% (p=0.000 n=8+8)
name old allocs/op new allocs/op delta
Issue34127-8 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=8+8)
Instead of adding another standalone test, we convert an existing
"string tag" test to be table-based, and add another test case there.
One test case from the original CL also had to be amended, due to the
same problem - when escaping '<' due to SetEscapeHTML(true), we need to
end up with double escaping, since we're using ",string".
Fixes #38173.
Change-Id: I2b0df9e4f1d3452fff74fe910e189c930dde4b5b
Reviewed-on: https://go-review.googlesource.com/c/go/+/226498
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
2020-03-31 12:20:15 +01:00
|
|
|
|
t.Errorf("Encode(%s): %s", tt.name, err)
|
|
|
|
|
|
continue
|
2016-04-09 21:18:22 -07:00
|
|
|
|
}
|
|
|
|
|
|
if got := strings.TrimSpace(buf.String()); got != tt.wantEscape {
|
|
|
|
|
|
t.Errorf("Encode(%s) = %#q, want %#q", tt.name, got, tt.wantEscape)
|
|
|
|
|
|
}
|
|
|
|
|
|
buf.Reset()
|
2016-05-23 11:41:00 -04:00
|
|
|
|
enc.SetEscapeHTML(false)
|
2016-04-09 21:18:22 -07:00
|
|
|
|
if err := enc.Encode(tt.v); err != nil {
|
encoding/json: properly encode strings with ",string" again
golang.org/cl/193604 fixed one bug when one encodes a string with the
",string" option: if SetEscapeHTML(false) is used, we should not be
using HTML escaping for the inner string encoding. The CL correctly
fixed that.
The CL also tried to speed up this edge case. By avoiding an entire new
call to Marshal, the new Issue34127 benchmark reduced its time/op by
45%, and lowered the allocs/op from 3 to 2.
However, that last optimization wasn't correct:
Since Go 1.2 every string can be marshaled to JSON without error
even if it contains invalid UTF-8 byte sequences. Therefore
there is no need to use Marshal again for the only reason of
enclosing the string in double quotes.
JSON string encoding isn't just about adding quotes and taking care of
invalid UTF-8. We also need to escape some characters, like tabs and
newlines.
The new code failed to do that. The bug resulted in the added test case
failing to roundtrip properly; before our fix here, we'd see an error:
invalid use of ,string struct tag, trying to unmarshal "\"\b\f\n\r\t\"\\\"" into string
If you pay close attention, you'll notice that the special characters
like tab and newline are only encoded once, not twice. When decoding
with the ",string" option, the outer string decode works, but the inner
string decode fails, as we are now decoding a JSON string with unescaped
special characters.
The fix we apply here isn't to go back to Marshal, as that would
re-introduce the bug with SetEscapeHTML(false). Instead, we can use a
new encode state from the pool - it results in minimal performance
impact, and even reduces allocs/op further. The performance impact seems
fair, given that we need to check the entire string for characters that
need to be escaped.
name old time/op new time/op delta
Issue34127-8 89.7ns ± 2% 100.8ns ± 1% +12.27% (p=0.000 n=8+8)
name old alloc/op new alloc/op delta
Issue34127-8 40.0B ± 0% 32.0B ± 0% -20.00% (p=0.000 n=8+8)
name old allocs/op new allocs/op delta
Issue34127-8 2.00 ± 0% 1.00 ± 0% -50.00% (p=0.000 n=8+8)
Instead of adding another standalone test, we convert an existing
"string tag" test to be table-based, and add another test case there.
One test case from the original CL also had to be amended, due to the
same problem - when escaping '<' due to SetEscapeHTML(true), we need to
end up with double escaping, since we're using ",string".
Fixes #38173.
Change-Id: I2b0df9e4f1d3452fff74fe910e189c930dde4b5b
Reviewed-on: https://go-review.googlesource.com/c/go/+/226498
Run-TryBot: Daniel Martí <mvdan@mvdan.cc>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Joe Tsai <thebrokentoaster@gmail.com>
2020-03-31 12:20:15 +01:00
|
|
|
|
t.Errorf("SetEscapeHTML(false) Encode(%s): %s", tt.name, err)
|
|
|
|
|
|
continue
|
2016-04-09 21:18:22 -07:00
|
|
|
|
}
|
|
|
|
|
|
if got := strings.TrimSpace(buf.String()); got != tt.want {
|
2016-05-23 11:41:00 -04:00
|
|
|
|
t.Errorf("SetEscapeHTML(false) Encode(%s) = %#q, want %#q",
|
2016-04-09 21:18:22 -07:00
|
|
|
|
tt.name, got, tt.want)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2010-04-27 10:46:37 -07:00
|
|
|
|
func TestDecoder(t *testing.T) {
|
|
|
|
|
|
for i := 0; i <= len(streamTest); i++ {
|
|
|
|
|
|
// Use stream without newlines as input,
|
|
|
|
|
|
// just to stress the decoder even more.
|
|
|
|
|
|
// Our test input does not include back-to-back numbers.
|
|
|
|
|
|
// Otherwise stripping the newlines would
|
|
|
|
|
|
// merge two adjacent JSON values.
|
|
|
|
|
|
var buf bytes.Buffer
|
|
|
|
|
|
for _, c := range nlines(streamEncoded, i) {
|
|
|
|
|
|
if c != '\n' {
|
|
|
|
|
|
buf.WriteRune(c)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
out := make([]interface{}, i)
|
|
|
|
|
|
dec := NewDecoder(&buf)
|
|
|
|
|
|
for j := range out {
|
|
|
|
|
|
if err := dec.Decode(&out[j]); err != nil {
|
|
|
|
|
|
t.Fatalf("decode #%d/%d: %v", j, i, err)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
if !reflect.DeepEqual(out, streamTest[0:i]) {
|
2010-12-07 16:42:54 -05:00
|
|
|
|
t.Errorf("decoding %d items: mismatch", i)
|
2010-04-27 10:46:37 -07:00
|
|
|
|
for j := range out {
|
|
|
|
|
|
if !reflect.DeepEqual(out[j], streamTest[j]) {
|
2010-12-07 16:42:54 -05:00
|
|
|
|
t.Errorf("#%d: have %v want %v", j, out[j], streamTest[j])
|
2010-04-27 10:46:37 -07:00
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
break
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2013-01-28 16:31:46 -08:00
|
|
|
|
func TestDecoderBuffered(t *testing.T) {
|
|
|
|
|
|
r := strings.NewReader(`{"Name": "Gopher"} extra `)
|
|
|
|
|
|
var m struct {
|
|
|
|
|
|
Name string
|
|
|
|
|
|
}
|
|
|
|
|
|
d := NewDecoder(r)
|
|
|
|
|
|
err := d.Decode(&m)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
|
}
|
|
|
|
|
|
if m.Name != "Gopher" {
|
|
|
|
|
|
t.Errorf("Name = %q; want Gopher", m.Name)
|
|
|
|
|
|
}
|
|
|
|
|
|
rest, err := ioutil.ReadAll(d.Buffered())
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
t.Fatal(err)
|
|
|
|
|
|
}
|
|
|
|
|
|
if g, w := string(rest), " extra "; g != w {
|
|
|
|
|
|
t.Errorf("Remaining = %q; want %q", g, w)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2010-04-27 10:46:37 -07:00
|
|
|
|
func nlines(s string, n int) string {
|
|
|
|
|
|
if n <= 0 {
|
|
|
|
|
|
return ""
|
|
|
|
|
|
}
|
|
|
|
|
|
for i, c := range s {
|
|
|
|
|
|
if c == '\n' {
|
|
|
|
|
|
if n--; n == 0 {
|
|
|
|
|
|
return s[0 : i+1]
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
return s
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func TestRawMessage(t *testing.T) {
|
|
|
|
|
|
var data struct {
|
|
|
|
|
|
X float64
|
2018-08-26 08:35:24 -06:00
|
|
|
|
Id RawMessage
|
2010-04-27 10:46:37 -07:00
|
|
|
|
Y float32
|
|
|
|
|
|
}
|
|
|
|
|
|
const raw = `["\u0056",null]`
|
|
|
|
|
|
const msg = `{"X":0.1,"Id":["\u0056",null],"Y":0.2}`
|
|
|
|
|
|
err := Unmarshal([]byte(msg), &data)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
t.Fatalf("Unmarshal: %v", err)
|
|
|
|
|
|
}
|
2018-08-26 08:35:24 -06:00
|
|
|
|
if string([]byte(data.Id)) != raw {
|
|
|
|
|
|
t.Fatalf("Raw mismatch: have %#q want %#q", []byte(data.Id), raw)
|
2010-04-27 10:46:37 -07:00
|
|
|
|
}
|
|
|
|
|
|
b, err := Marshal(&data)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
t.Fatalf("Marshal: %v", err)
|
|
|
|
|
|
}
|
|
|
|
|
|
if string(b) != msg {
|
|
|
|
|
|
t.Fatalf("Marshal: have %#q want %#q", b, msg)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
2011-09-19 11:50:41 -04:00
|
|
|
|
|
|
|
|
|
|
func TestNullRawMessage(t *testing.T) {
|
|
|
|
|
|
var data struct {
|
2018-08-26 08:35:24 -06:00
|
|
|
|
X float64
|
|
|
|
|
|
Id RawMessage
|
|
|
|
|
|
IdPtr *RawMessage
|
|
|
|
|
|
Y float32
|
2011-09-19 11:50:41 -04:00
|
|
|
|
}
|
2018-08-26 08:35:24 -06:00
|
|
|
|
const msg = `{"X":0.1,"Id":null,"IdPtr":null,"Y":0.2}`
|
2011-09-19 11:50:41 -04:00
|
|
|
|
err := Unmarshal([]byte(msg), &data)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
t.Fatalf("Unmarshal: %v", err)
|
|
|
|
|
|
}
|
2018-08-26 08:35:24 -06:00
|
|
|
|
if want, got := "null", string(data.Id); want != got {
|
|
|
|
|
|
t.Fatalf("Raw mismatch: have %q, want %q", got, want)
|
|
|
|
|
|
}
|
|
|
|
|
|
if data.IdPtr != nil {
|
|
|
|
|
|
t.Fatalf("Raw pointer mismatch: have non-nil, want nil")
|
2011-09-19 11:50:41 -04:00
|
|
|
|
}
|
|
|
|
|
|
b, err := Marshal(&data)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
t.Fatalf("Marshal: %v", err)
|
|
|
|
|
|
}
|
|
|
|
|
|
if string(b) != msg {
|
|
|
|
|
|
t.Fatalf("Marshal: have %#q want %#q", b, msg)
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
2012-09-18 14:22:55 -04:00
|
|
|
|
|
|
|
|
|
|
var blockingTests = []string{
|
|
|
|
|
|
`{"x": 1}`,
|
|
|
|
|
|
`[1, 2, 3]`,
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func TestBlocking(t *testing.T) {
|
|
|
|
|
|
for _, enc := range blockingTests {
|
|
|
|
|
|
r, w := net.Pipe()
|
|
|
|
|
|
go w.Write([]byte(enc))
|
|
|
|
|
|
var val interface{}
|
|
|
|
|
|
|
|
|
|
|
|
// If Decode reads beyond what w.Write writes above,
|
|
|
|
|
|
// it will block, and the test will deadlock.
|
|
|
|
|
|
if err := NewDecoder(r).Decode(&val); err != nil {
|
|
|
|
|
|
t.Errorf("decoding %s: %v", enc, err)
|
|
|
|
|
|
}
|
|
|
|
|
|
r.Close()
|
|
|
|
|
|
w.Close()
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
2013-05-14 15:50:46 -07:00
|
|
|
|
|
|
|
|
|
|
func BenchmarkEncoderEncode(b *testing.B) {
|
|
|
|
|
|
b.ReportAllocs()
|
|
|
|
|
|
type T struct {
|
|
|
|
|
|
X, Y string
|
|
|
|
|
|
}
|
|
|
|
|
|
v := &T{"foo", "bar"}
|
2017-02-10 15:48:30 -05:00
|
|
|
|
b.RunParallel(func(pb *testing.PB) {
|
|
|
|
|
|
for pb.Next() {
|
|
|
|
|
|
if err := NewEncoder(ioutil.Discard).Encode(v); err != nil {
|
|
|
|
|
|
b.Fatal(err)
|
|
|
|
|
|
}
|
2013-05-14 15:50:46 -07:00
|
|
|
|
}
|
2017-02-10 15:48:30 -05:00
|
|
|
|
})
|
2013-05-14 15:50:46 -07:00
|
|
|
|
}
|
2015-04-18 03:23:32 -04:00
|
|
|
|
|
|
|
|
|
|
type tokenStreamCase struct {
|
|
|
|
|
|
json string
|
|
|
|
|
|
expTokens []interface{}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
type decodeThis struct {
|
|
|
|
|
|
v interface{}
|
|
|
|
|
|
}
|
|
|
|
|
|
|
2019-03-05 20:44:29 +00:00
|
|
|
|
var tokenStreamCases = []tokenStreamCase{
|
2015-04-18 03:23:32 -04:00
|
|
|
|
// streaming token cases
|
|
|
|
|
|
{json: `10`, expTokens: []interface{}{float64(10)}},
|
|
|
|
|
|
{json: ` [10] `, expTokens: []interface{}{
|
|
|
|
|
|
Delim('['), float64(10), Delim(']')}},
|
|
|
|
|
|
{json: ` [false,10,"b"] `, expTokens: []interface{}{
|
|
|
|
|
|
Delim('['), false, float64(10), "b", Delim(']')}},
|
|
|
|
|
|
{json: `{ "a": 1 }`, expTokens: []interface{}{
|
|
|
|
|
|
Delim('{'), "a", float64(1), Delim('}')}},
|
|
|
|
|
|
{json: `{"a": 1, "b":"3"}`, expTokens: []interface{}{
|
|
|
|
|
|
Delim('{'), "a", float64(1), "b", "3", Delim('}')}},
|
|
|
|
|
|
{json: ` [{"a": 1},{"a": 2}] `, expTokens: []interface{}{
|
|
|
|
|
|
Delim('['),
|
|
|
|
|
|
Delim('{'), "a", float64(1), Delim('}'),
|
|
|
|
|
|
Delim('{'), "a", float64(2), Delim('}'),
|
|
|
|
|
|
Delim(']')}},
|
|
|
|
|
|
{json: `{"obj": {"a": 1}}`, expTokens: []interface{}{
|
|
|
|
|
|
Delim('{'), "obj", Delim('{'), "a", float64(1), Delim('}'),
|
|
|
|
|
|
Delim('}')}},
|
|
|
|
|
|
{json: `{"obj": [{"a": 1}]}`, expTokens: []interface{}{
|
|
|
|
|
|
Delim('{'), "obj", Delim('['),
|
|
|
|
|
|
Delim('{'), "a", float64(1), Delim('}'),
|
|
|
|
|
|
Delim(']'), Delim('}')}},
|
|
|
|
|
|
|
|
|
|
|
|
// streaming tokens with intermittent Decode()
|
|
|
|
|
|
{json: `{ "a": 1 }`, expTokens: []interface{}{
|
|
|
|
|
|
Delim('{'), "a",
|
|
|
|
|
|
decodeThis{float64(1)},
|
|
|
|
|
|
Delim('}')}},
|
|
|
|
|
|
{json: ` [ { "a" : 1 } ] `, expTokens: []interface{}{
|
|
|
|
|
|
Delim('['),
|
|
|
|
|
|
decodeThis{map[string]interface{}{"a": float64(1)}},
|
|
|
|
|
|
Delim(']')}},
|
|
|
|
|
|
{json: ` [{"a": 1},{"a": 2}] `, expTokens: []interface{}{
|
|
|
|
|
|
Delim('['),
|
|
|
|
|
|
decodeThis{map[string]interface{}{"a": float64(1)}},
|
|
|
|
|
|
decodeThis{map[string]interface{}{"a": float64(2)}},
|
|
|
|
|
|
Delim(']')}},
|
|
|
|
|
|
{json: `{ "obj" : [ { "a" : 1 } ] }`, expTokens: []interface{}{
|
|
|
|
|
|
Delim('{'), "obj", Delim('['),
|
|
|
|
|
|
decodeThis{map[string]interface{}{"a": float64(1)}},
|
|
|
|
|
|
Delim(']'), Delim('}')}},
|
|
|
|
|
|
|
|
|
|
|
|
{json: `{"obj": {"a": 1}}`, expTokens: []interface{}{
|
|
|
|
|
|
Delim('{'), "obj",
|
|
|
|
|
|
decodeThis{map[string]interface{}{"a": float64(1)}},
|
|
|
|
|
|
Delim('}')}},
|
|
|
|
|
|
{json: `{"obj": [{"a": 1}]}`, expTokens: []interface{}{
|
|
|
|
|
|
Delim('{'), "obj",
|
|
|
|
|
|
decodeThis{[]interface{}{
|
|
|
|
|
|
map[string]interface{}{"a": float64(1)},
|
|
|
|
|
|
}},
|
|
|
|
|
|
Delim('}')}},
|
|
|
|
|
|
{json: ` [{"a": 1} {"a": 2}] `, expTokens: []interface{}{
|
|
|
|
|
|
Delim('['),
|
|
|
|
|
|
decodeThis{map[string]interface{}{"a": float64(1)}},
|
2017-10-28 20:50:57 -04:00
|
|
|
|
decodeThis{&SyntaxError{"expected comma after array element", 11}},
|
2015-04-18 03:23:32 -04:00
|
|
|
|
}},
|
2017-10-28 20:50:57 -04:00
|
|
|
|
{json: `{ "` + strings.Repeat("a", 513) + `" 1 }`, expTokens: []interface{}{
|
|
|
|
|
|
Delim('{'), strings.Repeat("a", 513),
|
|
|
|
|
|
decodeThis{&SyntaxError{"expected colon after object key", 518}},
|
|
|
|
|
|
}},
|
|
|
|
|
|
{json: `{ "\a" }`, expTokens: []interface{}{
|
|
|
|
|
|
Delim('{'),
|
|
|
|
|
|
&SyntaxError{"invalid character 'a' in string escape code", 3},
|
|
|
|
|
|
}},
|
|
|
|
|
|
{json: ` \a`, expTokens: []interface{}{
|
|
|
|
|
|
&SyntaxError{"invalid character '\\\\' looking for beginning of value", 1},
|
2015-04-18 03:23:32 -04:00
|
|
|
|
}},
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
func TestDecodeInStream(t *testing.T) {
|
|
|
|
|
|
for ci, tcase := range tokenStreamCases {
|
|
|
|
|
|
|
|
|
|
|
|
dec := NewDecoder(strings.NewReader(tcase.json))
|
|
|
|
|
|
for i, etk := range tcase.expTokens {
|
|
|
|
|
|
|
|
|
|
|
|
var tk interface{}
|
|
|
|
|
|
var err error
|
|
|
|
|
|
|
|
|
|
|
|
if dt, ok := etk.(decodeThis); ok {
|
|
|
|
|
|
etk = dt.v
|
|
|
|
|
|
err = dec.Decode(&tk)
|
|
|
|
|
|
} else {
|
|
|
|
|
|
tk, err = dec.Token()
|
|
|
|
|
|
}
|
|
|
|
|
|
if experr, ok := etk.(error); ok {
|
2017-10-28 20:50:57 -04:00
|
|
|
|
if err == nil || !reflect.DeepEqual(err, experr) {
|
|
|
|
|
|
t.Errorf("case %v: Expected error %#v in %q, but was %#v", ci, experr, tcase.json, err)
|
2015-04-18 03:23:32 -04:00
|
|
|
|
}
|
|
|
|
|
|
break
|
|
|
|
|
|
} else if err == io.EOF {
|
|
|
|
|
|
t.Errorf("case %v: Unexpected EOF in %q", ci, tcase.json)
|
|
|
|
|
|
break
|
|
|
|
|
|
} else if err != nil {
|
2017-10-28 20:50:57 -04:00
|
|
|
|
t.Errorf("case %v: Unexpected error '%#v' in %q", ci, err, tcase.json)
|
2015-04-18 03:23:32 -04:00
|
|
|
|
break
|
|
|
|
|
|
}
|
|
|
|
|
|
if !reflect.DeepEqual(tk, etk) {
|
|
|
|
|
|
t.Errorf(`case %v: %q @ %v expected %T(%v) was %T(%v)`, ci, tcase.json, i, etk, etk, tk, tk)
|
|
|
|
|
|
break
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
|
|
|
|
|
}
|
2015-07-27 21:33:53 -04:00
|
|
|
|
|
2015-07-28 07:53:37 +02:00
|
|
|
|
// Test from golang.org/issue/11893
|
|
|
|
|
|
func TestHTTPDecoding(t *testing.T) {
|
|
|
|
|
|
const raw = `{ "foo": "bar" }`
|
2015-07-27 21:33:53 -04:00
|
|
|
|
|
2015-07-28 07:53:37 +02:00
|
|
|
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
2015-07-27 21:33:53 -04:00
|
|
|
|
w.Write([]byte(raw))
|
2015-07-28 07:53:37 +02:00
|
|
|
|
}))
|
2015-07-27 21:33:53 -04:00
|
|
|
|
defer ts.Close()
|
|
|
|
|
|
res, err := http.Get(ts.URL)
|
|
|
|
|
|
if err != nil {
|
|
|
|
|
|
log.Fatalf("GET failed: %v", err)
|
|
|
|
|
|
}
|
2015-07-28 07:53:37 +02:00
|
|
|
|
defer res.Body.Close()
|
2015-07-27 21:33:53 -04:00
|
|
|
|
|
|
|
|
|
|
foo := struct {
|
|
|
|
|
|
Foo string
|
|
|
|
|
|
}{}
|
|
|
|
|
|
|
2015-07-28 07:53:37 +02:00
|
|
|
|
d := NewDecoder(res.Body)
|
|
|
|
|
|
err = d.Decode(&foo)
|
2015-07-27 21:33:53 -04:00
|
|
|
|
if err != nil {
|
2015-07-28 07:53:37 +02:00
|
|
|
|
t.Fatalf("Decode: %v", err)
|
2015-07-27 21:33:53 -04:00
|
|
|
|
}
|
|
|
|
|
|
if foo.Foo != "bar" {
|
2015-07-28 07:53:37 +02:00
|
|
|
|
t.Errorf("decoded %q; want \"bar\"", foo.Foo)
|
2015-07-27 21:33:53 -04:00
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
// make sure we get the EOF the second time
|
|
|
|
|
|
err = d.Decode(&foo)
|
|
|
|
|
|
if err != io.EOF {
|
2015-07-28 07:53:37 +02:00
|
|
|
|
t.Errorf("err = %v; want io.EOF", err)
|
2015-07-27 21:33:53 -04:00
|
|
|
|
}
|
|
|
|
|
|
}
|