Stowage/go
2
0
Fork 0
mirror of https://github.com/golang/go.git synced 2025-11-07 20:21:01 +00:00
Code Issues Projects Releases Packages Wiki Activity

encoding/gob: reduce the amount of memory allocations.

Browse source 
Benchmark results:

benchmark                              old ns/op     new ns/op     delta
BenchmarkEndToEndPipe-4                7547          7294          -3.35%
BenchmarkEndToEndByteBuffer-4          5146          5092          -1.05%
BenchmarkEndToEndSliceByteBuffer-4     552779        439768        -20.44%
BenchmarkEncodeComplex128Slice-4       266370        266184        -0.07%
BenchmarkEncodeFloat64Slice-4          111891        110258        -1.46%
BenchmarkEncodeInt32Slice-4            74482         74080         -0.54%
BenchmarkEncodeStringSlice-4           84404         84279         -0.15%
BenchmarkEncodeInterfaceSlice-4        3942925       3045995       -22.75%
BenchmarkDecodeComplex128Slice-4       451837        415282        -8.09%
BenchmarkDecodeFloat64Slice-4          283584        262558        -7.41%
BenchmarkDecodeInt32Slice-4            246571        237383        -3.73%
BenchmarkDecodeStringSlice-4           734210        479625        -34.67%
BenchmarkDecodeInterfaceSlice-4        4778225       4160935       -12.92%

benchmark                              old allocs     new allocs     delta
BenchmarkEndToEndPipe-4                3              2              -33.33%
BenchmarkEndToEndByteBuffer-4          3              2              -33.33%
BenchmarkEndToEndSliceByteBuffer-4     1002           402            -59.88%
BenchmarkEncodeComplex128Slice-4       1              1              +0.00%
BenchmarkEncodeFloat64Slice-4          1              1              +0.00%
BenchmarkEncodeInt32Slice-4            1              1              +0.00%
BenchmarkEncodeStringSlice-4           1              1              +0.00%
BenchmarkEncodeInterfaceSlice-4        3001           1              -99.97%
BenchmarkDecodeComplex128Slice-4       188            185            -1.60%
BenchmarkDecodeFloat64Slice-4          188            185            -1.60%
BenchmarkDecodeInt32Slice-4            188            185            -1.60%
BenchmarkDecodeStringSlice-4           2188           1185           -45.84%
BenchmarkDecodeInterfaceSlice-4        6197           4194           -32.32%

benchmark                              old bytes     new bytes     delta
BenchmarkEndToEndPipe-4                64            48            -25.00%
BenchmarkEndToEndByteBuffer-4          64            48            -25.00%
BenchmarkEndToEndSliceByteBuffer-4     34551         10554         -69.45%
BenchmarkEncodeComplex128Slice-4       55            55            +0.00%
BenchmarkEncodeFloat64Slice-4          33            33            +0.00%
BenchmarkEncodeInt32Slice-4            32            32            +0.00%
BenchmarkEncodeStringSlice-4           36            36            +0.00%
BenchmarkEncodeInterfaceSlice-4        144555        347           -99.76%
BenchmarkDecodeComplex128Slice-4       28240         28097         -0.51%
BenchmarkDecodeFloat64Slice-4          11840         11697         -1.21%
BenchmarkDecodeInt32Slice-4            10817         10673         -1.33%
BenchmarkDecodeStringSlice-4           56128         39985         -28.76%
BenchmarkDecodeInterfaceSlice-4        132565        100421        -24.25%

Change-Id: Ief7c7706b1f2916486ab7190b81aafbb16b70f1e
Reviewed-on: https://go-review.googlesource.com/13660
Reviewed-by: Russ Cox <rsc@golang.org>
This commit is contained in:
Aliaksandr Valialkin 2015-08-15 13:04:46 +03:00 • committed by Russ Cox
parent c4be790c0e
commit a48de745b2
7 changed files with 105 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/encoding/gob/codec_test.go
View file

@ -88,7 +88,6 @@ func verifyInt(i int64, t *testing.T) {
encState := newEncoderState(b)
encState.encodeInt(i)
decState := newDecodeState(newDecBuffer(b.Bytes()))
decState.buf = make([]byte, 8)
j := decState.decodeInt()
if i != j {
t.Errorf("Encode/Decode: sent %#x received %#x", uint64(i), uint64(j))
@ -127,7 +126,6 @@ var bytesResult = []byte{0x07, 0x05, 'h', 'e', 'l', 'l', 'o'}
func newDecodeState(buf *decBuffer) *decoderState {
d := new(decoderState)
d.b = buf
d.buf = make([]byte, uint64Size)
return d
}

9
src/encoding/gob/dec_helpers.go
View file

@ -327,11 +327,12 @@ func decStringSlice(state *decoderState, v reflect.Value, length int, ovfl error
errorf("string data too long for buffer: %d", n)
}
// Read the data.
data := make([]byte, n)
if _, err := state.b.Read(data); err != nil {
errorf("error decoding string: %s", err)
data := state.b.Bytes()
if len(data) < n {
errorf("invalid string length %d: exceeds input size %d", n, len(data))
}
slice[i] = string(data)
slice[i] = string(data[:n])
state.b.Drop(n)
}
return true
}

9
src/encoding/gob/decgen.go
View file

@ -112,11 +112,12 @@ var types = []Type{
errorf("string data too long for buffer: %d", n)
}
// Read the data.
data := make([]byte, n)
if _, err := state.b.Read(data); err != nil {
errorf("error decoding string: %s", err)
data := state.b.Bytes()
if len(data) < n {
errorf("invalid string length %d: exceeds input size %d", n, len(data))
}
slice[i] = string(data)`,
slice[i] = string(data[:n])
state.b.Drop(n)`,
},
{
"uint",

60
src/encoding/gob/decode.go
View file

@ -30,7 +30,6 @@ type decoderState struct {
// if we load a type during decode (when reading an interface value).
b *decBuffer
fieldnum int // the last field number read.
buf []byte
next *decoderState // for free list
}
@ -97,7 +96,6 @@ func (dec *Decoder) newDecoderState(buf *decBuffer) *decoderState {
if d == nil {
d = new(decoderState)
d.dec = dec
d.buf = make([]byte, uint64Size)
} else {
dec.freeList = d.next
}
@ -160,15 +158,16 @@ func (state *decoderState) decodeUint() (x uint64) {
if n > uint64Size {
error_(errBadUint)
}
width, err := state.b.Read(state.buf[0:n])
if err != nil {
error_(err)
buf := state.b.Bytes()
if len(buf) < n {
errorf("invalid uint data length %d: exceeds input size %d", n, len(buf))
}
// Don't need to check error; it's safe to loop regardless.
// Could check that the high byte is zero but it's not worth it.
for _, b := range state.buf[0:width] {
for _, b := range buf[0:n] {
x = x<<8 | uint64(b)
}
state.b.Drop(n)
return x
}
@ -397,11 +396,13 @@ func decString(i *decInstr, state *decoderState, value reflect.Value) {
errorf("bad %s slice length: %d", value.Type(), n)
}
// Read the data.
data := make([]byte, n)
if _, err := state.b.Read(data); err != nil {
errorf("error decoding string: %s", err)
data := state.b.Bytes()
if len(data) < n {
errorf("invalid string length %d: exceeds input size %d", n, len(data))
}
value.SetString(string(data))
s := string(data[:n])
state.b.Drop(n)
value.SetString(s)
}
// ignoreUint8Array skips over the data for a byte slice value with no destination.
@ -410,8 +411,11 @@ func ignoreUint8Array(i *decInstr, state *decoderState, value reflect.Value) {
if !ok {
errorf("slice length too large")
}
b := make([]byte, n)
state.b.Read(b)
bn := state.b.Len()
if bn < n {
errorf("invalid slice length %d: exceeds input size %d", n, bn)
}
state.b.Drop(n)
}
// Execution engine
@ -640,9 +644,9 @@ func (dec *Decoder) decodeInterface(ityp reflect.Type, state *decoderState, valu
if nr > uint64(state.b.Len()) {
errorf("invalid type name length %d: exceeds input size", nr)
}
b := make([]byte, nr)
state.b.Read(b)
name := string(b)
n := int(nr)
name := string(state.b.Bytes()[:n])
state.b.Drop(n)
// Allocate the destination interface value.
if name == "" {
// Copy the nil interface value to the target.
@ -689,11 +693,11 @@ func (dec *Decoder) ignoreInterface(state *decoderState) {
if !ok {
errorf("bad interface encoding: name too large for buffer")
}
b := make([]byte, n)
_, err := state.b.Read(b)
if err != nil {
error_(err)
bn := state.b.Len()
if bn < n {
errorf("invalid interface value length %d: exceeds input size %d", n, bn)
}
state.b.Drop(n)
id := dec.decodeTypeSequence(true)
if id < 0 {
error_(dec.err)
@ -714,11 +718,13 @@ func (dec *Decoder) decodeGobDecoder(ut *userTypeInfo, state *decoderState, valu
if !ok {
errorf("GobDecoder: length too large for buffer")
}
b := make([]byte, n)
_, err := state.b.Read(b)
if err != nil {
error_(err)
b := state.b.Bytes()
if len(b) < n {
errorf("GobDecoder: invalid data length %d: exceeds input size %d", n, len(b))
}
b = b[:n]
state.b.Drop(n)
var err error
// We know it's one of these.
switch ut.externalDec {
case xGob:
@ -740,11 +746,11 @@ func (dec *Decoder) ignoreGobDecoder(state *decoderState) {
if !ok {
errorf("GobDecoder: length too large for buffer")
}
b := make([]byte, n)
_, err := state.b.Read(b)
if err != nil {
error_(err)
bn := state.b.Len()
if bn < n {
errorf("GobDecoder: invalid data length %d: exceeds input size %d", n, bn)
}
state.b.Drop(n)
}
// Index by Go types.

17
src/encoding/gob/encode.go
View file

@ -10,6 +10,7 @@ import (
"encoding"
"math"
"reflect"
"sync"
)
const uint64Size = 8
@ -36,6 +37,14 @@ type encBuffer struct {
scratch [64]byte
}
var encBufferPool = sync.Pool{
New: func() interface{} {
e := new(encBuffer)
e.data = e.scratch[0:0]
return e
},
}
func (e *encBuffer) WriteByte(c byte) {
e.data = append(e.data, c)
}
@ -58,7 +67,11 @@ func (e *encBuffer) Bytes() []byte {
}
func (e *encBuffer) Reset() {
if len(e.data) >= tooBig {
e.data = e.scratch[0:0]
} else {
e.data = e.data[0:0]
}
}
func (enc *Encoder) newEncoderState(b *encBuffer) *encoderState {
@ -407,7 +420,7 @@ func (enc *Encoder) encodeInterface(b *encBuffer, iv reflect.Value) {
// Encode the value into a new buffer. Any nested type definitions
// should be written to b, before the encoded value.
enc.pushWriter(b)
data := new(encBuffer)
data := encBufferPool.Get().(*encBuffer)
data.Write(spaceForLength)
enc.encode(data, elem, ut)
if enc.err != nil {
@ -415,6 +428,8 @@ func (enc *Encoder) encodeInterface(b *encBuffer, iv reflect.Value) {
}
enc.popWriter()
enc.writeMessage(b, data)
data.Reset()
encBufferPool.Put(data)
if enc.err != nil {
error_(enc.err)
}

2
src/encoding/gob/encoder_test.go
View file

@ -978,7 +978,7 @@ var badDataTests = []badDataTest{
{"0f1000fb285d003316020735ff023a65c5", "interface encoding", nil},
{"03fffb0616fffc00f902ff02ff03bf005d02885802a311a8120228022c028ee7", "GobDecoder", nil},
// Issue 10491.
{"10fe010f020102fe01100001fe010e000016fe010d030102fe010e00010101015801fe01100000000bfe011000f85555555555555555", "length exceeds input size", nil},
{"10fe010f020102fe01100001fe010e000016fe010d030102fe010e00010101015801fe01100000000bfe011000f85555555555555555", "exceeds input size", nil},
}
// TestBadData tests that various problems caused by malformed input

45
src/encoding/gob/timing_test.go
View file

@ -127,8 +127,8 @@ func TestCountDecodeMallocs(t *testing.T) {
t.Fatal("decode:", err)
}
})
if allocs != 4 {
t.Fatalf("mallocs per decode of type Bench: %v; wanted 4\n", allocs)
if allocs != 3 {
t.Fatalf("mallocs per decode of type Bench: %v; wanted 3\n", allocs)
}
}
@ -200,6 +200,23 @@ func BenchmarkEncodeStringSlice(b *testing.B) {
}
}
func BenchmarkEncodeInterfaceSlice(b *testing.B) {
var buf bytes.Buffer
enc := NewEncoder(&buf)
a := make([]interface{}, 1000)
for i := range a {
a[i] = "now is the time"
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
buf.Reset()
err := enc.Encode(a)
if err != nil {
b.Fatal(err)
}
}
}
// benchmarkBuf is a read buffer we can reset
type benchmarkBuf struct {
offset int
@ -323,3 +340,27 @@ func BenchmarkDecodeStringSlice(b *testing.B) {
}
}
}
func BenchmarkDecodeInterfaceSlice(b *testing.B) {
var buf bytes.Buffer
enc := NewEncoder(&buf)
a := make([]interface{}, 1000)
for i := range a {
a[i] = "now is the time"
}
err := enc.Encode(a)
if err != nil {
b.Fatal(err)
}
x := make([]interface{}, 1000)
bbuf := benchmarkBuf{data: buf.Bytes()}
b.ResetTimer()
for i := 0; i < b.N; i++ {
bbuf.reset()
dec := NewDecoder(&bbuf)
err := dec.Decode(&x)
if err != nil {
b.Fatal(i, err)
}
}
}