encoding/json/jsontext: add float32 support

This adds the following API:
	Float32
	Token.Float32
	AppendFloat

This provides helper functionality for formatting and parsing
JSON numbers encoded to only 32 bits of precision.

Note that the "json" package itself already sets the precedence
for using shorter representation for encoding float32.
The new API surfaces something that the "json" package
is already able to do.

Fixes #76430

Change-Id: I643e5a33afdadddeb706eceebf3e1e22bb58740a
Reviewed-on: https://go-review.googlesource.com/c/go/+/741041
Reviewed-by: Damien Neil <dneil@google.com>
LUCI-TryBot-Result: golang-scoped@luci-project-accounts.iam.gserviceaccount.com <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Reviewed-by: Johan Brandhorst-Satzkorn <johan.brandhorst@gmail.com>
Reviewed-by: Michael Pratt <mpratt@google.com>
This commit is contained in:
Joe Tsai 2026-02-02 17:03:15 -08:00 committed by Joseph Tsai
parent 1225feb0da
commit 02b3e0d4dd
3 changed files with 161 additions and 59 deletions

View file

@ -48,21 +48,22 @@ type Token struct {
// The Encoder accepts Tokens in either the "raw" or "exact" form.
//
// The following chart shows the possible values for each Token type:
// ╔═════════════════╦════════════╤════════════╤════════════╗
// ║ Token type ║ raw field │ str field │ num field ║
// ╠═════════════════╬════════════╪════════════╪════════════╣
// ║ null (raw) ║ "null" │ "" │ 0 ║
// ║ false (raw) ║ "false" │ "" │ 0 ║
// ║ true (raw) ║ "true" │ "" │ 0 ║
// ║ string (raw) ║ non-empty │ "" │ offset ║
// ║ string (string) ║ nil │ non-empty │ 0 ║
// ║ number (raw) ║ non-empty │ "" │ offset ║
// ║ number (float) ║ nil │ "f" │ non-zero ║
// ║ number (int64) ║ nil │ "i" │ non-zero ║
// ║ number (uint64) ║ nil │ "u" │ non-zero ║
// ║ object (delim) ║ "{" or "}" │ "" │ 0 ║
// ║ array (delim) ║ "[" or "]" │ "" │ 0 ║
// ╚═════════════════╩════════════╧════════════╧════════════╝
// ╔══════════════════╦════════════╤════════════╤════════════╗
// ║ Token type ║ raw field │ str field │ num field ║
// ╠══════════════════╬════════════╪════════════╪════════════╣
// ║ null (raw) ║ "null" │ "" │ 0 ║
// ║ false (raw) ║ "false" │ "" │ 0 ║
// ║ true (raw) ║ "true" │ "" │ 0 ║
// ║ string (raw) ║ non-empty │ "" │ offset ║
// ║ string (string) ║ nil │ non-empty │ 0 ║
// ║ number (raw) ║ non-empty │ "" │ offset ║
// ║ number (float32) ║ nil │ "F" │ non-zero ║
// ║ number (float64) ║ nil │ "f" │ non-zero ║
// ║ number (int64) ║ nil │ "i" │ non-zero ║
// ║ number (uint64) ║ nil │ "u" │ non-zero ║
// ║ object (delim) ║ "{" or "}" │ "" │ 0 ║
// ║ array (delim) ║ "[" or "]" │ "" │ 0 ║
// ╚══════════════════╩════════════╧════════════╧════════════╝
//
// Notes:
// - For tokens stored in "raw" form, the num field contains the
@ -80,11 +81,12 @@ type Token struct {
raw *decodeBuffer
// str is the unescaped JSON string if num is zero.
// Otherwise, it is "f", "i", or "u" if num should be interpreted
// as a float64, int64, or uint64, respectively.
// Otherwise, it is "F", "f", "i", or "u" if num should be interpreted
// as a float32, float64, int64, or uint64, respectively.
str string
// num is a float64, int64, or uint64 stored as a uint64 value.
// num is a float32, float64, int64, or uint64 stored as a uint64 value.
// For floating-point values, it stores the raw IEEE-754 bit-pattern.
// It is non-zero for any JSON number in the "exact" form.
num uint64
}
@ -131,7 +133,29 @@ func String(s string) Token {
return Token{str: s}
}
// Float constructs a Token representing a JSON number.
// Float32 constructs a Token representing a JSON number as
// a 32-bit floating-point number formatted according to
// ECMA-262, 10th edition, section 7.1.12.1,
// with the exception that -0 is still formatted as -0.
// The values NaN, +Inf, and -Inf will be represented
// as a JSON string with the values "NaN", "Infinity", and "-Infinity".
//
// Note that most JSON libraries and standards assume that JSON numbers
// are 64-bit floating-point numbers. Use of 32-bit precision should
// only be used if the corresponding decoder knows that
// this JSON number token is expected to only have 32-bit precision.
// For all other situations, prefer using the [Float] constructor instead.
func Float32(n float32) Token {
if n != 0 && !math.IsNaN(float64(n)) && !math.IsInf(float64(n), 0) {
return Token{str: "F", num: uint64(math.Float32bits(n))}
}
return Float(float64(n)) // handles ±0, NaN, and ±Inf
}
// Float constructs a Token representing a JSON number as
// a 64-bit floating-point number formatted according to
// ECMA-262, 10th edition, section 7.1.12.1 and RFC 8785, section 3.2.2.3.
// with the exception that -0 is still formatted as -0.
// The values NaN, +Inf, and -Inf will be represented
// as a JSON string with the values "NaN", "Infinity", and "-Infinity".
func Float(n float64) Token {
@ -265,8 +289,10 @@ func (t Token) string() (string, []byte) {
// Handle tokens that are not JSON strings for fmt.Stringer.
if t.num > 0 {
switch t.str[0] {
case 'F':
return string(jsonwire.AppendFloat(nil, float64(math.Float32frombits(uint32(t.num))), 32)), nil
case 'f':
return string(jsonwire.AppendFloat(nil, math.Float64frombits(t.num), 64)), nil
return string(jsonwire.AppendFloat(nil, float64(math.Float64frombits(uint64(t.num))), 64)), nil
case 'i':
return strconv.FormatInt(int64(t.num), 10), nil
case 'u':
@ -289,8 +315,10 @@ func (t Token) appendNumber(dst []byte, flags *jsonflags.Flags) ([]byte, error)
} else if t.num != 0 {
// Handle exact number value.
switch t.str[0] {
case 'F':
return jsonwire.AppendFloat(dst, float64(math.Float32frombits(uint32(t.num))), 32), nil
case 'f':
return jsonwire.AppendFloat(dst, math.Float64frombits(t.num), 64), nil
return jsonwire.AppendFloat(dst, float64(math.Float64frombits(uint64(t.num))), 64), nil
case 'i':
return strconv.AppendInt(dst, int64(t.num), 10), nil
case 'u':
@ -301,11 +329,35 @@ func (t Token) appendNumber(dst []byte, flags *jsonflags.Flags) ([]byte, error)
panic("invalid JSON token kind: " + t.Kind().String())
}
// Float returns the floating-point value for a JSON number.
// Float32 returns the floating-point value for a JSON number
// parsed according to 32 bits of precision.
//
// Note that most JSON libraries and standards assume that JSON numbers
// are 64-bit floating-point numbers.
// This method should only be used if the caller knows
// from other context that this token is a JSON number
// formatted only to 32 bits of precision (such as being encoded
// using the [Float32] constructor). For all other situations,
// prefer using the [Token.Float] accessor instead.
//
// It returns a NaN, +Inf, or -Inf value for any JSON string
// with the values "NaN", "Infinity", or "-Infinity".
// It panics for all other cases.
func (t Token) Float32() float32 {
return float32(t.float(32))
}
// Float returns the floating-point value for a JSON number
// parsed according to 64 bits of precision.
//
// It returns a NaN, +Inf, or -Inf value for any JSON string
// with the values "NaN", "Infinity", or "-Infinity".
// It panics for all other cases.
func (t Token) Float() float64 {
return float64(t.float(64))
}
func (t Token) float(bits int) float64 {
if raw := t.raw; raw != nil {
// Handle raw number value.
if uint64(raw.previousOffsetStart()) != t.num {
@ -313,14 +365,16 @@ func (t Token) Float() float64 {
}
buf := raw.previousBuffer()
if Kind(buf[0]).normalize() == '0' {
fv, _ := jsonwire.ParseFloat(buf, 64)
fv, _ := jsonwire.ParseFloat(buf, bits)
return fv
}
} else if t.num != 0 {
// Handle exact number value.
switch t.str[0] {
case 'F':
return float64(math.Float32frombits(uint32(t.num)))
case 'f':
return math.Float64frombits(t.num)
return float64(math.Float64frombits(uint64(t.num)))
case 'i':
return float64(int64(t.num))
case 'u':

View file

@ -33,12 +33,13 @@ func TestTokenStringAllocations(t *testing.T) {
func TestTokenAccessors(t *testing.T) {
type token struct {
Bool bool
String string
Float float64
Int int64
Uint uint64
Kind Kind
Bool bool
String string
Float32 float32
Float float64
Int int64
Uint uint64
Kind Kind
}
tests := []struct {
@ -58,35 +59,56 @@ func TestTokenAccessors(t *testing.T) {
{String(""), token{String: "", Kind: '"'}},
{String("hello, world!"), token{String: "hello, world!", Kind: '"'}},
{rawToken(`"hello, world!"`), token{String: "hello, world!", Kind: '"'}},
{Float(0), token{String: "0", Float: 0, Int: 0, Uint: 0, Kind: '0'}},
{Float(math.Copysign(0, -1)), token{String: "-0", Float: math.Copysign(0, -1), Int: 0, Uint: 0, Kind: '0'}},
{Float(math.NaN()), token{String: "NaN", Float: math.NaN(), Int: 0, Uint: 0, Kind: '"'}},
{Float(math.Inf(+1)), token{String: "Infinity", Float: math.Inf(+1), Kind: '"'}},
{Float(math.Inf(-1)), token{String: "-Infinity", Float: math.Inf(-1), Kind: '"'}},
{Int(minInt64), token{String: "-9223372036854775808", Float: minInt64, Int: minInt64, Uint: minUint64, Kind: '0'}},
{Int(minInt64 + 1), token{String: "-9223372036854775807", Float: minInt64 + 1, Int: minInt64 + 1, Uint: minUint64, Kind: '0'}},
{Int(-1), token{String: "-1", Float: -1, Int: -1, Uint: minUint64, Kind: '0'}},
{Int(0), token{String: "0", Float: 0, Int: 0, Uint: 0, Kind: '0'}},
{Int(+1), token{String: "1", Float: +1, Int: +1, Uint: +1, Kind: '0'}},
{Int(maxInt64 - 1), token{String: "9223372036854775806", Float: maxInt64 - 1, Int: maxInt64 - 1, Uint: maxInt64 - 1, Kind: '0'}},
{Int(maxInt64), token{String: "9223372036854775807", Float: maxInt64, Int: maxInt64, Uint: maxInt64, Kind: '0'}},
{Float32(float32(0)), token{String: "0", Float32: 0, Float: 0, Int: 0, Uint: 0, Kind: '0'}},
{Float32(float32(math.Copysign(0, -1))), token{String: "-0", Float32: float32(math.Copysign(0, -1)), Float: math.Copysign(0, -1), Int: 0, Uint: 0, Kind: '0'}},
{Float32(float32(math.NaN())), token{String: "NaN", Float32: float32(math.NaN()), Float: math.NaN(), Int: 0, Uint: 0, Kind: '"'}},
{Float32(float32(math.Inf(+1))), token{String: "Infinity", Float32: float32(math.Inf(+1)), Float: math.Inf(+1), Kind: '"'}},
{Float32(float32(math.Inf(-1))), token{String: "-Infinity", Float32: float32(math.Inf(-1)), Float: math.Inf(-1), Kind: '"'}},
{Float32(float32(math.Pi)), token{String: "3.1415927", Float32: math.Pi, Float: float64(float32(math.Pi)), Int: 3, Uint: 3, Kind: '0'}},
{Float(0), token{String: "0", Float32: 0, Float: 0, Int: 0, Uint: 0, Kind: '0'}},
{Float(math.Copysign(0, -1)), token{String: "-0", Float32: float32(math.Copysign(0, -1)), Float: math.Copysign(0, -1), Int: 0, Uint: 0, Kind: '0'}},
{Float(math.NaN()), token{String: "NaN", Float32: float32(math.NaN()), Float: math.NaN(), Int: 0, Uint: 0, Kind: '"'}},
{Float(math.Inf(+1)), token{String: "Infinity", Float32: float32(math.Inf(+1)), Float: math.Inf(+1), Kind: '"'}},
{Float(math.Inf(-1)), token{String: "-Infinity", Float32: float32(math.Inf(-1)), Float: math.Inf(-1), Kind: '"'}},
{Float(math.Pi), token{String: "3.141592653589793", Float32: math.Pi, Float: math.Pi, Int: 3, Uint: 3, Kind: '0'}},
{Int(minInt64), token{String: "-9223372036854775808", Float32: minInt64, Float: minInt64, Int: minInt64, Uint: minUint64, Kind: '0'}},
{Int(minInt64 + 1), token{String: "-9223372036854775807", Float32: minInt64 + 1, Float: minInt64 + 1, Int: minInt64 + 1, Uint: minUint64, Kind: '0'}},
{Int(-1), token{String: "-1", Float32: -1, Float: -1, Int: -1, Uint: minUint64, Kind: '0'}},
{Int(0), token{String: "0", Float32: 0, Float: 0, Int: 0, Uint: 0, Kind: '0'}},
{Int(+1), token{String: "1", Float32: +1, Float: +1, Int: +1, Uint: +1, Kind: '0'}},
{Int(maxInt64 - 1), token{String: "9223372036854775806", Float32: maxInt64 - 1, Float: maxInt64 - 1, Int: maxInt64 - 1, Uint: maxInt64 - 1, Kind: '0'}},
{Int(maxInt64), token{String: "9223372036854775807", Float32: maxInt64, Float: maxInt64, Int: maxInt64, Uint: maxInt64, Kind: '0'}},
{Uint(minUint64), token{String: "0", Kind: '0'}},
{Uint(minUint64 + 1), token{String: "1", Float: minUint64 + 1, Int: minUint64 + 1, Uint: minUint64 + 1, Kind: '0'}},
{Uint(maxUint64 - 1), token{String: "18446744073709551614", Float: maxUint64 - 1, Int: maxInt64, Uint: maxUint64 - 1, Kind: '0'}},
{Uint(maxUint64), token{String: "18446744073709551615", Float: maxUint64, Int: maxInt64, Uint: maxUint64, Kind: '0'}},
{rawToken(`-0`), token{String: "-0", Float: math.Copysign(0, -1), Int: 0, Uint: 0, Kind: '0'}},
{rawToken(`1e1000`), token{String: "1e1000", Float: math.MaxFloat64, Int: maxInt64, Uint: maxUint64, Kind: '0'}},
{rawToken(`-1e1000`), token{String: "-1e1000", Float: -math.MaxFloat64, Int: minInt64, Uint: minUint64, Kind: '0'}},
{rawToken(`0.1`), token{String: "0.1", Float: 0.1, Int: 0, Uint: 0, Kind: '0'}},
{rawToken(`0.5`), token{String: "0.5", Float: 0.5, Int: 0, Uint: 0, Kind: '0'}},
{rawToken(`0.9`), token{String: "0.9", Float: 0.9, Int: 0, Uint: 0, Kind: '0'}},
{rawToken(`1.1`), token{String: "1.1", Float: 1.1, Int: 1, Uint: 1, Kind: '0'}},
{rawToken(`-0.1`), token{String: "-0.1", Float: -0.1, Int: 0, Uint: 0, Kind: '0'}},
{rawToken(`-0.5`), token{String: "-0.5", Float: -0.5, Int: 0, Uint: 0, Kind: '0'}},
{rawToken(`-0.9`), token{String: "-0.9", Float: -0.9, Int: 0, Uint: 0, Kind: '0'}},
{rawToken(`-1.1`), token{String: "-1.1", Float: -1.1, Int: -1, Uint: 0, Kind: '0'}},
{rawToken(`99999999999999999999`), token{String: "99999999999999999999", Float: 1e20 - 1, Int: maxInt64, Uint: maxUint64, Kind: '0'}},
{rawToken(`-99999999999999999999`), token{String: "-99999999999999999999", Float: -1e20 - 1, Int: minInt64, Uint: minUint64, Kind: '0'}},
{Uint(minUint64 + 1), token{String: "1", Float32: minUint64 + 1, Float: minUint64 + 1, Int: minUint64 + 1, Uint: minUint64 + 1, Kind: '0'}},
{Uint(maxUint64 - 1), token{String: "18446744073709551614", Float32: maxUint64 - 1, Float: maxUint64 - 1, Int: maxInt64, Uint: maxUint64 - 1, Kind: '0'}},
{Uint(maxUint64), token{String: "18446744073709551615", Float32: maxUint64 - 1, Float: maxUint64 - 1, Int: maxInt64, Uint: maxUint64, Kind: '0'}},
{rawToken(`-0`), token{String: "-0", Float32: float32(math.Copysign(0, -1)), Float: math.Copysign(0, -1), Int: 0, Uint: 0, Kind: '0'}},
{rawToken(`1e1000`), token{String: "1e1000", Float32: math.MaxFloat32, Float: math.MaxFloat64, Int: maxInt64, Uint: maxUint64, Kind: '0'}},
{rawToken(`-1e1000`), token{String: "-1e1000", Float32: -math.MaxFloat32, Float: -math.MaxFloat64, Int: minInt64, Uint: minUint64, Kind: '0'}},
{rawToken(`0.1`), token{String: "0.1", Float32: 0.1, Float: 0.1, Int: 0, Uint: 0, Kind: '0'}},
{rawToken(`0.5`), token{String: "0.5", Float32: 0.5, Float: 0.5, Int: 0, Uint: 0, Kind: '0'}},
{rawToken(`0.9`), token{String: "0.9", Float32: 0.9, Float: 0.9, Int: 0, Uint: 0, Kind: '0'}},
{rawToken(`1.1`), token{String: "1.1", Float32: 1.1, Float: 1.1, Int: 1, Uint: 1, Kind: '0'}},
{rawToken(`-0.1`), token{String: "-0.1", Float32: -0.1, Float: -0.1, Int: 0, Uint: 0, Kind: '0'}},
{rawToken(`-0.5`), token{String: "-0.5", Float32: -0.5, Float: -0.5, Int: 0, Uint: 0, Kind: '0'}},
{rawToken(`-0.9`), token{String: "-0.9", Float32: -0.9, Float: -0.9, Int: 0, Uint: 0, Kind: '0'}},
{rawToken(`-1.1`), token{String: "-1.1", Float32: -1.1, Float: -1.1, Int: -1, Uint: 0, Kind: '0'}},
{rawToken(`99999999999999999999`), token{String: "99999999999999999999", Float32: 1e20 - 1, Float: 1e20 - 1, Int: maxInt64, Uint: maxUint64, Kind: '0'}},
{rawToken(`-99999999999999999999`), token{String: "-99999999999999999999", Float32: -1e20 - 1, Float: -1e20 - 1, Int: minInt64, Uint: minUint64, Kind: '0'}},
{rawToken(`3.1415927`), token{String: "3.1415927", Float32: math.Pi, Float: 3.1415927, Int: 3, Uint: 3, Kind: '0'}},
{rawToken(`3.141592653589793`), token{String: "3.141592653589793", Float32: math.Pi, Float: math.Pi, Int: 3, Uint: 3, Kind: '0'}},
// NOTE: There exist many raw JSON numbers where:
// float32(ParseFloat(s, 32)) != float32(ParseFloat(s, 64))
// due to issues with double rounding in opposite directions.
// This suggests the need for a Token.Float32 accessor.
{rawToken(`9000000000.0000001`), token{String: "9000000000.0000001", Float32: 9000000000.0000001, Float: 9000000000.0000001, Int: 9e9, Uint: 9e9, Kind: '0'}},
// NOTE: ±7.038531e-26 is the only 32-bit precision float where:
// f != float32(ParseFloat(FormatFloat(f, 32), 64))
// assuming FormatFloat uses ECMA-262, 10th edition, section 7.1.12.1.
{rawToken(`7.038531e-26`), token{String: "7.038531e-26", Float32: 7.038531e-26, Float: 7.038531e-26, Int: 0, Uint: 0, Kind: '0'}},
{Float32(7.038531e-26), token{String: "7.038531e-26", Float32: 7.038531e-26, Float: 7.038530691851209e-26, Int: 0, Uint: 0, Kind: '0'}},
{Float(7.038531e-26), token{String: "7.038531e-26", Float32: 7.0385313e-26, Float: 7.038531e-26, Int: 0, Uint: 0, Kind: '0'}},
}
for _, tt := range tests {
@ -97,6 +119,10 @@ func TestTokenAccessors(t *testing.T) {
return tt.in.Bool()
}(),
String: tt.in.String(),
Float32: func() float32 {
defer func() { recover() }()
return tt.in.Float32()
}(),
Float: func() float64 {
defer func() { recover() }()
return tt.in.Float()
@ -118,6 +144,9 @@ func TestTokenAccessors(t *testing.T) {
if got.String != tt.want.String {
t.Errorf("Token(%s).String() = %v, want %v", tt.in, got.String, tt.want.String)
}
if math.Float32bits(got.Float32) != math.Float32bits(tt.want.Float32) {
t.Errorf("Token(%s).Float32() = %v, want %v", tt.in, got.Float32, tt.want.Float32)
}
if math.Float64bits(got.Float) != math.Float64bits(tt.want.Float) {
t.Errorf("Token(%s).Float() = %v, want %v", tt.in, got.Float, tt.want.Float)
}

View file

@ -17,7 +17,24 @@ import (
"encoding/json/internal/jsonwire"
)
// NOTE: Value is analogous to v1 json.RawMessage.
// AppendFloat appends src to dst as a JSON number per RFC 8259, section 6.
//
// Except for -0, which is formatted as -0 instead of 0,
// the output is identical to ECMA-262, 10th edition, section 7.1.12.1
// and (for 64-bit precision) identical to RFC 8785, section 3.2.2.3.
// The values NaN, +Inf, and -Inf will be represented as a JSON string
// with the values "NaN", "Infinity", and "-Infinity".
//
// Note that most JSON libraries and standards assume that JSON numbers
// are 64-bit floating-point numbers. As such, prefer using 64 bits
// of precision unless the recipient can know from other context
// that the encoded number uses 32 bits of precision.
func AppendFloat(dst []byte, src float64, bits int) []byte {
if bits != 32 && bits != 64 {
panic("illegal AppendFloat bit size")
}
return jsonwire.AppendFloat(dst, src, bits)
}
// AppendFormat formats the JSON value in src and appends it to dst
// according to the specified options.
@ -35,6 +52,8 @@ func AppendFormat(dst, src []byte, opts ...Options) ([]byte, error) {
return append(dst, e.s.Buf...), nil
}
// NOTE: Value is analogous to v1 json.RawMessage.
// Value represents a single raw JSON value, which may be one of the following:
// - a JSON literal (i.e., null, true, or false)
// - a JSON string (e.g., "hello, world!")