encoding/json: add JSON streaming parse API

This change adds new methods to Decoder.

 * Decoder.Token steps through a JSON document, returning a value for each token.
 * Decoder.Decode unmarshals the entire value at the token stream's current
   position (in addition to its existing function in a stream of JSON values)

Fixes #6050.
Fixes #6499.

Change-Id: Iff283e0e7b537221ae256392aca6529f06ebe211
Reviewed-on: https://go-review.googlesource.com/9073
Reviewed-by: Russ Cox <rsc@golang.org>

src/encoding/json/example_test.go

@@ -83,6 +83,97 @@ func ExampleDecoder() {
 	// Ed: Go fmt yourself!
 }
 
+// This example uses a Decoder to decode a stream of distinct JSON values.
+func ExampleDecoder_Token() {
+	const jsonStream = `
+		{"Message": "Hello", "Array": [1, 2, 3], "Null": null, "Number": 1.234}
+	`
+	dec := json.NewDecoder(strings.NewReader(jsonStream))
+	for {
+		t, err := dec.Token()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Printf("%T: %v", t, t)
+		if dec.More() {
+			fmt.Printf(" (more)")
+		}
+		fmt.Printf("\n")
+	}
+	// Output:
+	// json.Delim: { (more)
+	// string: Message (more)
+	// string: Hello (more)
+	// string: Array (more)
+	// json.Delim: [ (more)
+	// float64: 1 (more)
+	// float64: 2 (more)
+	// float64: 3
+	// json.Delim: ] (more)
+	// string: Null (more)
+	// <nil>: <nil> (more)
+	// string: Number (more)
+	// float64: 1.234
+	// json.Delim: }
+}
+
+// This example uses a Decoder to decode a streaming array of JSON objects.
+func ExampleDecoder_Decode_stream() {
+	const jsonStream = `
+		[
+			{"Name": "Ed", "Text": "Knock knock."},
+			{"Name": "Sam", "Text": "Who's there?"},
+			{"Name": "Ed", "Text": "Go fmt."},
+			{"Name": "Sam", "Text": "Go fmt who?"},
+			{"Name": "Ed", "Text": "Go fmt yourself!"}
+		]
+	`
+	type Message struct {
+		Name, Text string
+	}
+	dec := json.NewDecoder(strings.NewReader(jsonStream))
+
+	// read open bracket
+	t, err := dec.Token()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%T: %v\n", t, t)
+
+	var m Message
+	// while the array contains values
+	for dec.More() {
+
+		// decode an array value (Message)
+		err := dec.Decode(&m)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		fmt.Printf("%v: %v\n", m.Name, m.Text)
+	}
+
+	// read closing bracket
+	t, err = dec.Token()
+	if err != nil {
+		log.Fatal(err)
+	}
+	fmt.Printf("%T: %v\n", t, t)
+
+	// Output:
+	// json.Delim: [
+	// Ed: Knock knock.
+	// Sam: Who's there?
+	// Ed: Go fmt.
+	// Sam: Go fmt who?
+	// Ed: Go fmt yourself!
+	// json.Delim: ]
+
+}
+
 // This example uses RawMessage to delay parsing part of a JSON message.
 func ExampleRawMessage() {
 	type Color struct {